From 6764b0a5d6d8100fb24b0cf1d6ddee9119219364 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Thu, 21 Nov 2024 12:20:39 -0300 Subject: [PATCH 1/5] feat: adjusted liquid frequency indicator --- src/libs/routes_v1/adjustedLiquidFrequency.js | 336 ++++++++++++++++++ src/libs/routes_v1/api.js | 3 + 2 files changed, 339 insertions(+) create mode 100644 src/libs/routes_v1/adjustedLiquidFrequency.js diff --git a/src/libs/routes_v1/adjustedLiquidFrequency.js b/src/libs/routes_v1/adjustedLiquidFrequency.js new file mode 100644 index 00000000..b90443ba --- /dev/null +++ b/src/libs/routes_v1/adjustedLiquidFrequency.js @@ -0,0 +1,336 @@ +/* +Copyright (C) 2024 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +const express = require('express'); + +const adjustedLiquidFrequency = express.Router(); + +const libs = `${process.cwd()}/libs`; + +const squel = require('squel'); + +const query = require(`${libs}/middlewares/query`).query; + +const multiQuery = require(`${libs}/middlewares/multiQuery`); + +const response = require(`${libs}/middlewares/response`); + +const ReqQueryFields = require(`${libs}/middlewares/reqQueryFields`); + +const id2str = require(`${libs}/middlewares/id2str`); + +const config = require(`${libs}/config`); + +const cache = require('apicache').options({ debug: config.debug, statusCodes: {include: [200]} }).middleware; + +let rqf = new ReqQueryFields(); + +adjustedLiquidFrequency.use(cache('15 day')); + +adjustedLiquidFrequency.get('/years', (req, res, next) => { + req.sql.from('pnad_novo') + .field('DISTINCT pnad_novo.ano_ref', 'year') + .where('pnad_novo.ano_ref >= 2019') + next(); +}, query, response('years')); + +adjustedLiquidFrequency.get('/income_range', (req, res, next) => { + req.result = [] + for (let i = 1; i < 8; i++) { + req.result.push({ + id: i, name: id2str.incomeRange(i) + }); + } + req.result.push({id: 10, name: id2str.incomeRange(10)}); + next(); +}, response('income_range')); + +adjustedLiquidFrequency.get('/gender', (req, res, next) => { + req.result = [] + for (let i = 1; i < 3; i++) { + req.result.push({ + id: i, name: id2str.gender(i) + }); + } + next(); +}, response('gender')); + +adjustedLiquidFrequency.get('/new_pnad_ethnic_group', (req, res, next) => { + req.result = [] + for (let i = 1; i < 6; i++) { + req.result.push({ + id: i, name: id2str.ethnicGroupNewPnad(i) + }); + } + req.result.push({id: 9, name: id2str.ethnicGroupNewPnad(9)}); + next(); +}, response('new_pnad_ethnic_group')); + +adjustedLiquidFrequency.get('/bolsa_familia', (req, res, next) => { + req.result = [] + for (let i = 1; i < 3; i++) { + req.result.push({ + id: i, name: id2str.attendsSchool(i) + }); + } + req.result.push({id: 9, name: id2str.attendsSchool(9)}); + next(); +}, response('bolsa_familia')); + +rqf.addField({ + name: 'filter', + field: false, + where: true +}).addField({ + name: 'dims', + field: true, + where: false +}).addValue({ + name: 'id', + table: 'pnad_novo', + tableField: 'id', + where: { + relation: '=', + type: 'integer', + field: 'id' + } +}).addValue({ + name: 'state', + table: 'estado', + tableField: ['id', 'nome'], + resultField: ['state_id', 'state_nome'], + where: { + relation: '=', + type: 'integer', + field: 'id', + }, + join: { + primary: 'id', + foreign: 'cod_uf', + foreignTable: 'pnad_novo' + } +}).addValue({ + name: 'state_not', + table: 'estado', + tableField: ['nome', 'id'], + resultField: ['state_name', 'state_id'], + where: { + relation: '<>', + type: 'integer', + field: 'cod_uf', + table: 'pnad_novo' + }, + join: { + primary: 'id', + foreign: 'cod_uf', + foreignTable: 'pnad_novo' + } +}).addValue({ + name: 'years_of_study', + table: 'pnad_novo', + tableField: 'anos_de_estudo', + resultField: 'years_of_study_id', + where: { + relation: '=', + type: 'integer', + field: 'anos_de_estudo' + } +}).addValue({ + name: 'bolsa_familia', + table: 'pnad_novo', + tableField: 'recebeu_rendimentos_de_programa_bolsa_familia', + resultField: 'bolsa_familia_id', + where: { + relation: '=', + type: 'integer', + field: 'recebeu_rendimentos_de_programa_bolsa_familia' + } +}).addValue({ + name: 'new_pnad_ethnic_group', + table: 'pnad_novo', + tableField: 'cor_raca', + resultField: 'new_pnad_ethnic_group_id', + where: { + relation: '=', + type: 'integer', + field: 'cor_raca' + } +}).addValue({ + name: 'income_range', + table: 'pnad_novo', + tableField: 'faixa_rendimento_aux_tx', + resultField: 'income_range_id', + where: { + relation: '=', + type: 'integer', + field: 'faixa_rendimento_aux_tx' + } +}).addValue({ + name: 'gender', + table: 'pnad_novo', + tableField: 'sexo', + resultField: 'gender_id', + where: { + relation: '=', + type: 'integer', + field: 'sexo' + } +}).addValue({ + name: 'region', + table: 'pnad_novo', + tableField: 'cod_regiao', + resultField: 'region_id', + where: { + relation: '=', + type: 'integer', + field: 'cod_regiao' + } +}).addValue({ + name: 'min_year', + table: 'pnad_novo', + tableField: 'ano_ref', + resultField: 'year', + where: { + relation: '>=', + type: 'integer', + field: 'ano_ref' + } +}).addValue({ + name: 'max_year', + table: 'pnad_novo', + tableField: '', + resultField: 'year', + where: { + relation: '<=', + type: 'integer', + field: 'ano_ref' + } +}).addField({ + name: 'search', + field: false, + where: true +}).addValueToField({ + name: 'name', + table: 'pnad_novo', + tableField: 'nome', + where: { + relation: 'LIKE', + type: 'string', + field: 'nome' + } +}, 'search').addValue({ + name: 'mesoregion', + table: 'pnad_novo', + tableField: 'mesorregiao_id', + where: { + relation: '=', + type: 'integer', + field: 'mesorregiao_id' + } +}).addValue({ + name: 'microregion', + table: 'pnad_novo', + tableField: 'microrregiao_id', + where: { + relation: '=', + type: 'integer', + field: 'microrregiao_id' + } +}); + +function matchQueries(queryPartial, queryTotal) { + let match = []; + queryTotal.forEach((result) => { + let newObj = {}; + let keys = Object.keys(result); + keys.forEach((key) => { + newObj[key] = result[key]; + }); + let index = keys.indexOf('total'); + if(index > -1) keys.splice(index, 1); + let objMatch = null; + + for(let i = 0; i < queryPartial.length; ++i) { + let partial = queryPartial[i]; + let foundMatch = true; + for(let j = 0; j < keys.length; ++j) { + let key = keys[j]; + if(partial[key] !== result[key]) { + foundMatch = false; + break; + } + } + if(foundMatch) { + objMatch = partial; + break; + } + } + + if(objMatch) { + newObj.denominator = result.total; + newObj.partial = objMatch.total; + newObj.total = (objMatch.total / result.total) * 100; + match.push(newObj); + } + }); + return match; +} + + +adjustedLiquidFrequency.get('/', rqf.parse(), rqf.build(), (req, res, next) => { + let totalPop = req.sql.clone(); + totalPop.field("ano_ref") + .field("faixa_etaria") + .field("SUM(peso_domicilio_pessoas_com_cal)", "total") + .from("pnad_novo") + .group("ano_ref") + .group("faixa_etaria"); + + // Aplicando filtros dinâmicos à subquery total_pop + totalPop = rqf.buildQuery(req, totalPop); + + // Subquery: total_apoio_freq + let totalApoioFreq = req.sql.clone(); + totalApoioFreq.field("pnad_novo.ano_ref") + .field("pnad_novo.apoio_frequencia_ajustada") + .field("total_pop.total") + .field("SUM(pnad_novo.peso_domicilio_pessoas_com_cal)", "total_freq") + .from("pnad_novo") + .join( + totalPop, + "total_pop", + "total_pop.ano_ref = pnad_novo.ano_ref AND total_pop.faixa_etaria = pnad_novo.apoio_frequencia_ajustada" + ) + .group("pnad_novo.ano_ref") + .group("pnad_novo.apoio_frequencia_ajustada") + .group("total_pop.total"); + + req.sql.field("ano_ref") + .field("apoio_frequencia_ajustada") + .field("round((total_freq / total) * 100,1)", "total") + .from(totalApoioFreq, "total_apoio_freq") + .where("ano_ref >= 2019") + .order("ano_ref") + .order("apoio_frequencia_ajustada"); + + next(); +}, query, id2str.transform(false), response('adjusted_liquid_frequency')); + +module.exports = adjustedLiquidFrequency; diff --git a/src/libs/routes_v1/api.js b/src/libs/routes_v1/api.js index 12d89282..6b6e1c99 100644 --- a/src/libs/routes_v1/api.js +++ b/src/libs/routes_v1/api.js @@ -159,6 +159,8 @@ const uniLocalOfferAggregate = require(`${libs}/routes_v1/uniLocalOfferAggregate const basicEducationConclusion = require(`${libs}/routes_v1/basicEducationConclusion`); +const adjustedLiquidFrequency = require(`${libs}/routes_v1/adjustedLiquidFrequency`); + api.get('/', (req, res) => { res.json({ msg: 'SimCAQ API v1 is running' }); }); @@ -226,6 +228,7 @@ api.use('/course_aggregate', courseAggregate); api.use('/federativeEntity', federativeEntity); api.use('/uni_offer_aggregate', uniLocalOfferAggregate); api.use('/basic_education_conclusion', basicEducationConclusion); +api.use('/adjusted_liquid_frequency', adjustedLiquidFrequency); //Publication api.use('/publication', publication); -- GitLab From b672b91b97119617985462401522663b7a91fa4f Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Wed, 11 Dec 2024 11:00:14 -0300 Subject: [PATCH 2/5] maybe working --- src/libs/routes_v1/adjustedLiquidFrequency.js | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/src/libs/routes_v1/adjustedLiquidFrequency.js b/src/libs/routes_v1/adjustedLiquidFrequency.js index b90443ba..a682dac2 100644 --- a/src/libs/routes_v1/adjustedLiquidFrequency.js +++ b/src/libs/routes_v1/adjustedLiquidFrequency.js @@ -182,6 +182,16 @@ rqf.addField({ type: 'integer', field: 'faixa_rendimento_aux_tx' } +}).addValue({ + name: 'age_range_all', + table: 'pnad_novo', + tableField: 'faixa_etaria', + resultField: 'age_range_all_id', + where: { + relation: '=', + type: 'integer', + field: 'faixa_etaria' + } }).addValue({ name: 'gender', table: 'pnad_novo', @@ -294,23 +304,24 @@ function matchQueries(queryPartial, queryTotal) { } -adjustedLiquidFrequency.get('/', rqf.parse(), rqf.build(), (req, res, next) => { - let totalPop = req.sql.clone(); - totalPop.field("ano_ref") - .field("faixa_etaria") - .field("SUM(peso_domicilio_pessoas_com_cal)", "total") - .from("pnad_novo") - .group("ano_ref") - .group("faixa_etaria"); +adjustedLiquidFrequency.get('/', rqf.parse(), (req, res, next) => { + // Subquery: total_pop + let totalPop = squel.select() + .field("ano_ref") + .field("faixa_etaria") + .field("SUM(peso_domicilio_pessoas_com_cal)", "total") + .from("pnad_novo") + .group("ano_ref") + .group("faixa_etaria"); // Aplicando filtros dinâmicos à subquery total_pop totalPop = rqf.buildQuery(req, totalPop); // Subquery: total_apoio_freq - let totalApoioFreq = req.sql.clone(); - totalApoioFreq.field("pnad_novo.ano_ref") - .field("pnad_novo.apoio_frequencia_ajustada") - .field("total_pop.total") + let totalApoioFreq = squel.select() + .field("pnad_novo.ano_ref", "ano_ref") + .field("pnad_novo.apoio_frequencia_ajustada", "apoio_frequencia_ajustada") + .field("total_pop.total", "total") .field("SUM(pnad_novo.peso_domicilio_pessoas_com_cal)", "total_freq") .from("pnad_novo") .join( @@ -322,15 +333,20 @@ adjustedLiquidFrequency.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .group("pnad_novo.apoio_frequencia_ajustada") .group("total_pop.total"); - req.sql.field("ano_ref") - .field("apoio_frequencia_ajustada") - .field("round((total_freq / total) * 100,1)", "total") + // Aplicando filtros à subquery total_apoio_freq + totalApoioFreq = rqf.buildQuery(req, totalApoioFreq); + + // Query principal + req.sql = squel.select() + .field("total_apoio_freq.ano_ref") + .field("total_apoio_freq.apoio_frequencia_ajustada") + .field("ROUND((total_apoio_freq.total_freq / total_apoio_freq.total) * 100, 1)", "total") .from(totalApoioFreq, "total_apoio_freq") - .where("ano_ref >= 2019") - .order("ano_ref") - .order("apoio_frequencia_ajustada"); + .where("total_apoio_freq.ano_ref >= 2019") // Corrigido para usar alias correto + .order("total_apoio_freq.ano_ref") + .order("total_apoio_freq.apoio_frequencia_ajustada"); - next(); + next(); }, query, id2str.transform(false), response('adjusted_liquid_frequency')); module.exports = adjustedLiquidFrequency; -- GitLab From aad36ec1943204fa5b98953f25c7d45ad3ab72cf Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Thu, 16 Jan 2025 11:25:50 -0300 Subject: [PATCH 3/5] feat: dimensions for all necessary data --- src/libs/routes_v1/adjustedLiquidFrequency.js | 109 +++++++----------- 1 file changed, 44 insertions(+), 65 deletions(-) diff --git a/src/libs/routes_v1/adjustedLiquidFrequency.js b/src/libs/routes_v1/adjustedLiquidFrequency.js index a682dac2..bb02e81e 100644 --- a/src/libs/routes_v1/adjustedLiquidFrequency.js +++ b/src/libs/routes_v1/adjustedLiquidFrequency.js @@ -142,16 +142,6 @@ rqf.addField({ foreign: 'cod_uf', foreignTable: 'pnad_novo' } -}).addValue({ - name: 'years_of_study', - table: 'pnad_novo', - tableField: 'anos_de_estudo', - resultField: 'years_of_study_id', - where: { - relation: '=', - type: 'integer', - field: 'anos_de_estudo' - } }).addValue({ name: 'bolsa_familia', table: 'pnad_novo', @@ -265,57 +255,32 @@ rqf.addField({ } }); -function matchQueries(queryPartial, queryTotal) { - let match = []; - queryTotal.forEach((result) => { - let newObj = {}; - let keys = Object.keys(result); - keys.forEach((key) => { - newObj[key] = result[key]; - }); - let index = keys.indexOf('total'); - if(index > -1) keys.splice(index, 1); - let objMatch = null; - - for(let i = 0; i < queryPartial.length; ++i) { - let partial = queryPartial[i]; - let foundMatch = true; - for(let j = 0; j < keys.length; ++j) { - let key = keys[j]; - if(partial[key] !== result[key]) { - foundMatch = false; - break; - } - } - if(foundMatch) { - objMatch = partial; - break; - } - } - - if(objMatch) { - newObj.denominator = result.total; - newObj.partial = objMatch.total; - newObj.total = (objMatch.total / result.total) * 100; - match.push(newObj); - } - }); - return match; -} - - adjustedLiquidFrequency.get('/', rqf.parse(), (req, res, next) => { + const dimensions = req.dims || {}; // Obter dimensões solicitadas pela requisição + + // Subquery: total_pop let totalPop = squel.select() - .field("ano_ref") - .field("faixa_etaria") - .field("SUM(peso_domicilio_pessoas_com_cal)", "total") - .from("pnad_novo") - .group("ano_ref") - .group("faixa_etaria"); - - // Aplicando filtros dinâmicos à subquery total_pop + .field("ano_ref") + .field("faixa_etaria") + .field("SUM(peso_domicilio_pessoas_com_cal)", "total") + .from("pnad_novo") + .where("faixa_etaria <= 6") + .group("ano_ref") + .group("faixa_etaria"); + + + // Aplicar filtros dinâmicos à subquery total_pop totalPop = rqf.buildQuery(req, totalPop); + + const joinQuery = ` + total_pop.ano_ref = pnad_novo.ano_ref AND + total_pop.faixa_etaria = pnad_novo.apoio_frequencia_ajustada + ${dimensions.gender ? " AND total_pop.gender_id = pnad_novo.sexo" : ""} + ${dimensions.bolsa_familia ? "AND total_pop.bolsa_familia_id = pnad_novo.recebeu_rendimentos_de_programa_bolsa_familia" : ""} + ${dimensions.new_pnad_ethnic_group ? "AND total_pop.new_pnad_ethnic_group_id = pnad_novo.cor_raca" : ""} + ${dimensions.income_range ? "AND total_pop.income_range_id = pnad_novo.faixa_rendimento_aux_tx" : ""} + ` // Subquery: total_apoio_freq let totalApoioFreq = squel.select() @@ -326,26 +291,40 @@ adjustedLiquidFrequency.get('/', rqf.parse(), (req, res, next) => { .from("pnad_novo") .join( totalPop, - "total_pop", - "total_pop.ano_ref = pnad_novo.ano_ref AND total_pop.faixa_etaria = pnad_novo.apoio_frequencia_ajustada" + "total_pop", + joinQuery ) .group("pnad_novo.ano_ref") .group("pnad_novo.apoio_frequencia_ajustada") .group("total_pop.total"); - // Aplicando filtros à subquery total_apoio_freq + // Aplicar filtros dinâmicos à subquery total_apoio_freq totalApoioFreq = rqf.buildQuery(req, totalApoioFreq); - // Query principal - req.sql = squel.select() + // Query Principal + let mainQuery = squel.select() + .from(totalApoioFreq, "total_apoio_freq") .field("total_apoio_freq.ano_ref") - .field("total_apoio_freq.apoio_frequencia_ajustada") + .field("total_apoio_freq.apoio_frequencia_ajustada", "age_range_all_id") .field("ROUND((total_apoio_freq.total_freq / total_apoio_freq.total) * 100, 1)", "total") - .from(totalApoioFreq, "total_apoio_freq") - .where("total_apoio_freq.ano_ref >= 2019") // Corrigido para usar alias correto + .where("total_apoio_freq.ano_ref >= 2019") .order("total_apoio_freq.ano_ref") .order("total_apoio_freq.apoio_frequencia_ajustada"); + if (dimensions.gender) + mainQuery.field("total_apoio_freq.gender_id") + + if (dimensions.bolsa_familia) + mainQuery.field("total_apoio_freq.bolsa_familia_id") + + if (dimensions.new_pnad_ethnic_group) + mainQuery.field("total_apoio_freq.new_pnad_ethnic_group_id") + + if (dimensions.income_range) + mainQuery.field("total_apoio_freq.income_range_id") + + req.sql = mainQuery; + next(); }, query, id2str.transform(false), response('adjusted_liquid_frequency')); -- GitLab From 3b74666e40b75651c2d8fcf36f780c8178a2d2a1 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Fri, 17 Jan 2025 11:09:38 -0300 Subject: [PATCH 4/5] feat: require mandatory field to be on dimensions --- src/libs/routes_v1/adjustedLiquidFrequency.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libs/routes_v1/adjustedLiquidFrequency.js b/src/libs/routes_v1/adjustedLiquidFrequency.js index bb02e81e..5f2ce8e8 100644 --- a/src/libs/routes_v1/adjustedLiquidFrequency.js +++ b/src/libs/routes_v1/adjustedLiquidFrequency.js @@ -83,6 +83,16 @@ adjustedLiquidFrequency.get('/new_pnad_ethnic_group', (req, res, next) => { next(); }, response('new_pnad_ethnic_group')); +adjustedLiquidFrequency.get('/age_range_all', (req, res, next) => { + req.result = [] + for (let i = 1; i <= 6; i++) { + req.result.push({ + id: i, name: id2str.ageRangeAll(i) + }); + } + next() +}, response('age_range_all')); + adjustedLiquidFrequency.get('/bolsa_familia', (req, res, next) => { req.result = [] for (let i = 1; i < 3; i++) { @@ -258,6 +268,10 @@ rqf.addField({ adjustedLiquidFrequency.get('/', rqf.parse(), (req, res, next) => { const dimensions = req.dims || {}; // Obter dimensões solicitadas pela requisição + if (!dimensions.age_range_all) { + req.result = { error: "age_range_all should be selected" }; + next(); + } // Subquery: total_pop let totalPop = squel.select() -- GitLab From 80d1964c6e774c04ac59e949c4b09fe7a182143e Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Mon, 20 Jan 2025 11:40:48 -0300 Subject: [PATCH 5/5] Fix: ano_ref as year on query --- src/libs/routes_v1/adjustedLiquidFrequency.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libs/routes_v1/adjustedLiquidFrequency.js b/src/libs/routes_v1/adjustedLiquidFrequency.js index 5f2ce8e8..5b79740d 100644 --- a/src/libs/routes_v1/adjustedLiquidFrequency.js +++ b/src/libs/routes_v1/adjustedLiquidFrequency.js @@ -318,7 +318,7 @@ adjustedLiquidFrequency.get('/', rqf.parse(), (req, res, next) => { // Query Principal let mainQuery = squel.select() .from(totalApoioFreq, "total_apoio_freq") - .field("total_apoio_freq.ano_ref") + .field("total_apoio_freq.ano_ref", "year") .field("total_apoio_freq.apoio_frequencia_ajustada", "age_range_all_id") .field("ROUND((total_apoio_freq.total_freq / total_apoio_freq.total) * 100, 1)", "total") .where("total_apoio_freq.ano_ref >= 2019") -- GitLab