From 24dcfc9ff79fb7ed4462ea9646b3f5b9efcc4793 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Tue, 23 Jul 2024 16:16:22 -0500 Subject: [PATCH 1/9] Adds initial file for aggragete enrollment --- src/libs/routes_v1/api.js | 3 + src/libs/routes_v1/enrollmentAggregate.js | 279 ++++++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 src/libs/routes_v1/enrollmentAggregate.js diff --git a/src/libs/routes_v1/api.js b/src/libs/routes_v1/api.js index 53f9e4df..17af8276 100644 --- a/src/libs/routes_v1/api.js +++ b/src/libs/routes_v1/api.js @@ -143,6 +143,8 @@ const activity = require(`${libs}/routes_v1/activity`); const newPnad = require(`${libs}/routes_v1/newPnad`); +const enrollmentAggregate = require(`${libs}/routes_v1/enrollmentAggregate`); + const email = require(`${libs}/routes_v1/email`); api.get('/', (req, res) => { @@ -205,6 +207,7 @@ api.use('/universityLocalOffer', universityLocalOffer); api.use('/message', message); api.use('/course_students', courseStudents); api.use('/new_pnad', newPnad); +api.use('/enrollmentAggregate', enrollmentAggregate); //Publication api.use('/publication', publication); diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js new file mode 100644 index 00000000..27a8e0fe --- /dev/null +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -0,0 +1,279 @@ +/* +Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +const express = require('express'); +const { result } = require('lodash'); + +const enrollmentAggregateApp = express.Router(); + +const libs = `${process.cwd()}/libs`; + +const log = require(`${libs}/log`)(module); + +const squel = require('squel'); + +const query = require(`${libs}/middlewares/query`).query; + +const response = require(`${libs}/middlewares/response`); + +const ReqQueryFields = require(`${libs}/middlewares/reqQueryFields`); + +const id2str = require(`${libs}/middlewares/id2str`); + +const config = require(`${libs}/config`); + +const addMissing = require(`${libs}/middlewares/addMissing`); + +const cache = require('apicache').options({ debug: config.debug, statusCodes: {include: [200]} }).middleware; + +enrollmentAggregateApp.use(cache('15 day')); + +let rqf = new ReqQueryFields(); + +rqf.addField({ + name: 'filter', + field: false, + where: true +}).addField({ + name: 'dims', + field: true, + where: false +}).addValue({ + name: 'adm_dependency', + table: 'escola', + tableField: 'dependencia_adm_id', + resultField: 'adm_dependency_id', + where: { + relation: '=', + type: 'integer', + field: 'dependencia_adm_id' + } +}).addValue({ + name: 'adm_dependency_detailed', + table: 'escola', + tableField: 'dependencia_adm_priv', + resultField: 'adm_dependency_detailed_id', + where: { + relation: '=', + type: 'integer', + field: 'dependencia_adm_priv' + } +}).addValue({ + name: 'education_level_mod', + table: 'escola', + tableField: 'etapas_mod_ensino_segmento_id', + resultField: 'education_level_mod_id', + where: { + relation: '=', + type: 'integer', + field: 'etapas_mod_ensino_segmento_id' + } +}).addValue({ + name: 'education_level_short', + table: 'escola', + tableField: 'etapa_resumida', + resultField: 'education_level_short_id', + where: { + relation: '=', + type: 'integer', + field: 'etapa_resumida' + } +}).addValue({ + name: 'region', + table: 'regiao', + tableField: ['nome', 'id'], + resultField: ['region_name', 'region_id'], + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: 'id', + foreign: 'regiao_id', + foreignTable: 'escola' + } +}).addValue({ + name: 'mesoregion', + table: 'municipio', + tableField: ['nome_mesorregiao', 'mesorregiao_id'], + resultField: ['mesoregion_name', 'mesoregion_id'], + where: { + relation: '=', + type: 'integer', + field: 'mesorregiao_id', + table: 'municipio' + }, + join: { + primary: 'id', + foreign: 'municipio_id', + foreignTable: 'escola' + } +}).addValue({ + name: 'microregion', + table: 'municipio', + tableField: ['nome_microrregiao', 'microrregiao_id'], + resultField: ['microregion_name', 'microregion_id'], + where: { + relation: '=', + type: 'integer', + field: 'microrregiao_id', + table: 'municipio' + }, + join: { + primary: 'id', + foreign: 'municipio_id', + foreignTable: 'escola' + } +}).addValueToField({ + name: 'state', + table: 'estado', + tableField: ['nome', 'id'], + resultField: ['state_name', 'state_id'], + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: 'id', + foreign: 'estado_id', + foreignTable: 'escola' + } +}, 'dims').addValueToField({ + name: 'state', + table: 'estado', + tableField: 'nome', + resultField: 'state_name', + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: 'id', + foreign: 'estado_id', + foreignTable: 'escola' + } +}, 'filter').addValueToField({ + name: 'city', + table: 'municipio', + tableField: ['nome', 'id'], + resultField: ['city_name', 'city_id'], + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: 'id', + foreign: 'municipio_id', + foreignTable: 'escola' + } +}, 'dims').addValueToField({ + name: 'city', + table: 'municipio', + tableField: 'nome', + resultField: 'city_name', + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: 'id', + foreign: 'municipio_id', + foreignTable: 'escola' + } +}, 'filter').addValueToField({ + name: 'school', + table: 'escola', + tableField: ['nome_escola', 'id'], + resultField: ['school_name', 'school_id'], + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: ['id', 'ano_censo'], + foreign: ['escola_id', 'ano_censo'], + foreignTable: 'escola' + } +}, 'dims').addValueToField({ + name: 'locale_id', + table: 'escola', + tableField: 'localizacao_id', + resultField: 'locale_id', + where: { + relation: '=', + type: 'integer', + field: 'localizacao_id' + } +}, 'dims').addValueToField({ + name: 'school_id', + table: 'escola', + tableField: 'id', + resultField: 'school_id', + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: ['id', 'ano_censo'], + foreign: ['escola_id', 'ano_censo'], + foreignTable: 'escola' + } +}, 'dims').addValueToField({ + name: 'school', + table: 'escola', + tableField: 'nome_escola', + resultField: 'school_name', + where: { + relation: '=', + type: 'integer', + field: 'id' + }, + join: { + primary: ['id', 'ano_censo'], + foreign: ['escola_id', 'ano_censo'], + foreignTable: 'escola' + } +}, 'filter').addValue({ + name: 'location', + table: 'escola', + tableField: 'localizacao_id', + resultField: 'location_id', + where: { + relation: '=', + type: 'integer', + field: 'localizacao_id' + } +}); + +// Return all cities +enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { + req.sql.from('escola') + .field('SUM(escola.qt_mat_bas)', 'total') + next(); +}, query, response('enrollment_aggregate')); + +module.exports = enrollmentAggregateApp; -- GitLab From 892a5cf7fe45e16ef941d8ab7e3a00bb2167f8cf Mon Sep 17 00:00:00 2001 From: ems19 <ems19@inf.ufpr.br> Date: Tue, 23 Jul 2024 19:09:58 -0300 Subject: [PATCH 2/9] adds year filter --- src/libs/routes_v1/enrollmentAggregate.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index 27a8e0fe..78cfe4e6 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -142,6 +142,26 @@ rqf.addField({ foreign: 'municipio_id', foreignTable: 'escola' } +}).addValue({ + name: 'min_year', + table: 'escola', + tableField: 'ano_censo', + resultField: 'year', + where: { + relation: '>=', + type: 'integer', + field: 'ano_censo', + }, +}).addValue({ + name: 'max_year', + table: 'escola', + tableField: 'ano_censo', + resultField: 'year', + where: { + relation: '<=', + type: 'integer', + field: 'ano_censo', + }, }).addValueToField({ name: 'state', table: 'estado', @@ -273,6 +293,9 @@ rqf.addField({ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); next(); }, query, response('enrollment_aggregate')); -- GitLab From 7dc4874fef7ed3fcfb6a68368bdd1a439933d999 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Tue, 30 Jul 2024 10:17:21 -0500 Subject: [PATCH 3/9] trying --- src/libs/routes_v1/enrollmentAggregate.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index 78cfe4e6..e5b877f0 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -291,6 +291,19 @@ rqf.addField({ // Return all cities enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { + + console.log(req.query); + + if (req.query.dims.includes('gender')) + { + req.sql.from('escola') + .field('SUM(escola.qt_mat_bas_mas)', 'total_mas') + .field('SUM(escola.qt_mat_bas_fem)', 'total_fem') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); + } + req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') .field('escola.ano_censo', 'year') -- GitLab From 06b9f3d31aa6a3d2b9f76a0e3bb49f07562bb56d Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Tue, 30 Jul 2024 14:41:22 -0300 Subject: [PATCH 4/9] praying --- src/libs/middlewares/aggregateData.js | 41 +++++++++++++++++++++++ src/libs/routes_v1/enrollmentAggregate.js | 21 +++++++----- 2 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 src/libs/middlewares/aggregateData.js diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js new file mode 100644 index 00000000..a7d24c42 --- /dev/null +++ b/src/libs/middlewares/aggregateData.js @@ -0,0 +1,41 @@ +const id2str = require(`./id2str`); + +function aggregateData(req, res, next) { + const newResult = [] + const aggregateFields = ['gender'] + let id; + const fields = req.query.dims.split(','); + let currentAggregateField; + let currentNonAggregateField; + fields.forEach(field => {if (aggregateFields.includes(field)) currentAggregateField = field; else currentNonAggregateField = field}); + if (currentAggregateField) { + console.log(currentAggregateField); + req.result.forEach((r) => { + id = 1; + for (const property in r) { + if (property.includes("total_")) { + const data = { + total: r[property], + year: r.year, + [`${currentAggregateField}_id`]: id, + [`${currentAggregateField}_name`]: id2str[currentAggregateField](id) + } + + if (currentNonAggregateField) { + data[`${currentNonAggregateField}_id`] = r[`${currentNonAggregateField}_id`]; + console.log(currentNonAggregateField) + data[`${currentNonAggregateField}_name`] = id2str[`${currentNonAggregateField}_id`](id); + } + + newResult.push(data) + ++id; + } + } + }) + req.result = newResult; + } + console.log(req.result); + next(); +} + +module.exports = aggregateData; \ No newline at end of file diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index e5b877f0..6f225d65 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -35,6 +35,8 @@ const response = require(`${libs}/middlewares/response`); const ReqQueryFields = require(`${libs}/middlewares/reqQueryFields`); +const aggregateData = require(`${libs}/middlewares/aggregateData`); + const id2str = require(`${libs}/middlewares/id2str`); const config = require(`${libs}/config`); @@ -294,22 +296,23 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { console.log(req.query); - if (req.query.dims.includes('gender')) + if (req.query.dims && req.query.dims.includes('gender')) { req.sql.from('escola') - .field('SUM(escola.qt_mat_bas_mas)', 'total_mas') + .field('SUM(escola.qt_mat_bas_masc)', 'total_mas') .field('SUM(escola.qt_mat_bas_fem)', 'total_fem') .field('escola.ano_censo', 'year') .group('escola.ano_censo') .order('escola.ano_censo'); } - - req.sql.from('escola') - .field('SUM(escola.qt_mat_bas)', 'total') - .field('escola.ano_censo', 'year') - .group('escola.ano_censo') - .order('escola.ano_censo'); + else { + req.sql.from('escola') + .field('SUM(escola.qt_mat_bas)', 'total') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); + } next(); -}, query, response('enrollment_aggregate')); +}, query, aggregateData, response('enrollment_aggregate')); module.exports = enrollmentAggregateApp; -- GitLab From 95c251efdf55e2ae08927927c8a3f830d038c426 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Tue, 30 Jul 2024 16:20:17 -0300 Subject: [PATCH 5/9] Apparently working --- src/libs/middlewares/aggregateData.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index a7d24c42..58a098c9 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -1,5 +1,9 @@ const id2str = require(`./id2str`); +const convert = { + "adm_dependency_detailed": "admDependencyPriv" +} + function aggregateData(req, res, next) { const newResult = [] const aggregateFields = ['gender'] @@ -14,7 +18,7 @@ function aggregateData(req, res, next) { id = 1; for (const property in r) { if (property.includes("total_")) { - const data = { + let data = { total: r[property], year: r.year, [`${currentAggregateField}_id`]: id, @@ -23,8 +27,7 @@ function aggregateData(req, res, next) { if (currentNonAggregateField) { data[`${currentNonAggregateField}_id`] = r[`${currentNonAggregateField}_id`]; - console.log(currentNonAggregateField) - data[`${currentNonAggregateField}_name`] = id2str[`${currentNonAggregateField}_id`](id); + data[`${currentNonAggregateField}_name`] = id2str[convert[currentNonAggregateField]](r[`${currentNonAggregateField}_id`]); } newResult.push(data) -- GitLab From 85fb606618123555fdfdb0477d68e681392440e2 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Wed, 31 Jul 2024 14:27:04 -0300 Subject: [PATCH 6/9] Almost finished --- src/libs/convert/ageRangeAggregate.js | 38 +++++++++++++++++ src/libs/convert/educationLevelModAgg.js | 46 +++++++++++++++++++++ src/libs/middlewares/aggregateData.js | 12 ++++-- src/libs/middlewares/id2str.js | 7 ++++ src/libs/routes_v1/enrollmentAggregate.js | 50 +++++++++++++++++++++++ 5 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 src/libs/convert/ageRangeAggregate.js create mode 100644 src/libs/convert/educationLevelModAgg.js diff --git a/src/libs/convert/ageRangeAggregate.js b/src/libs/convert/ageRangeAggregate.js new file mode 100644 index 00000000..850f0038 --- /dev/null +++ b/src/libs/convert/ageRangeAggregate.js @@ -0,0 +1,38 @@ +/* +Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +module.exports = function ageRangeAggregate(id) { + switch (id) { + case 1: + return '0 a 3 anos'; + case 2: + return '4 a 5 anos'; + case 3: + return '6 a 10 anos'; + case 4: + return '11 a 14 anos'; + case 5: + return '15 a 17 anos'; + case 6: + return '18 anos ou mais'; + default: + return 'Não declarada'; + } +}; diff --git a/src/libs/convert/educationLevelModAgg.js b/src/libs/convert/educationLevelModAgg.js new file mode 100644 index 00000000..c55a7e8f --- /dev/null +++ b/src/libs/convert/educationLevelModAgg.js @@ -0,0 +1,46 @@ +/* +Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +module.exports = function educationLevelMod(id) { + switch (id) { + case 1: + return 'Creche'; + case 2: + return 'Pré-Escola'; + case 3: + return 'Ensino Fundamental - anos iniciais'; + case 4: + return 'Ensino Fundamental - anos finais'; + case 5: + return 'Ensino Médio'; + case 6: + return 'Ensino Medio Integrado ou Normal - tecnico'; + case 7: + return 'EJA - Ensino Fundamental'; + case 8: + return 'EJA - Ensino Médio'; + case 9: + return 'EJA - EF e EM Integrado - tecnico'; + case 10: + return 'Educacao Profissional - concomitante e subsequente'; + default: + return 'Não classificada'; + } +}; diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index 58a098c9..41c32836 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -1,12 +1,16 @@ const id2str = require(`./id2str`); const convert = { - "adm_dependency_detailed": "admDependencyPriv" + "adm_dependency_detailed": "admDependencyPriv", + "age_range": "ageRangeAggregate", + "gender": "gender", + "ethnic_group": "ethnicGroup", + "education_level_mod_agg": "educationLevelModAgg" } function aggregateData(req, res, next) { const newResult = [] - const aggregateFields = ['gender'] + const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg'] let id; const fields = req.query.dims.split(','); let currentAggregateField; @@ -15,14 +19,14 @@ function aggregateData(req, res, next) { if (currentAggregateField) { console.log(currentAggregateField); req.result.forEach((r) => { - id = 1; + id = currentAggregateField === 'ethnic_group' ? 0 : 1; for (const property in r) { if (property.includes("total_")) { let data = { total: r[property], year: r.year, [`${currentAggregateField}_id`]: id, - [`${currentAggregateField}_name`]: id2str[currentAggregateField](id) + [`${currentAggregateField}_name`]: id2str[convert[currentAggregateField]](id) } if (currentNonAggregateField) { diff --git a/src/libs/middlewares/id2str.js b/src/libs/middlewares/id2str.js index 02273a8b..0a48e4ce 100644 --- a/src/libs/middlewares/id2str.js +++ b/src/libs/middlewares/id2str.js @@ -32,6 +32,7 @@ const agreement = require(`${libs}/convert/agreement`); const booleanVariable = require(`${libs}/convert/booleanVariable`); const educationLevel = require(`${libs}/convert/educationLevel`); const educationLevelMod = require(`${libs}/convert/educationLevelMod`); +const educationLevelModAgg = require(`${libs}/convert/educationLevelModAgg`); const educationLevelShort = require(`${libs}/convert/educationLevelShort`); const educationType = require(`${libs}/convert/educationType`); const citySize = require(`${libs}/convert/citySize`); @@ -114,6 +115,8 @@ const regionCode = require(`${libs}/convert/regionCode`); const metroCode = require(`${libs}/convert/metroCode`); const modalityShift = require(`${libs}/convert/modalityShift`); const incomeRange = require(`${libs}/convert/incomeRange`); +const ageRangeAggregate = require(`${libs}/convert/ageRangeAggregate`); + const ids = { gender_id: gender, @@ -122,6 +125,7 @@ const ids = { education_level_id: educationLevel, education_level_basic_id: educationLevelBasic, education_level_mod_id: educationLevelMod, + education_level_mod_agg_id: educationLevelModAgg, education_level_short_id: educationLevelShort, adm_dependency_id: admDependency, adm_dependency_detailed_id: admDependencyPriv, @@ -149,6 +153,7 @@ const ids = { ethnic_group_pnad_id: ethnicGroupPnad, age_range_id: ageRange, age_range_all_id: ageRangeAll, + age_range_aggregate_id: ageRangeAggregate, full_age_range_id: fullAgeRange, gender_pnad_id: genderPnad, fifth_household_income_id: fifthHouseholdIncome, @@ -267,6 +272,7 @@ module.exports = { educationLevel, educationLevelBasic, educationLevelMod, + educationLevelModAgg, educationLevelShort, educationLevelSchoolYear, admDependency, @@ -284,6 +290,7 @@ module.exports = { contractType, ethnicGroupPnad, ageRange, + ageRangeAggregate, ageRangeAll, ageStudentCode, fullAgeRange, diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index 6f225d65..95de1c25 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -88,6 +88,16 @@ rqf.addField({ field: 'etapas_mod_ensino_segmento_id' } }).addValue({ + name:'integral_time', + table: 'escola', + tableField: 'tempo_integral', + resultField: 'integral_time_id', + where: { + relation: '=', + type: 'integer', + field: 'tempo_integral' + } + }).addValue({ name: 'education_level_short', table: 'escola', tableField: 'etapa_resumida', @@ -305,6 +315,46 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .group('escola.ano_censo') .order('escola.ano_censo'); } + else if (req.query.dims && req.query.dims.includes('age_range')) { + req.sql.from('escola') + .field('SUM(escola.qt_mat_bas_0_3)', 'total_0_3') + .field('SUM(escola.qt_mat_bas_4_5)', 'total_4_5') + .field('SUM(escola.qt_mat_bas_6_10)', 'total_6_10') + .field('SUM(escola.qt_mat_bas_11_14)', 'total_11_14') + .field('SUM(escola.qt_mat_bas_15_17)', 'total_15_17') + .field('SUM(escola.qt_mat_bas_18_mais)', 'total_18_mais') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); + } + else if (req.query.dims && req.query.dims.includes('ethnic_group')) { + req.sql.from('escola') + .field('SUM(escola.qt_mat_bas_nd)', 'total_nd') + .field('SUM(escola.qt_mat_bas_branca)', 'total_branca') + .field('SUM(escola.qt_mat_bas_preta)', 'total_preta') + .field('SUM(escola.qt_mat_bas_parda)', 'total_parda') + .field('SUM(escola.qt_mat_bas_amarela)', 'total_amarela') + .field('SUM(escola.qt_mat_bas_indigena)', 'total_indigena') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); + } + else if (req.query.dims && req.query.dims.includes('education_level_mod_agg')) { + req.sql.from('escola') + .field('SUM(escola.qt_mat_inf_cre)', 'total_cre') + .field('SUM(escola.qt_mat_inf_pre)', 'total_pre') + .field('SUM(escola.qt_mat_fund_ai)', 'total_fund_ai') + .field('SUM(escola.qt_mat_fund_af)', 'total_fund_af') + .field('SUM(escola.qt_mat_med_agg)', 'total_med') + .field('SUM(escola.qt_mat_med_ct + escola.qt_mat_med_nm)', 'total_med_in') + .field('SUM(escola.qt_mat_eja_fund_agg)', 'total_eja_fund') + .field('SUM(escola.qt_mat_eja_med_agg)', 'total_eja_med') + .field('SUM(escola.qt_mat_eja_fund_fic + escola.qt_mat_eja_med_fic + escola.qt_mat_eja_med_tec)', 'total_tec') + .field('SUM(escola.qt_mat_prof_agg)', 'total_prof') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo'); + } else { req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') -- GitLab From 959f9a125ac27f51a5a5ae502d3480ae6399d94d Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Thu, 1 Aug 2024 17:25:36 -0300 Subject: [PATCH 7/9] integral_time filter working --- src/libs/middlewares/aggregateData.js | 7 ++++--- src/libs/routes_v1/enrollmentAggregate.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index 41c32836..f0378d0b 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -5,12 +5,13 @@ const convert = { "age_range": "ageRangeAggregate", "gender": "gender", "ethnic_group": "ethnicGroup", - "education_level_mod_agg": "educationLevelModAgg" + "education_level_mod_agg": "educationLevelModAgg", + 'integral_time_agg': 'integralTime' } function aggregateData(req, res, next) { const newResult = [] - const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg'] + const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg', 'integral_time_agg'] let id; const fields = req.query.dims.split(','); let currentAggregateField; @@ -19,7 +20,7 @@ function aggregateData(req, res, next) { if (currentAggregateField) { console.log(currentAggregateField); req.result.forEach((r) => { - id = currentAggregateField === 'ethnic_group' ? 0 : 1; + id = ['ethnic_group', 'integral_time_agg'].includes(currentAggregateField) ? 0 : 1; for (const property in r) { if (property.includes("total_")) { let data = { diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index 95de1c25..c12329cd 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -355,6 +355,17 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .group('escola.ano_censo') .order('escola.ano_censo'); } + else if (req.query.dims && req.query.dims.includes('integral_time_agg')) { + + req.sql.from('turma,escola') + .field('SUM(CASE WHEN turma.tempo_integral=0 then turma.num_matricula ELSE 0 END)', 'total_int_nao') + .field('SUM(CASE WHEN turma.tempo_integral=1 then turma.num_matricula ELSE 0 END)', 'total_int') + .field('SUM(CASE WHEN turma.tempo_integral=2 then turma.num_matricula ELSE 0 END)', 'total_int_nao_aplica') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo') + .where('turma.tipo_atendimento_id <= 2 and turma.escola_id = escola.id and turma.ano_censo = escola.ano_censo'); + } else { req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') -- GitLab From ecd2aed68f22e46878bd42266fe8f430d7027b6c Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Thu, 1 Aug 2024 23:06:54 -0300 Subject: [PATCH 8/9] all filters completed --- src/libs/middlewares/aggregateData.js | 5 +++-- src/libs/routes_v1/enrollmentAggregate.js | 25 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index f0378d0b..265e973c 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -6,12 +6,13 @@ const convert = { "gender": "gender", "ethnic_group": "ethnicGroup", "education_level_mod_agg": "educationLevelModAgg", - 'integral_time_agg': 'integralTime' + 'integral_time_agg': 'integralTime', + 'period_agg': 'period' } function aggregateData(req, res, next) { const newResult = [] - const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg', 'integral_time_agg'] + const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg', 'integral_time_agg', 'period_agg'] let id; const fields = req.query.dims.split(','); let currentAggregateField; diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index c12329cd..feae2681 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -366,6 +366,31 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .order('escola.ano_censo') .where('turma.tipo_atendimento_id <= 2 and turma.escola_id = escola.id and turma.ano_censo = escola.ano_censo'); } + else if (req.query.dims && req.query.dims.includes('period_agg')) { + + req.sql.from('turma,escola') + .field('SUM(CASE WHEN turma.turma_turno_id=1 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_matutino') + .field('SUM(CASE WHEN turma.turma_turno_id=2 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_vespertino') + .field('SUM(CASE WHEN turma.turma_turno_id=3 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_noturno') + .field('SUM(CASE WHEN turma.turma_turno_id=4 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_integral') + .field('SUM(CASE WHEN turma.turma_turno_id=99 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_semi_ead') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo') + .where('turma.tipo_atendimento_id <= 2 and turma.escola_id = escola.id and turma.ano_censo = escola.ano_censo'); + } + else if (req.query.dims && req.query.dims.includes('modality_integral_time')) { + + req.sql.from('escola') + .field('SUM(escola.qt_mat_inf_cre_int)', 'total_qt_mat_inf_cre_int') + .field('SUM(escola.qt_mat_inf_pre_int)', 'total_qt_mat_inf_pre_int') + .field('SUM(escola.qt_mat_fund_ai_int)', 'total_qt_mat_fund_ai_int') + .field('SUM(escola.qt_mat_fund_af_int)', 'total_qt_mat_fund_af_int') + .field('SUM(escola.qt_mat_med_int)', 'total_qt_mat_med_int') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo') + } else { req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') -- GitLab From adbb6ade766216a0cc67f202e16d42c5367e67e6 Mon Sep 17 00:00:00 2001 From: Lewis <lgtg20@inf.ufpr.br> Date: Fri, 2 Aug 2024 00:22:12 -0300 Subject: [PATCH 9/9] Everything implemented --- src/libs/convert/especialEducation.js | 31 ++++ src/libs/middlewares/aggregateData.js | 19 ++- src/libs/middlewares/id2str.js | 8 +- src/libs/routes_v1/enrollmentAggregate.js | 199 +++++++++++++++++++++- 4 files changed, 247 insertions(+), 10 deletions(-) create mode 100644 src/libs/convert/especialEducation.js diff --git a/src/libs/convert/especialEducation.js b/src/libs/convert/especialEducation.js new file mode 100644 index 00000000..46349784 --- /dev/null +++ b/src/libs/convert/especialEducation.js @@ -0,0 +1,31 @@ +/* +Copyright (C) 2022 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of simcaq-node. + +simcaq-node is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +simcaq-node is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with simcaq-node. If not, see <https://www.gnu.org/licenses/>. +*/ + +module.exports = function especialEducation(id) { + switch (id) { + case 1: + return 'Inclusiva'; + case 2: + return 'Exclusiva'; + default: + return 'Não definido'; + } +}; + diff --git a/src/libs/middlewares/aggregateData.js b/src/libs/middlewares/aggregateData.js index 265e973c..a721bb1b 100644 --- a/src/libs/middlewares/aggregateData.js +++ b/src/libs/middlewares/aggregateData.js @@ -1,5 +1,12 @@ const id2str = require(`./id2str`); +/* + Middleware que tem como função formatar dados agregados. + Dados agregados não podem ser "puxados" do banco de dados da mesma maneira que os dados tradicioanis, dessa maneira é + necessário formatá-los para que sigam o padrão. +*/ + +// Faz o mapeamento dos filtros com seus respectivos "id2str" const convert = { "adm_dependency_detailed": "admDependencyPriv", "age_range": "ageRangeAggregate", @@ -7,22 +14,28 @@ const convert = { "ethnic_group": "ethnicGroup", "education_level_mod_agg": "educationLevelModAgg", 'integral_time_agg': 'integralTime', - 'period_agg': 'period' + 'period_agg': 'period', + 'modality_integral_time': 'educationLevelBasic', + 'especial_education': 'especialEducation' } function aggregateData(req, res, next) { const newResult = [] - const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg', 'integral_time_agg', 'period_agg'] + const aggregateFields = ['gender', 'age_range', 'ethnic_group', 'education_level_mod_agg', 'integral_time_agg', 'period_agg', 'modality_integral_time', 'especial_education'] let id; const fields = req.query.dims.split(','); let currentAggregateField; let currentNonAggregateField; + + // Verifica se o filtro passado está presente nos filtros agregados fields.forEach(field => {if (aggregateFields.includes(field)) currentAggregateField = field; else currentNonAggregateField = field}); + console.log(convert[currentAggregateField]) if (currentAggregateField) { - console.log(currentAggregateField); req.result.forEach((r) => { + // Alguns filtros começam com o id = 0 outros id = 1 id = ['ethnic_group', 'integral_time_agg'].includes(currentAggregateField) ? 0 : 1; for (const property in r) { + // Dados agregados são identificados com a substring "total_" em sua chave if (property.includes("total_")) { let data = { total: r[property], diff --git a/src/libs/middlewares/id2str.js b/src/libs/middlewares/id2str.js index 0a48e4ce..06f24488 100644 --- a/src/libs/middlewares/id2str.js +++ b/src/libs/middlewares/id2str.js @@ -116,7 +116,7 @@ const metroCode = require(`${libs}/convert/metroCode`); const modalityShift = require(`${libs}/convert/modalityShift`); const incomeRange = require(`${libs}/convert/incomeRange`); const ageRangeAggregate = require(`${libs}/convert/ageRangeAggregate`); - +const especialEducation = require(`${libs}/convert/especialEducation`); const ids = { gender_id: gender, @@ -225,7 +225,8 @@ const ids = { region_id: regionCode, metro_code_id: metroCode, modality_shift_id: modalityShift, - income_range_id: incomeRange + income_range_id: incomeRange, + especial_education: especialEducation }; function transform(removeId=false) { @@ -356,5 +357,6 @@ module.exports = { regionCode, metroCode, modalityShift, - incomeRange + incomeRange, + especialEducation }; diff --git a/src/libs/routes_v1/enrollmentAggregate.js b/src/libs/routes_v1/enrollmentAggregate.js index feae2681..e7de586c 100644 --- a/src/libs/routes_v1/enrollmentAggregate.js +++ b/src/libs/routes_v1/enrollmentAggregate.js @@ -49,6 +49,182 @@ enrollmentAggregateApp.use(cache('15 day')); let rqf = new ReqQueryFields(); +enrollmentAggregateApp.get('/years', (req, res, next) => { + req.sql.from('escola') + .field('DISTINCT escola.ano_censo', 'year') + .where('escola.ano_censo >= 2021') + next(); +}, query, response('years')); + +enrollmentAggregateApp.get('/adm_dependency', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 5; i++) { + req.result.push({ + id: i, name: id2str.admDependency(i) + }); + } + next(); +}, response('adm_dependency')); + +enrollmentAggregateApp.get('/adm_dependency_detailed', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 9; i++) { + req.result.push({ + id: i, name: id2str.admDependencyPriv(i) + }); + } + next(); +}, response('adm_dependency_detailed')); + +enrollmentAggregateApp.get('/education_level_mod', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 13; i++) { + req.result.push({ + id: i, name: id2str.educationLevelMod(i) + }); + } + next(); +}, response('education_level_mod')); + +enrollmentAggregateApp.get('/integral_time', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 3; i++) { + req.result.push({ + id: i, name: id2str.integralTime(i) + }); + } + next(); +}, response('education_level_mod')); + +enrollmentAggregateApp.get('/location', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 2; i++) { + req.result.push({ + id: i, name: id2str.location(i) + }); + } + next(); +}, response('location')); + +enrollmentAggregateApp.get('/diff_location', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 4; i++) { + req.result.push({ + id: i, name: id2str.diffLocation(i) + }); + } + + req.result.push({ + id: 8, name: 'Área onde se localizam povos e comunidades tradicionais' + }) + + next(); +}, response('diff_location')); + +enrollmentAggregateApp.get('/modality_integral_time', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 7; i++) { + req.result.push({ + id: i, name: id2str.educationLevelBasic(i) + }); + } + + next(); +}, response('modality_integral_time')); + +enrollmentAggregateApp.get('/gender', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 2; i++) { + req.result.push({ + id: i, name: id2str.gender(i) + }); + } + + next(); +}, response('gender')); + +enrollmentAggregateApp.get('/age_range', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 7; i++) { + req.result.push({ + id: i, name: id2str.ageRangeAggregate(i) + }); + } + + next(); +}, response('age_range')); + +enrollmentAggregateApp.get('/ethnic_group', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 6; i++) { + req.result.push({ + id: i, name: id2str.ethnicGroup(i) + }); + } + + next(); +}, response('ethnic_group')); + +enrollmentAggregateApp.get('/period_agg', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 5; i++) { + req.result.push({ + id: i, name: id2str.period(i) + }); + } + + next(); +}, response('period_agg')); + +enrollmentAggregateApp.get('/region', (req, res, next) => { + req.result = [] + + for (let i = 0; i <= 5; i++) { + req.result.push({ + id: i, name: id2str.regionCode(i) + }); + } + + next(); +}, response('period_agg')); + +enrollmentAggregateApp.get('/state', (req, res, next) => { + req.result = [] + for (let i = 11; i < 54; i++) { + if (id2str.stateName(i) !== 'Não declarada') { + req.result.push({ + id: i, name: id2str.stateName(i) + }); + } + } + + next(); +}, response('state')); + +enrollmentAggregateApp.get('/especial_education', (req, res, next) => { + req.result = [] + + for (let i = 1; i <= 2; i++) { + req.result.push({ + id: i, name: id2str.especialEducation(i) + }); + } + + next(); +}, response('especial_education')); + + rqf.addField({ name: 'filter', field: false, @@ -299,6 +475,16 @@ rqf.addField({ type: 'integer', field: 'localizacao_id' } +}).addValue({ + name: 'diff_location', + table: 'escola', + tableField: 'localizacao_diferenciada_par', + resultField: 'diff_location_id', + where: { + relation: '=', + type: 'integer', + field: 'localizacao_diferenciada_par' + } }); // Return all cities @@ -356,7 +542,6 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .order('escola.ano_censo'); } else if (req.query.dims && req.query.dims.includes('integral_time_agg')) { - req.sql.from('turma,escola') .field('SUM(CASE WHEN turma.tempo_integral=0 then turma.num_matricula ELSE 0 END)', 'total_int_nao') .field('SUM(CASE WHEN turma.tempo_integral=1 then turma.num_matricula ELSE 0 END)', 'total_int') @@ -367,7 +552,6 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .where('turma.tipo_atendimento_id <= 2 and turma.escola_id = escola.id and turma.ano_censo = escola.ano_censo'); } else if (req.query.dims && req.query.dims.includes('period_agg')) { - req.sql.from('turma,escola') .field('SUM(CASE WHEN turma.turma_turno_id=1 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_matutino') .field('SUM(CASE WHEN turma.turma_turno_id=2 then turma.num_matricula ELSE 0 END)', 'total_qt_mat_bas_vespertino') @@ -380,7 +564,6 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .where('turma.tipo_atendimento_id <= 2 and turma.escola_id = escola.id and turma.ano_censo = escola.ano_censo'); } else if (req.query.dims && req.query.dims.includes('modality_integral_time')) { - req.sql.from('escola') .field('SUM(escola.qt_mat_inf_cre_int)', 'total_qt_mat_inf_cre_int') .field('SUM(escola.qt_mat_inf_pre_int)', 'total_qt_mat_inf_pre_int') @@ -391,6 +574,14 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .group('escola.ano_censo') .order('escola.ano_censo') } + else if (req.query.dims && req.query.dims.includes('especial_education')) { + req.sql.from('escola') + .field('SUM(escola.qt_mat_esp_cc)', 'total_qt_mat_esp_cc') + .field('SUM(escola.qt_mat_esp_ce)', 'total_qt_mat_esp_ce') + .field('escola.ano_censo', 'year') + .group('escola.ano_censo') + .order('escola.ano_censo') + } else { req.sql.from('escola') .field('SUM(escola.qt_mat_bas)', 'total') @@ -399,6 +590,6 @@ enrollmentAggregateApp.get('/', rqf.parse(), rqf.build(), (req, res, next) => { .order('escola.ano_censo'); } next(); -}, query, aggregateData, response('enrollment_aggregate')); +}, query, aggregateData, id2str.transform(false), response('enrollment_aggregate')); module.exports = enrollmentAggregateApp; -- GitLab