diff --git a/README.md b/README.md index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5019a6cbb8fa55fc7eb4e461b97c6c1441d32811 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,32 @@ +Este diretório contém os arquivos com os scrpits sql utilizados pelo SIMCAQ e SMPPIR para a criação de tabelas base e agregadas. + +## O arquivo groups.py + +Este é o arquivo de configuração para a criação/remoção de várias tabelas de uma vez utilizando as funções do HOTMapper execute_sql_group, drop_group e rebuild_group. + +## O Script auto.sh + +Esse script tem como objetivo facilitar a criação do banco de dados do projeto SIMCAQ, conforme a necessidade dos desenvolvedores. + +## Grupos de tabelas e projetos origem + +### SIMCAQ + +#### BASE +* regiao +* estado +* municipio +* siope_uf +* siope_mun +* siope_mun_seed +* instituicao_superior +* formacao_superior +* formacao_superior_seed +* ibge_pib +* cub + +#### SIMCAQ_AGGREGATE +* docente_por_escola +* idm +* projecao_matricula +* matricula_por_localizacao \ No newline at end of file diff --git a/auto.sh b/auto.sh new file mode 100644 index 0000000000000000000000000000000000000000..581de09e6f1bb9e3a5e58c9bdc4b89327d5c8cac --- /dev/null +++ b/auto.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +# Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre +# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR +# +# This file is part of HOTMapper. +# +# HOTMapper is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# HOTMapper is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with HOTMapper. If not, see <https://www.gnu.org/licenses/>. + +# ---------------------------------------------------------------------------------------# +# Esse script tem como objetivo facilitar a criação do banco de dados do projeto SIMCAQ, +# conforme a necessidade dos desenvolvedores. O código é livre para modificações contanto +# que os que utilizam o script sejam notificados das mudanças decorrentes. +# ---------------------------------------------------------------------------------------# + + +# ---------------------------------------------------------------------------------------# +# Função para criar as tabelas que são consideradas bases para o banco de dados +# ---------------------------------------------------------------------------------------# +fBase () +{ + ./manage.py execute_sql_group base +} +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Função para criar as tabelas a partir dos protocolos de mapeamento +# ---------------------------------------------------------------------------------------# +fCreate () +{ + ./manage.py create escola + ./manage.py create turma + ./manage.py create matricula + ./manage.py create docente +} +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Função para inserir dados nas tabelas criadas a partir dos protocolos de mapeamento +# ---------------------------------------------------------------------------------------# +fInsert() +{ + local alpha="$2" + + while [ "$alpha" -le "$3" ]; do + ./manage.py insert $1${alpha}_ESCOLAS.CSV escola $alpha --sep=\| + ./manage.py insert $1${alpha}_TURMAS.CSV turma $alpha --sep=\| + ./manage.py insert $1${alpha}_DOCENTES_CO.CSV docente $alpha --sep=\| + ./manage.py insert $1${alpha}_DOCENTES_NORTE.CSV docente $alpha --sep=\| + ./manage.py insert $1${alpha}_DOCENTES_NORDESTE.CSV docente $alpha --sep=\| + ./manage.py insert $1${alpha}_DOCENTES_SUDESTE.CSV docente $alpha --sep=\| + ./manage.py insert $1${alpha}_DOCENTES_SUL.CSV docente $alpha --sep=\| + ./manage.py insert $1${alpha}_MATRICULA_CO.CSV matricula $alpha --sep=\| + ./manage.py insert $1${alpha}_MATRICULA_NORTE.CSV matricula $alpha --sep=\| + ./manage.py insert $1${alpha}_MATRICULA_NORDESTE.CSV matricula $alpha --sep=\| + ./manage.py insert $1${alpha}_MATRICULA_SUDESTE.CSV matricula $alpha --sep=\| + ./manage.py insert $1${alpha}_MATRICULA_SUL.CSV matricula $alpha --sep=\| + alpha=$(($alpha + 1)) + done +} +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Função para criar tabelas agregadas a partir de sql +# ---------------------------------------------------------------------------------------# +fAggregate() +{ + ./manage.py execute_sql_group simcaq_aggregate +} +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Retorna uma ajuda caso não haja parâmetros de entrada +# ---------------------------------------------------------------------------------------# +if [ ! $1 ]; then + printf "\n# WARNING: Don't forget to check the settings file for the database name.\n" + printf "\n# This script has 4 commands:\n" + printf "# 1. all: execute all commands to create the database and insert data.\n" + printf "# 2. base: execute the commands to create de base tables.\n" + printf "# 3. create: execute the commands to create the tables.\n" + printf "# 4. insert: execute the commands to insert data to tables.\n\n" + printf "# Estructure of commands:\n" + printf "# 1. ./auto.sh all [path_to_files] [initial_year]" + printf " [final_year]\n" + printf "# 2. ./auto.sh base\n" + printf "# 3. ./auto.sh create\n" + printf "# 4. ./auto.sh insert [path_to_files] [initial_year] [final_year]\n\n" + exit 0; +fi +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Execução do script conforme os comandos passados +# ---------------------------------------------------------------------------------------# +source ./env/bin/activate +if [ $? = 0 ]; then + printf "\n# Environment activated!\n" + if [ "$1" = 'all' ]; then + if [ $2 ] && [ $3 ] && [ $4 ]; then + printf "\n# Initializing the creation of base tables...\n" + sleep 1 + fBase + printf "\n# Initializing the creation of mapping tables...\n" + sleep 1 + fCreate + printf "\n# Initializing the insertion of data, this may take a while...\n" + sleep 2 + fInsert "$2" "$3" "$4" + sleep 1 + printf "\n# Initializing the creation of aggregate tables...\n" + sleep 1 + fAggregate + else + printf "# ERROR: Missing parameters!\n" + exit -1; + fi + elif [ "$1" = 'base' ]; then + printf "\n# Initializing the creation of base tables...\n" + sleep 1 + fBase + sleep 1 + elif [ "$1" = 'create' ]; then + printf "\n# Initializing the creation of tables...\n" + sleep 1 + fCreate + sleep 1 + elif [ "$1" = 'insert' ]; then + if [ $2 ] && [ $3 ] && [ $4 ]; then + printf "\n# Initializing the insertion of data, this may take a while...\n" + sleep 2 + fInsert "$2" "$3" "$4" + sleep 1 + else + printf "# ERROR: Missing parameters!\n" + exit -1; + fi + else + printf "\n# ERROR: Missing parameters!\n" + deactivate + printf "\n# Environment deactivated!\n" + printf "# Terminating...\n" + sleep 1 + exit -1; + fi + deactivate + printf "\n# Environment deactivated!\n" + printf "\n# All done! Terminating...\n" + sleep 1 + exit 0; +else + printf "# ERROR: can't find the directory for environment!\n" + exit -1; +fi diff --git a/groups.py b/groups.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd2126fa3d34894e57338cfaaf24c8c82c0ee75 --- /dev/null +++ b/groups.py @@ -0,0 +1,126 @@ +''' +Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of HOTMapper. + +HOTMapper is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +HOTMapper is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with HOTMapper. If not, see <https://www.gnu.org/licenses/>. +''' + +'''Group Settings''' + +# ---------------------------------------------------------------------------------------# +# SMPPIR +# ---------------------------------------------------------------------------------------# +INEP = [ + 'admission.sql', + 'course.sql', + 'evader.sql', + 'extracurricular_activities.sql', + 'graduate.sql', + 'institution.sql', + 'institutionPrivate.sql', + 'social_support.sql', + 'student_loans.sql' +] +PROUNI = [ + 'coursePROUNI.sql', + 'institutionPROUNI.sql', + 'prouni.sql' +] +PNAD = [ + 'pnad.sql' +] +CADUNICO = [ + 'eixo2.sql', + 'eixo3.sql', + 'eixo4.sql', + 'african_sustentability.sql', + 'african_rights.sql', + 'african_culture.sql' +] +FIES = [ + 'courseFIES.sql', + 'fies.sql', + 'institutionFIES.sql' +] +ALL_GROUPS_SMPPIR = INEP + PROUNI + PNAD + CADUNICO + FIES +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# SIMCAQ +# ---------------------------------------------------------------------------------------# +BASE = [ + 'regiao.sql', + 'estado.sql', + 'municipio.sql', + 'siope_uf.sql', + 'siope_mun.sql', + 'siope_mun_seed.sql', + 'instituicao_superior.sql', + 'formacao_superior.sql', + 'formacao_superior_seed.sql', + 'ibge_pib.sql', + 'cub.sql', +] + +SIMCAQ_AGGREGATE = [ + 'docente_por_escola.sql', + 'idm.sql', + 'projecao_matricula.sql' + 'matricula_por_localizacao.sql' +] +# ---------------------------------------------------------------------------------------# + +# ---------------------------------------------------------------------------------------# +# Usado para chamar os grupos corretos +# ---------------------------------------------------------------------------------------# +DATA_GROUP = { + "INEP": INEP, + "PROUNI": PROUNI, + "PNAD": PNAD, + "CADUNICO": CADUNICO, + "FIES": FIES, + "ALL_GROUPS_SMPPIR": ALL_GROUPS_SMPPIR, + "BASE": BASE, + "SIMCAQ_AGGREGATE": SIMCAQ_AGGREGATE +} +# ---------------------------------------------------------------------------------------# +# Nome da tabela caso seja diferente do nome do sql +# ---------------------------------------------------------------------------------------# +DATABASE_TABLE_NAME = { + 'admission.sql': 'admission_ag', + 'course.sql': 'course_ag', + 'evader.sql': 'evader_ag', + 'extracurricular_activities.sql': 'extracurricular_activities_ag', + 'graduate.sql': 'graduate_ag', + 'institution.sql': 'institution_ag', + 'institutionPrivate.sql': 'institution_private_ag', + 'social_support.sql': 'social_support_ag', + 'student_loans.sql': 'student_loans_ag', + 'coursePROUNI.sql': 'course_prouni_ag', + 'institutionPROUNI.sql': 'institution_prouni_ag', + 'prouni.sql': 'prouni_ag', + 'eixo2.sql': 'quilombola_eixo_2_ag', + 'eixo3.sql': 'quilombola_eixo_3_ag', + 'eixo4.sql': 'quilombola_eixo_4_ag', + 'african_sustentability.sql': 'african_sustentability_ag', + 'african_rights.sql': 'african_rights_ag', + 'african_culture.sql': 'african_culture_ag', + 'pnad.sql': 'pnad_ag', + 'courseFIES.sql': 'course_fies_ag', + 'fies.sql': 'fies_ag', + 'institutionFIES.sql': 'institution_fies_ag', + 'idm.sql': 'indice_distribuicao_matriculas' +}