From a19e18c2ebd5abc5b06804662f3787d2d12dbe63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=C3=A3o=20Victor=20Frans=20Pondaco=20Winandy?=
<jvfpw18@inf.ufpr.br>
Date: Thu, 13 Jun 2019 10:00:22 -0300
Subject: [PATCH] Add auto.sh, groups.py and update readme
---
README.md | 32 +++++++++++
auto.sh | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
groups.py | 126 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 322 insertions(+)
create mode 100644 auto.sh
create mode 100644 groups.py
diff --git a/README.md b/README.md
index e69de29..5019a6c 100644
--- a/README.md
+++ b/README.md
@@ -0,0 +1,32 @@
+Este diretório contém os arquivos com os scrpits sql utilizados pelo SIMCAQ e SMPPIR para a criação de tabelas base e agregadas.
+
+## O arquivo groups.py
+
+Este é o arquivo de configuração para a criação/remoção de várias tabelas de uma vez utilizando as funções do HOTMapper execute_sql_group, drop_group e rebuild_group.
+
+## O Script auto.sh
+
+Esse script tem como objetivo facilitar a criação do banco de dados do projeto SIMCAQ, conforme a necessidade dos desenvolvedores.
+
+## Grupos de tabelas e projetos origem
+
+### SIMCAQ
+
+#### BASE
+* regiao
+* estado
+* municipio
+* siope_uf
+* siope_mun
+* siope_mun_seed
+* instituicao_superior
+* formacao_superior
+* formacao_superior_seed
+* ibge_pib
+* cub
+
+#### SIMCAQ_AGGREGATE
+* docente_por_escola
+* idm
+* projecao_matricula
+* matricula_por_localizacao
\ No newline at end of file
diff --git a/auto.sh b/auto.sh
new file mode 100644
index 0000000..581de09
--- /dev/null
+++ b/auto.sh
@@ -0,0 +1,164 @@
+#!/bin/bash
+
+# Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+#
+# This file is part of HOTMapper.
+#
+# HOTMapper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# HOTMapper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+
+# ---------------------------------------------------------------------------------------#
+# Esse script tem como objetivo facilitar a criação do banco de dados do projeto SIMCAQ,
+# conforme a necessidade dos desenvolvedores. O código é livre para modificações contanto
+# que os que utilizam o script sejam notificados das mudanças decorrentes.
+# ---------------------------------------------------------------------------------------#
+
+
+# ---------------------------------------------------------------------------------------#
+# Função para criar as tabelas que são consideradas bases para o banco de dados
+# ---------------------------------------------------------------------------------------#
+fBase ()
+{
+ ./manage.py execute_sql_group base
+}
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Função para criar as tabelas a partir dos protocolos de mapeamento
+# ---------------------------------------------------------------------------------------#
+fCreate ()
+{
+ ./manage.py create escola
+ ./manage.py create turma
+ ./manage.py create matricula
+ ./manage.py create docente
+}
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Função para inserir dados nas tabelas criadas a partir dos protocolos de mapeamento
+# ---------------------------------------------------------------------------------------#
+fInsert()
+{
+ local alpha="$2"
+
+ while [ "$alpha" -le "$3" ]; do
+ ./manage.py insert $1${alpha}_ESCOLAS.CSV escola $alpha --sep=\|
+ ./manage.py insert $1${alpha}_TURMAS.CSV turma $alpha --sep=\|
+ ./manage.py insert $1${alpha}_DOCENTES_CO.CSV docente $alpha --sep=\|
+ ./manage.py insert $1${alpha}_DOCENTES_NORTE.CSV docente $alpha --sep=\|
+ ./manage.py insert $1${alpha}_DOCENTES_NORDESTE.CSV docente $alpha --sep=\|
+ ./manage.py insert $1${alpha}_DOCENTES_SUDESTE.CSV docente $alpha --sep=\|
+ ./manage.py insert $1${alpha}_DOCENTES_SUL.CSV docente $alpha --sep=\|
+ ./manage.py insert $1${alpha}_MATRICULA_CO.CSV matricula $alpha --sep=\|
+ ./manage.py insert $1${alpha}_MATRICULA_NORTE.CSV matricula $alpha --sep=\|
+ ./manage.py insert $1${alpha}_MATRICULA_NORDESTE.CSV matricula $alpha --sep=\|
+ ./manage.py insert $1${alpha}_MATRICULA_SUDESTE.CSV matricula $alpha --sep=\|
+ ./manage.py insert $1${alpha}_MATRICULA_SUL.CSV matricula $alpha --sep=\|
+ alpha=$(($alpha + 1))
+ done
+}
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Função para criar tabelas agregadas a partir de sql
+# ---------------------------------------------------------------------------------------#
+fAggregate()
+{
+ ./manage.py execute_sql_group simcaq_aggregate
+}
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Retorna uma ajuda caso não haja parâmetros de entrada
+# ---------------------------------------------------------------------------------------#
+if [ ! $1 ]; then
+ printf "\n# WARNING: Don't forget to check the settings file for the database name.\n"
+ printf "\n# This script has 4 commands:\n"
+ printf "# 1. all: execute all commands to create the database and insert data.\n"
+ printf "# 2. base: execute the commands to create de base tables.\n"
+ printf "# 3. create: execute the commands to create the tables.\n"
+ printf "# 4. insert: execute the commands to insert data to tables.\n\n"
+ printf "# Estructure of commands:\n"
+ printf "# 1. ./auto.sh all [path_to_files] [initial_year]"
+ printf " [final_year]\n"
+ printf "# 2. ./auto.sh base\n"
+ printf "# 3. ./auto.sh create\n"
+ printf "# 4. ./auto.sh insert [path_to_files] [initial_year] [final_year]\n\n"
+ exit 0;
+fi
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Execução do script conforme os comandos passados
+# ---------------------------------------------------------------------------------------#
+source ./env/bin/activate
+if [ $? = 0 ]; then
+ printf "\n# Environment activated!\n"
+ if [ "$1" = 'all' ]; then
+ if [ $2 ] && [ $3 ] && [ $4 ]; then
+ printf "\n# Initializing the creation of base tables...\n"
+ sleep 1
+ fBase
+ printf "\n# Initializing the creation of mapping tables...\n"
+ sleep 1
+ fCreate
+ printf "\n# Initializing the insertion of data, this may take a while...\n"
+ sleep 2
+ fInsert "$2" "$3" "$4"
+ sleep 1
+ printf "\n# Initializing the creation of aggregate tables...\n"
+ sleep 1
+ fAggregate
+ else
+ printf "# ERROR: Missing parameters!\n"
+ exit -1;
+ fi
+ elif [ "$1" = 'base' ]; then
+ printf "\n# Initializing the creation of base tables...\n"
+ sleep 1
+ fBase
+ sleep 1
+ elif [ "$1" = 'create' ]; then
+ printf "\n# Initializing the creation of tables...\n"
+ sleep 1
+ fCreate
+ sleep 1
+ elif [ "$1" = 'insert' ]; then
+ if [ $2 ] && [ $3 ] && [ $4 ]; then
+ printf "\n# Initializing the insertion of data, this may take a while...\n"
+ sleep 2
+ fInsert "$2" "$3" "$4"
+ sleep 1
+ else
+ printf "# ERROR: Missing parameters!\n"
+ exit -1;
+ fi
+ else
+ printf "\n# ERROR: Missing parameters!\n"
+ deactivate
+ printf "\n# Environment deactivated!\n"
+ printf "# Terminating...\n"
+ sleep 1
+ exit -1;
+ fi
+ deactivate
+ printf "\n# Environment deactivated!\n"
+ printf "\n# All done! Terminating...\n"
+ sleep 1
+ exit 0;
+else
+ printf "# ERROR: can't find the directory for environment!\n"
+ exit -1;
+fi
diff --git a/groups.py b/groups.py
new file mode 100644
index 0000000..1fd2126
--- /dev/null
+++ b/groups.py
@@ -0,0 +1,126 @@
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+
+This file is part of HOTMapper.
+
+HOTMapper is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+HOTMapper is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
+
+'''Group Settings'''
+
+# ---------------------------------------------------------------------------------------#
+# SMPPIR
+# ---------------------------------------------------------------------------------------#
+INEP = [
+ 'admission.sql',
+ 'course.sql',
+ 'evader.sql',
+ 'extracurricular_activities.sql',
+ 'graduate.sql',
+ 'institution.sql',
+ 'institutionPrivate.sql',
+ 'social_support.sql',
+ 'student_loans.sql'
+]
+PROUNI = [
+ 'coursePROUNI.sql',
+ 'institutionPROUNI.sql',
+ 'prouni.sql'
+]
+PNAD = [
+ 'pnad.sql'
+]
+CADUNICO = [
+ 'eixo2.sql',
+ 'eixo3.sql',
+ 'eixo4.sql',
+ 'african_sustentability.sql',
+ 'african_rights.sql',
+ 'african_culture.sql'
+]
+FIES = [
+ 'courseFIES.sql',
+ 'fies.sql',
+ 'institutionFIES.sql'
+]
+ALL_GROUPS_SMPPIR = INEP + PROUNI + PNAD + CADUNICO + FIES
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# SIMCAQ
+# ---------------------------------------------------------------------------------------#
+BASE = [
+ 'regiao.sql',
+ 'estado.sql',
+ 'municipio.sql',
+ 'siope_uf.sql',
+ 'siope_mun.sql',
+ 'siope_mun_seed.sql',
+ 'instituicao_superior.sql',
+ 'formacao_superior.sql',
+ 'formacao_superior_seed.sql',
+ 'ibge_pib.sql',
+ 'cub.sql',
+]
+
+SIMCAQ_AGGREGATE = [
+ 'docente_por_escola.sql',
+ 'idm.sql',
+ 'projecao_matricula.sql'
+ 'matricula_por_localizacao.sql'
+]
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Usado para chamar os grupos corretos
+# ---------------------------------------------------------------------------------------#
+DATA_GROUP = {
+ "INEP": INEP,
+ "PROUNI": PROUNI,
+ "PNAD": PNAD,
+ "CADUNICO": CADUNICO,
+ "FIES": FIES,
+ "ALL_GROUPS_SMPPIR": ALL_GROUPS_SMPPIR,
+ "BASE": BASE,
+ "SIMCAQ_AGGREGATE": SIMCAQ_AGGREGATE
+}
+# ---------------------------------------------------------------------------------------#
+# Nome da tabela caso seja diferente do nome do sql
+# ---------------------------------------------------------------------------------------#
+DATABASE_TABLE_NAME = {
+ 'admission.sql': 'admission_ag',
+ 'course.sql': 'course_ag',
+ 'evader.sql': 'evader_ag',
+ 'extracurricular_activities.sql': 'extracurricular_activities_ag',
+ 'graduate.sql': 'graduate_ag',
+ 'institution.sql': 'institution_ag',
+ 'institutionPrivate.sql': 'institution_private_ag',
+ 'social_support.sql': 'social_support_ag',
+ 'student_loans.sql': 'student_loans_ag',
+ 'coursePROUNI.sql': 'course_prouni_ag',
+ 'institutionPROUNI.sql': 'institution_prouni_ag',
+ 'prouni.sql': 'prouni_ag',
+ 'eixo2.sql': 'quilombola_eixo_2_ag',
+ 'eixo3.sql': 'quilombola_eixo_3_ag',
+ 'eixo4.sql': 'quilombola_eixo_4_ag',
+ 'african_sustentability.sql': 'african_sustentability_ag',
+ 'african_rights.sql': 'african_rights_ag',
+ 'african_culture.sql': 'african_culture_ag',
+ 'pnad.sql': 'pnad_ag',
+ 'courseFIES.sql': 'course_fies_ag',
+ 'fies.sql': 'fies_ag',
+ 'institutionFIES.sql': 'institution_fies_ag',
+ 'idm.sql': 'indice_distribuicao_matriculas'
+}
--
GitLab