From ade6c326d4b46540231a43a9760b620603e15695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=C3=A3o=20Victor=20Frans=20Pondaco=20Winandy?=
<jvfpw18@inf.ufpr.br>
Date: Tue, 23 Apr 2019 08:38:35 -0300
Subject: [PATCH] Merge HOTMapper with database development
---
auto.sh | 68 +++++----
database/__init__.py | 9 +-
database/actions.py | 50 ++++++-
database/base.py | 9 +-
database/database_table.py | 9 +-
database/groups.py | 125 +++++++++++++++++
database/protocol.py | 9 +-
database/types.py | 9 +-
generate_schema.py | 9 +-
manage.py | 47 +++++--
pnad_protocol_from_dic.py | 10 +-
protocols_comparison.py | 9 +-
settings.py | 15 +-
sql/regiao.sql | 32 +++++
tests/__init__.py | 21 ---
tests/database_table_test.py | 265 +----------------------------------
16 files changed, 323 insertions(+), 373 deletions(-)
create mode 100644 database/groups.py
create mode 100644 sql/regiao.sql
diff --git a/auto.sh b/auto.sh
index c532eec..581de09 100755
--- a/auto.sh
+++ b/auto.sh
@@ -1,5 +1,23 @@
#!/bin/bash
+# Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+# Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+#
+# This file is part of HOTMapper.
+#
+# HOTMapper is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# HOTMapper is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+
# ---------------------------------------------------------------------------------------#
# Esse script tem como objetivo facilitar a criação do banco de dados do projeto SIMCAQ,
# conforme a necessidade dos desenvolvedores. O código é livre para modificações contanto
@@ -12,16 +30,7 @@
# ---------------------------------------------------------------------------------------#
fBase ()
{
- mclient -d $1 base/regiao.sql
- mclient -d $1 base/estado.sql
- mclient -d $1 base/municipio.sql
- mclient -d $1 base/siope_uf.sql
- mclient -d $1 base/siope_mun.sql
- mclient -d $1 base/siope_mun_seed.sql
- mclient -d $1 base/instituicao_superior.sql
- mclient -d $1 base/formacao_superior.sql
- mclient -d $1 base/formacao_superior_seed.sql
- mclient -d $1 base/ibge_pib.sql
+ ./manage.py execute_sql_group base
}
# ---------------------------------------------------------------------------------------#
@@ -62,6 +71,15 @@ fInsert()
}
# ---------------------------------------------------------------------------------------#
+# ---------------------------------------------------------------------------------------#
+# Função para criar tabelas agregadas a partir de sql
+# ---------------------------------------------------------------------------------------#
+fAggregate()
+{
+ ./manage.py execute_sql_group simcaq_aggregate
+}
+# ---------------------------------------------------------------------------------------#
+
# ---------------------------------------------------------------------------------------#
# Retorna uma ajuda caso não haja parâmetros de entrada
# ---------------------------------------------------------------------------------------#
@@ -73,9 +91,9 @@ if [ ! $1 ]; then
printf "# 3. create: execute the commands to create the tables.\n"
printf "# 4. insert: execute the commands to insert data to tables.\n\n"
printf "# Estructure of commands:\n"
- printf "# 1. ./auto.sh all [database_name] [path_to_files] [initial_year]"
+ printf "# 1. ./auto.sh all [path_to_files] [initial_year]"
printf " [final_year]\n"
- printf "# 2. ./auto.sh base [database_name]\n"
+ printf "# 2. ./auto.sh base\n"
printf "# 3. ./auto.sh create\n"
printf "# 4. ./auto.sh insert [path_to_files] [initial_year] [final_year]\n\n"
exit 0;
@@ -89,33 +107,29 @@ source ./env/bin/activate
if [ $? = 0 ]; then
printf "\n# Environment activated!\n"
if [ "$1" = 'all' ]; then
- if [ $2 ] && [ $3 ] && [ $4 ] && [ $5 ]; then
- printf "\n# Initializing the creation of base tables (may need database"
- printf " password)...\n"
+ if [ $2 ] && [ $3 ] && [ $4 ]; then
+ printf "\n# Initializing the creation of base tables...\n"
sleep 1
- fBase "$2"
+ fBase
printf "\n# Initializing the creation of mapping tables...\n"
sleep 1
fCreate
printf "\n# Initializing the insertion of data, this may take a while...\n"
sleep 2
- fInsert "$3" "$4" "$5"
- sleep 1
- else
- printf "# ERROR: Missing parameters!\n"
- exit -1;
- fi
- elif [ "$1" = 'base' ]; then
- if [ $2 ]; then
- printf "\n# Initializing the creation of base tables (may need database"
- printf " password)...\n"
+ fInsert "$2" "$3" "$4"
sleep 1
- fBase "$2"
+ printf "\n# Initializing the creation of aggregate tables...\n"
sleep 1
+ fAggregate
else
printf "# ERROR: Missing parameters!\n"
exit -1;
fi
+ elif [ "$1" = 'base' ]; then
+ printf "\n# Initializing the creation of base tables...\n"
+ sleep 1
+ fBase
+ sleep 1
elif [ "$1" = 'create' ]; then
printf "\n# Initializing the creation of tables...\n"
sleep 1
diff --git a/database/__init__.py b/database/__init__.py
index 705b9c6..cd54961 100644
--- a/database/__init__.py
+++ b/database/__init__.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,7 +15,6 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
diff --git a/database/actions.py b/database/actions.py
index 9ba4c38..26733f1 100644
--- a/database/actions.py
+++ b/database/actions.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,23 +15,25 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''Database manipulation actions - these can be used as models for other modules.'''
import logging
-from sqlalchemy import create_engine, MetaData
+from sqlalchemy import create_engine, MetaData, text
from os import chdir
from datetime import datetime
from database.base import MissingTableError
from database.database_table import gen_data_table, copy_tabbed_to_csv
+import database.groups
import settings
+from database.groups import DATA_GROUP, DATABASE_TABLE_NAME
ENGINE = create_engine(settings.DATABASE_URI, echo=settings.ECHO)
META = MetaData(bind=ENGINE)
logging.basicConfig(format = settings.LOGGING_FORMAT)
+logger = logging.getLogger(__name__)
database_table_logger = logging.getLogger('database.database_table')
database_table_logger.setLevel(settings.LOGGING_LEVEL)
@@ -101,7 +103,7 @@ def csv_from_tabbed(table_name, input_file, output_file, year, sep=';'):
copy_tabbed_to_csv(input_file, column_mappings, settings.CHUNK_SIZE, output_file,
column_names=column_names, sep=sep)
-def update_from_file(file_name, table, year, columns=None, target_list=None,
+def update_from_file(file_name, table, year, columns=None,
offset=2, delimiters=[';', '\\n', '"'], null=''):
'''Updates table columns from an input csv file'''
table = gen_data_table(table, META)
@@ -140,3 +142,37 @@ def generate_backup():
f = open(settings.BACKUP_FILE,"w")
f.write(str(datetime.now()))
f.close()
+
+def execute_sql_script(sql_scripts, sql_path=settings.SCRIPTS_FOLDER):
+ if type(sql_scripts) == str:
+ sql_scripts = [sql_scripts]
+ with ENGINE.connect() as connection:
+ trans = connection.begin()
+ for script in sql_scripts:
+ with open(sql_path + '/' + script) as sql:
+ connection.execute(text(sql.read()))
+ trans.commit()
+
+def execute_sql_group(script_group, sql_path=settings.SCRIPTS_FOLDER, files=False):
+ if not files:
+ sql_script = [DATA_GROUP[group.upper()] for group in script_group.split(",")]
+ else:
+ sql_script = script_group.split(",")
+ for sql in sql_script:
+ execute_sql_script(sql, sql_path + '/')
+
+def drop_group(script_group, files=False):
+ script_group = script_group.split(",")
+ selected_tables = []
+ if not files:
+ for group in script_group:
+ selected_tables += DATA_GROUP[group.upper()]
+ else:
+ selected_tables = script_group
+
+ for table in reversed(selected_tables):
+ if table in DATABASE_TABLE_NAME:
+ table_name = DATABASE_TABLE_NAME[table]
+ else:
+ table_name = table.replace('.sql', '')
+ drop(table_name)
diff --git a/database/base.py b/database/base.py
index d00aa5b..4312529 100644
--- a/database/base.py
+++ b/database/base.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''Module containing base declarations'''
diff --git a/database/database_table.py b/database/database_table.py
index ef3d809..ab81092 100644
--- a/database/database_table.py
+++ b/database/database_table.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''This module contains the definition of the DatabaseTable class and a constructor'''
import os
diff --git a/database/groups.py b/database/groups.py
new file mode 100644
index 0000000..98e2580
--- /dev/null
+++ b/database/groups.py
@@ -0,0 +1,125 @@
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+
+This file is part of HOTMapper.
+
+HOTMapper is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+HOTMapper is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
+
+'''Group Settings'''
+
+# ---------------------------------------------------------------------------------------#
+# SMPPIR
+# ---------------------------------------------------------------------------------------#
+INEP = [
+ 'admission.sql',
+ 'course.sql',
+ 'evader.sql',
+ 'extracurricular_activities.sql',
+ 'graduate.sql',
+ 'institution.sql',
+ 'institutionPrivate.sql',
+ 'social_support.sql',
+ 'student_loans.sql'
+]
+PROUNI = [
+ 'coursePROUNI.sql',
+ 'institutionPROUNI.sql',
+ 'prouni.sql'
+]
+PNAD = [
+ 'pnad.sql'
+]
+CADUNICO = [
+ 'eixo2.sql',
+ 'eixo3.sql',
+ 'eixo4.sql',
+ 'african_sustentability.sql',
+ 'african_rights.sql',
+ 'african_culture.sql'
+]
+FIES = [
+ 'courseFIES.sql',
+ 'fies.sql',
+ 'institutionFIES.sql'
+]
+ALL_GROUPS_SMPPIR = INEP + PROUNI + PNAD + CADUNICO + FIES
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# SIMCAQ
+# ---------------------------------------------------------------------------------------#
+BASE = [
+ 'regiao.sql',
+ 'estado.sql',
+ 'municipio.sql',
+ 'siope_uf.sql',
+ 'siope_mun.sql',
+ 'siope_mun_seed.sql',
+ 'instituicao_superior.sql',
+ 'formacao_superior.sql',
+ 'formacao_superior_seed.sql',
+ 'ibge_pib.sql',
+ 'cub.sql',
+]
+
+SIMCAQ_AGGREGATE = [
+ 'docente_por_escola.sql',
+ 'idm.sql',
+ 'projecao_matricula.sql'
+]
+# ---------------------------------------------------------------------------------------#
+
+# ---------------------------------------------------------------------------------------#
+# Usado para chamar os grupos corretos
+# ---------------------------------------------------------------------------------------#
+DATA_GROUP = {
+ "INEP": INEP,
+ "PROUNI": PROUNI,
+ "PNAD": PNAD,
+ "CADUNICO": CADUNICO,
+ "FIES": FIES,
+ "ALL_GROUPS_SMPPIR": ALL_GROUPS_SMPPIR,
+ "BASE": BASE,
+ "SIMCAQ_AGGREGATE": SIMCAQ_AGGREGATE
+}
+# ---------------------------------------------------------------------------------------#
+# Nome da tabela caso seja diferente do nome do sql
+# ---------------------------------------------------------------------------------------#
+DATABASE_TABLE_NAME = {
+ 'admission.sql': 'admission_ag',
+ 'course.sql': 'course_ag',
+ 'evader.sql': 'evader_ag',
+ 'extracurricular_activities.sql': 'extracurricular_activities_ag',
+ 'graduate.sql': 'graduate_ag',
+ 'institution.sql': 'institution_ag',
+ 'institutionPrivate.sql': 'institution_private_ag',
+ 'social_support.sql': 'social_support_ag',
+ 'student_loans.sql': 'student_loans_ag',
+ 'coursePROUNI.sql': 'course_prouni_ag',
+ 'institutionPROUNI.sql': 'institution_prouni_ag',
+ 'prouni.sql': 'prouni_ag',
+ 'eixo2.sql': 'quilombola_eixo_2_ag',
+ 'eixo3.sql': 'quilombola_eixo_3_ag',
+ 'eixo4.sql': 'quilombola_eixo_4_ag',
+ 'african_sustentability.sql': 'african_sustentability_ag',
+ 'african_rights.sql': 'african_rights_ag',
+ 'african_culture.sql': 'african_culture_ag',
+ 'pnad.sql': 'pnad_ag',
+ 'courseFIES.sql': 'course_fies_ag',
+ 'fies.sql': 'fies_ag',
+ 'institutionFIES.sql': 'institution_fies_ag',
+ 'idm.sql': 'indice_distribuicao_matriculas'
+}
diff --git a/database/protocol.py b/database/protocol.py
index d78edf9..2316a7f 100644
--- a/database/protocol.py
+++ b/database/protocol.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
''' Routines related to column dictionary generation.
Names comonly used:
diff --git a/database/types.py b/database/types.py
index 1f0b24a..7cac079 100644
--- a/database/types.py
+++ b/database/types.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
import re
from sqlalchemy_monetdb.types import MONETDB_TYPE_MAP, TINYINT, DOUBLE_PRECISION
diff --git a/generate_schema.py b/generate_schema.py
index ab990c2..04e515a 100644
--- a/generate_schema.py
+++ b/generate_schema.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''Generates schema in mysql dialect. Useful for documentation'''
from sqlalchemy import create_engine, MetaData, inspect
diff --git a/manage.py b/manage.py
index a4dba50..236e6a5 100755
--- a/manage.py
+++ b/manage.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -17,16 +17,14 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
-
-
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''CLI for database module'''
from manager import Manager
-
+import subprocess
import database.actions
+from settings import SCRIPTS_FOLDER
manager = Manager()
@@ -59,7 +57,7 @@ def update_from_file(csv_file, table, year, columns=None, target_list=None, offs
if target_list:
target_list = target_list.split(',')
database.actions.update_from_file(csv_file, table, year, columns=columns,
- target_list=target_list, offset=offset,
+ offset=offset,
delimiters=[sep, '\\n', '"'], null=null)
@manager.command
@@ -79,5 +77,36 @@ def generate_backup():
'''Create/Recriate file monitored by backup script in production'''
database.actions.generate_backup()
+@manager.command
+def execute_sql_group(script_group, script_path=SCRIPTS_FOLDER, files=False):
+ '''Execute a group of sql files from groups.py,
+ if you want only specific files use --files and a "file1,file2,..." pattern'''
+ database.actions.execute_sql_group(script_group, script_path, files)
+
+@manager.command
+def drop_group(script_group, files=False):
+ '''Drop a group of tables from groups.py,
+ if you want to drop only specif tables use --files and a "table1,table2,..." pattern'''
+ database.actions.drop_group(script_group, files)
+
+@manager.command
+def rebuild_group(script_group, sql_path=SCRIPTS_FOLDER, files=False):
+ database.actions.drop_group(script_group, files)
+ database.actions.execute_sql_group(script_group, sql_path, files)
+
+@manager.command
+def run_script(script_name, args="", folder=SCRIPTS_FOLDER):
+ '''Run a script from the scripts folder, the arguments of the script needs to be passed as a string'''
+ run_list = args.split(",")
+ run_list.insert(0, script_name)
+ if script_name[-2:] == 'py':
+ run_list.insert(0, 'python')
+ subprocess.run(run_list, cwd=folder)
+ elif script_name[-2:] == 'sh':
+ run_list.insert(0, 'sh')
+ subprocess.run(run_list, cwd=folder)
+ elif script_name[-3:] == 'sql':
+ database.actions.execute_sql_script(script_name)
+
if __name__ == "__main__":
manager.main()
diff --git a/pnad_protocol_from_dic.py b/pnad_protocol_from_dic.py
index 4879ce7..9394819 100644
--- a/pnad_protocol_from_dic.py
+++ b/pnad_protocol_from_dic.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,10 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
-
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
import sys
import pandas as pd
diff --git a/protocols_comparison.py b/protocols_comparison.py
index 661d24e..37516fc 100644
--- a/protocols_comparison.py
+++ b/protocols_comparison.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,9 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
import pandas as pd
import os
diff --git a/settings.py b/settings.py
index 94ca902..f4f4a68 100644
--- a/settings.py
+++ b/settings.py
@@ -1,5 +1,5 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+'''
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
This file is part of HOTMapper.
@@ -15,10 +15,8 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
-
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+'''
'''Settings used by the database module'''
import logging
@@ -35,7 +33,7 @@ DATABASE_USER_PASSWORD = 'monetdb'
DATABASE_HOST = 'localhost'
# Database to connect to
-DATABASE = 'hotmapper_demo'
+DATABASE = 'dev_simcaq'
# URI structure. Standards to login:password model, but can be changed as needed.
DATABASE_URI = '{}://{}:{}@{}/{}'.format(DATABASE_DIALECT, DATABASE_USER,
@@ -50,6 +48,9 @@ MAPPING_PROTOCOLS_FOLDER = 'mapping_protocols'
# Folder for table definitions files
TABLE_DEFINITIONS_FOLDER = 'table_definitions'
+# Folder for scripts and sql tables
+SCRIPTS_FOLDER = 'sql'
+
# Source table definitions
SOURCE_TABLE_NAME = 'fonte'
SOURCE_TABLE_COLUMNS = {
diff --git a/sql/regiao.sql b/sql/regiao.sql
new file mode 100644
index 0000000..640ff0e
--- /dev/null
+++ b/sql/regiao.sql
@@ -0,0 +1,32 @@
+/*
+Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+
+This file is part of HOTMapper.
+
+HOTMapper is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+HOTMapper is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with HOTMapper. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+CREATE TABLE regiao
+(
+ id serial,
+ nome text not null
+);
+
+COPY 5 RECORDS INTO regiao FROM stdin USING DELIMITERS ',','\n';
+1,Norte
+2,Nordeste
+3,Sudeste
+4,Sul
+5,Centro-Oeste
diff --git a/tests/__init__.py b/tests/__init__.py
index 705b9c6..e69de29 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,21 +0,0 @@
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
-Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
-
-This file is part of HOTMapper.
-
-HOTMapper is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-HOTMapper is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
-
diff --git a/tests/database_table_test.py b/tests/database_table_test.py
index e06c586..80be991 100644
--- a/tests/database_table_test.py
+++ b/tests/database_table_test.py
@@ -1,26 +1,4 @@
#!/usr/bin/env python3
-
-"""
-Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
-Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
-
-This file is part of HOTMapper.
-
-HOTMapper is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-HOTMapper is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with simcaq-cdn. If not, see <https://www.gnu.org/licenses/>.
-
-"""
-
'''Describes tests for the database.database_table module, concerning DatabaseTable objects
and their manipulation'''
import unittest
@@ -38,6 +16,7 @@ import database.protocol as protocol
# Disable no-member warnings to silence false positives from Table instances dinamically generated
# attributes. Disabled warning for access to protected member
# pylint: disable=no-member,W0212
+# You'll also need python 3.6+
class MainModuleTest(unittest.TestCase):
@@ -92,37 +71,6 @@ class MainModuleTest(unittest.TestCase):
self.assertTrue(column1 in primary_keys)
self.assertTrue(column2 in primary_keys)
- @patch('database.database_table.Table')
- @patch('database.database_table.get_type')
- def test_gen_temporary(self, mocked_get_type, mocked_table):
- '''Test temporary table object generation'''
- name = MagicMock(str)
- field_name = MagicMock(str)
- field_type = MagicMock(str)
- meta = MagicMock(sqlalchemy.MetaData)
- columns = (field_name, field_type)
-
- database_table.gen_temporary(name, meta, columns)
-
- mocked_get_type.assert_called_with(field_type)
- mocked_table.assert_called_with(name, meta, prefixes=['TEMPORARY'], schema='tmp')
-
- def test_copy_to_temporary(self):
- '''Test copy into temporary table'''
- connection = MagicMock(sqlalchemy.engine.base.Engine._trans_ctx)
- connection.execute = MagicMock(lambda query: None)
- ttable = MagicMock(sqlalchemy.Table)
-
- csv_file = ''.join([choice(string.ascii_lowercase) for _ in range(randint(1, 10))])
- ttable.name = ''.join([choice(string.ascii_lowercase) for _ in range(randint(1, 10))])
-
- database_table.copy_to_temporary(connection, csv_file, ttable)
-
- query = connection.execute.call_args[0][0]
- self.assertIn(csv_file, query)
- self.assertIn(ttable.name, query)
-
-
def gen_random_string(min_length, max_length):
'''Generates a random string to use as name for some feature'''
string_size = randint(min_length, max_length)
@@ -187,34 +135,6 @@ class DatabaseTableTest(unittest.TestCase):
self.assertEqual(definitions, test_dict)
- def test_translate_header(self):
- '''Tests translation of a header to local database names from a protocol'''
- with self.assertRaises(base.MissingProtocolError):
- self.table.columns_from_targets(None)
-
- self.table._protocol = MagicMock(protocol.Protocol())
- self.table._protocol.target_from_original = MagicMock(
- self.table._protocol.target_from_original,
- return_value='target')
- self.table._protocol.dbcolumn_from_target = MagicMock(
- self.table._protocol.dbcolumn_from_target, return_value=('column_name', 'column_type'))
-
- self.assertEqual(self.table.translate_header([], []), {})
-
- translated = self.table.translate_header(['title'], 2013)
-
- self.table._protocol.target_from_original.assert_called_with('title', 2013)
- self.table._protocol.dbcolumn_from_target.assert_called_with('target')
- self.assertEqual(len(translated.keys()), 1)
- self.assertEqual(translated['title'], {"column_name": 'column_name',
- "column_type": 'column_type'})
-
- size = randint(1, 10)
- header = []
- for i in range(0, size):
- header.append('title'+str(i))
- translated = self.table.translate_header(header, 2013)
- self.assertEqual(len(translated.keys()), size)
@patch('database.database_table.insert')
def test_create_mapping_table(self, mocked_insert):
@@ -402,6 +322,9 @@ class DatabaseTableTest(unittest.TestCase):
self.engine.execute.assert_not_called()
self.table.primary_key = MagicMock(self.table.primary_key)
+ primary_column = MagicMock(sqlalchemy.Column)
+ self.table.primary_key.columns = [primary_column]
+
self.table.redefine_column = MagicMock(self.table.redefine_column)
transfer_list = []
for _ in range(randint(2, 5)):
@@ -436,190 +359,10 @@ class DatabaseTableTest(unittest.TestCase):
p = protocol.Protocol()
self.table.load_protocol(p)
-
-
- def test_treat_derivative(self):
- '''Tests the treatment of a derivative field'''
- original = None
- ret = self.table.treat_derivative(original)
- self.assertIs(ret, None)
-
- original = gen_random_string(4, 10)
- ret = self.table.treat_derivative(original)
- self.assertIs(ret, None)
-
- original = '~' + original
- ret = self.table.treat_derivative(original)
- self.assertEqual('~'+str(ret), original)
- self.assertIsInstance(ret, sqlalchemy.sql.elements.TextClause)
-
- def test_set_temporary_primary_keys(self):
- '''Tests the setup of primary keys in temporary tables'''
- ttable = sqlalchemy.Table('t_' + self.name, self.meta)
- with self.assertRaises(base.MissingProtocolError):
- self.table.set_temporary_primary_keys(ttable)
-
- pk_columns = []
- for _ in range(randint(2, 5)):
- column = Column(gen_random_string(5, 10), Integer())
- pk_columns.append(column)
- self.table.append_column(column)
- ttable.append_column(column.copy())
-
- not_pk_columns = []
- for _ in range(randint(2, 5)):
- column = Column(gen_random_string(5, 10), Integer())
- not_pk_columns.append(column)
- self.table.append_column(column)
- ttable.append_column(column.copy())
-
- self.table.primary_key = PrimaryKeyConstraint(*pk_columns)
- self.table._protocol = MagicMock(protocol.Protocol())
-
- self.table._protocol.target_from_dbcolumn = lambda name: name
- self.table._protocol.original_from_target = lambda name, year: name
-
- year = gen_random_string(2, 5)
-
- self.table.set_temporary_primary_keys(ttable, year=year)
-
- tpks = list(ttable.primary_key)
- pks = list(self.table.primary_key)
-
- for tpk, pk in zip(tpks, pks):
- self.assertEqual(pk.name, tpk.name)
-
- def test_set_temporary_primary_keys_no_year(self):
- '''Tests the setup of primary keys in temporary tables'''
- ttable = sqlalchemy.Table('t_' + self.name, self.meta)
-
- pk_columns = []
- for _ in range(randint(2, 5)):
- column = Column(gen_random_string(5, 10), Integer())
- pk_columns.append(column)
- self.table.append_column(column)
- ttable.append_column(column.copy())
-
- not_pk_columns = []
- for _ in range(randint(2, 5)):
- column = Column(gen_random_string(5, 10), Integer())
- not_pk_columns.append(column)
- self.table.append_column(column)
- ttable.append_column(column.copy())
-
- self.table.primary_key = PrimaryKeyConstraint(*pk_columns)
- self.table._protocol = MagicMock(protocol.Protocol())
-
- self.table.set_temporary_primary_keys(ttable)
-
- tpks = list(ttable.primary_key)
- pks = list(self.table.primary_key)
-
- for tpk, pk in zip(tpks, pks):
- self.assertEqual(pk.name, tpk.name)
-
- def test_mount_original_columns(self):
- '''Tests the mounting of the original columns list for a given table'''
- self.table.translate_header = MagicMock(self.table.translate_header)
- header = []
- year = randint(1, 10)
- self.table.translate_header.return_value = {}
- ret = self.table.mount_original_columns(header, year)
- self.assertIsInstance(ret, list)
- self.assertEqual(len(ret), 0)
-
- header_len = randint(1, 10)
- for _ in range(header_len):
- header.append(gen_random_string(4, 10))
-
- self.table.translate_header = lambda header, year:\
- dict((k, {"column_type": gen_random_string(4, 10)}) for k in header)
- ret = self.table.mount_original_columns(header, year)
- self.assertIsInstance(ret, list)
- self.assertEqual(len(ret), header_len)
- for entry, header_entry in zip(ret, header):
- self.assertEqual(entry[0], header_entry)
-
- @patch('database.database_table.get_type')
- @patch('database.database_table.Column')
- @patch('database.database_table.Table')
- @patch('database.database_table.select')
- @patch('database.database_table.insert')
- def test_set_temporary_columns(self, mocked_insert, mocked_select, mocked_table, mocked_column,
- mocked_get_type):
- '''Tests setup of temporary columns in temporary tables'''
- ttable = sqlalchemy.Table('t_' + self.name, self.meta)
- year = gen_random_string(2, 5)
- with self.assertRaises(base.MissingProtocolError):
- self.table.set_temporary_columns(self.engine, ttable, year)
-
- self.table._protocol = MagicMock(protocol.Protocol)
-
- # No assertions should be made for the original columns.
- # This should be removed from the temporary columns table some time in future
- columns = []
- for _ in range(randint(2, 5)):
- columns.append(MagicMock(Column()))
-
- ttable.columns = columns
-
- get_type_calls = []
- column_calls = []
- treat_derivative_calls = []
-
- return_value = []
- for _ in range(randint(2, 5)):
- temporary_column = (gen_random_string(5, 10),
- gen_random_string(2, 5),
- gen_random_string(10, 20)
- )
- return_value.append(temporary_column)
- get_type_calls.append(call(temporary_column[1]))
- column_calls.append(call(temporary_column[0], mocked_get_type()))
- treat_derivative_calls.append(call(temporary_column[2]))
-
- self.table._protocol.get_temporary_columns = lambda year: return_value
- self.table.treat_derivative = MagicMock(self.table.treat_derivative)
-
- self.table.set_temporary_columns(self.engine, ttable, gen_random_string(2, 5))
-
- mocked_table.assert_called_once()
- mocked_get_type.assert_has_calls(get_type_calls)
- mocked_column.assert_has_calls(column_calls)
- mocked_select.assert_called()
- mocked_insert.assert_called()
- self.table.treat_derivative.assert_has_calls(treat_derivative_calls)
-
def test_insert_from_temporary(self):
'''Tests insertion in table from a previously created temporary table'''
pass
- def test_columns_from_targets(self):
- '''Tests capture of database columns from given targets on a table object'''
- with self.assertRaises(base.MissingProtocolError):
- self.table.columns_from_targets(None)
-
- self.table._protocol = MagicMock(protocol.Protocol())
- target_list = None
- ret = self.table.columns_from_targets(target_list)
- self.assertIsInstance(ret, list)
- self.assertEqual(len(ret), 0)
-
- target_list_len = randint(1, 10)
- target_list = []
- for _ in range(target_list_len):
- target_list.append(gen_random_string(4, 10))
- ret_column_name = gen_random_string(4, 10)
- self.table._protocol.dbcolumn_from_target.return_value = (ret_column_name, str)
- ret = self.table.columns_from_targets(target_list)
- self.assertIsInstance(ret, list)
- self.assertEqual(ret[0], ret_column_name)
- self.assertEqual(len(ret), target_list_len)
-
- self.table._protocol.dbcolumn_from_target.return_value = (ret_column_name, None)
- with self.assertRaises(base.InvalidTargetError):
- self.table.columns_from_targets(target_list)
-
def test_update_from_temporary(self):
'''Tests updating of given columns from a temporary table'''
pass
--
GitLab