diff --git a/__pycache__/combinationdims.cpython-310.pyc b/__pycache__/combinationdims.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..688dd4f21e67f3dc1984b1e51efe32eb469a3651 Binary files /dev/null and b/__pycache__/combinationdims.cpython-310.pyc differ diff --git a/__pycache__/regression_test.cpython-310.pyc b/__pycache__/regression_test.cpython-310.pyc index 6cd9dfa0e3be0e50e853caa00d5339c457e2deb2..4da1973c71178ef09eac9cc2aceb97fb1d606458 100644 Binary files a/__pycache__/regression_test.cpython-310.pyc and b/__pycache__/regression_test.cpython-310.pyc differ diff --git a/__pycache__/regression_test_dims.cpython-310.pyc b/__pycache__/regression_test_dims.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..433d45f534bae2aa106fbccb3bebef60e0bb0c4c Binary files /dev/null and b/__pycache__/regression_test_dims.cpython-310.pyc differ diff --git a/__pycache__/settings.cpython-310.pyc b/__pycache__/settings.cpython-310.pyc index 870d46bcfc2451b82f2ac422512e926f5762c6c6..0b06703373d2cbf5eb60062393b3950e61389439 100644 Binary files a/__pycache__/settings.cpython-310.pyc and b/__pycache__/settings.cpython-310.pyc differ diff --git a/combinationdims.py b/combinationdims.py new file mode 100644 index 0000000000000000000000000000000000000000..b1a6576e53fecf45022340baefc4235d628b216d --- /dev/null +++ b/combinationdims.py @@ -0,0 +1,77 @@ +from itertools import combinations + +BASE_URL = 'https://simcaq.c3sl.ufpr.br/api/v1/' + +BASE_ROUTE_LIST_COMBINATION = [] + +#### Enrollments +for i in list(combinations(['location','rural_location','school_year','educational_level','adm_dependency','gender','ethnic_group','period','adm_dependency_detailed','educational_level_mod','educational_level_short','region','state','city','school'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['enrollment','&dims='+i[0]+','+i[1]]) + +#### Course Count +for i in list(combinations(['academic_level','state','mesoregion','microregion','localOffer','city','course','campi','upper_adm_dependency','upper_education_mod','academic_organization','university','universityLocalOffer','ocde_specific','ocde_geral','ocde_detailed','cine_specific','cine_geral','cine_detailed','is_free','night_time','situation','year'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['course_count','&dims='+i[0]+','+i[1]]) + +#### Disciplines +for i in list(combinations(['adm_dependency','adm_dependency_detailed','contract_type','education_level_mod','education_level_short','education_type','region','mesoregion','microregion','state','city','location','rural_location','min_year','max_year','gender','ethnic_group','discipline'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['discipline','&dims='+i[0]+','+i[1]]) + +#### Employees +for i in list(combinations(['adm_dependency','adm_dependency_detailed','region','state','city','location','diff_location','rural_locations','function','year'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['employee','&dims='+i[0]+','+i[1]]) + +#### Distribution Factor +for i in list(combinations(['region','state','city'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['distribution_factor','&dims='+i[0]+','+i[1]]) + +#### Financial +for i in list(combinations(['min_year','max_year','sphere_adm','city','state','financial_data'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['financial','&dims='+i[0]+','+i[1]]) + +#### IDHM, IDHMR, IDHML, IDHME +for i in list(combinations(['state','city'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['idhm','&dims='+i[0]+','+i[1]]) + BASE_ROUTE_LIST_COMBINATION.append(['idhmr','&dims='+i[0]+','+i[1]]) + BASE_ROUTE_LIST_COMBINATION.append(['idhml','&dims='+i[0]+','+i[1]]) + BASE_ROUTE_LIST_COMBINATION.append(['idhme','&dims='+i[0]+','+i[1]]) + +#### Out of School +for i in list(combinations(['region','state','ethnic_group','full_age_range','gender','location','extremes_household_income','fifth_household_income','year'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['out_of_school','&dims='+i[0]+','+i[1]]) + +#### Infrastructure +for i in list(combinations(['region','state','location','rural_location','adm_dependency','adm_dependency_detailed'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['infrastructure','&dims='+i[0]+','+i[1]]) + +#### School Location +for i in list(combinations(['mesoregion','microregion','city'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['school_location','&dims='+i[0]+','+i[1]]) + +#### Auxiliar Count +for i in list(combinations(['adm_dependency','adm_dependency_detailed','education_level_mod','education_type','region','state','city','location','rural_location','school','gender','ethnic_group'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['auxiliar/count','&dims='+i[0]+','+i[1]]) + +#### School Count +for i in list(combinations(['location','adm_dependency','adm_dependency_detailed','government_agreement','agreement','education_day_care_child','education_preschool_child','education_begin_elementary_school','education_middle_school','education_professional','education_eja'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['school/count','&dims='+i[0]+','+i[1]]) + +#### Classroom +for i in list(combinations(['adm_dependency','adm_dependency_detailed','location','region','state','city'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['classroom','&dims='+i[0]+','+i[1]]) + +#### Population +for i in list(combinations(['region','state','city','city_size'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['population','&dims='+i[0]+','+i[1]]) + +#### Pibpercapita +for i in list(combinations(['region','state','city','income_level'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['pibpercapita','&dims='+i[0]+','+i[1]]) + +#### Teacher +for i in list(combinations(['adm_dependency','adm_dependency_detailed','education_level_mod','education_level_short','education_type','location','rural_location','region','state','city','school','gender','contract_type','ethnic_group'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['teacher','&dims='+i[0]+','+i[1]]) + +#### Enrollment Project +for i in list(combinations(['adm_dependency','state','city','min_year','max_year'],2)): + BASE_ROUTE_LIST_COMBINATION.append(['enrollment_project','&dims='+i[0]+','+i[1]]) + diff --git a/manage.py b/manage.py index d3635b8832820ce44cc491ed2e497869ce38f5f2..a9bbef516858816038f0d428a5a4f1403caf71a2 100644 --- a/manage.py +++ b/manage.py @@ -21,6 +21,7 @@ along with lde-api-regression-test. If not, see <https://www.gnu.org/licenses/> from manager import Manager from regression_test import RegressionTest +from regression_test_dims import RegressionTest_dims manager = Manager() @@ -28,7 +29,8 @@ manager = Manager() @manager.command def save(route=''): '''Save multiple or one route''' - rt = RegressionTest(route) + #rt = RegressionTest(route) + rt = RegressionTest_dims(route) rt.save() @manager.arg('route', help='specify a base route to compare') diff --git a/regression_test_dims.py b/regression_test_dims.py new file mode 100644 index 0000000000000000000000000000000000000000..fcf882fc24463b2e4dc6dd42a56bc0ffa613e474 --- /dev/null +++ b/regression_test_dims.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre +Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR + +This file is part of lde-api-regression-test. + +lde-api-regression-test is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +lde-api-regression-test is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with lde-api-regression-test. If not, see <https://www.gnu.org/licenses/>. +""" + +import sys +import pandas as pd +import numpy as np +import combinationdims +from termcolor import colored,cprint + +maxYear = 2021 + +class RegressionTest_dims: + + def __init__(self, route): + self.base_url = combinationdims.BASE_URL + self.route_list = combinationdims.BASE_ROUTE_LIST_COMBINATION + if len(route) > 0: + route = [route] + self.route_list = filter(lambda k: route[0] in k, self.route_list) + + def save(self): + for route in self.route_list: + name = route[0] + arguments = '' + if len(route) > 1: + arguments = arguments.join(route[1:]) + file_name = name+arguments + url = self.base_url+name+'?filter=min_year:1991,max_year:' + str(maxYear) + ','+arguments+'&format=csv' + url = url.replace(",&format","&format") + try: + api_csv = pd.read_csv(url,float_precision='round_trip', encoding="utf-8-sig") #get from api + file_name = file_name.replace("school/count","school_count") + file_name = file_name.replace("class_count/count","class_count_count") + file_name = file_name.replace("auxiliar/count", "auxiliar_count") + file_name = file_name.replace("simulation/time", "simulation_time") + file_name = file_name.replace("simulation/pqr", "simulation_pqr") + api_csv.to_csv('route_result/'+file_name+'.csv', encoding="utf-8-sig") #save + cprint("Saved "+name+" with arguments: ["+arguments+"] URL: <"+url+">",'green') + except Exception as ex: + cprint(str(ex)+" Not saved, a problem ocurred at "+file_name+" "+url,'red') + + def compare(self, verbose): + fail = 0 + for route in self.route_list: + name = route[0] + arguments = '' + if len(route) > 1: + arguments = arguments.join(route[1:]) + file_name = name+arguments + url = self.base_url+name+'?filter=min_year:1991,max_year:'+str(maxYear)+','+arguments+'&format=csv' + url = url.replace(",&format","&format") + try: + api_csv = pd.read_csv(url,float_precision='round_trip', encoding="utf-8-sig") #get from api + file_name = file_name.replace("school/count","school_count") + file_name = file_name.replace("class_count/count","class_count_count") + file_name = file_name.replace("auxiliar/count", "auxiliar_count") + file_name = file_name.replace("simulation/time", "simulation_time") + file_name = file_name.replace("simulation/pqr", "simulation_pqr") + csv_route = pd.read_csv('route_result/'+file_name+'.csv', index_col=0, float_precision='round_trip', encoding="utf-8-sig") #get file + if csv_route.equals(api_csv): #comparing csv + cprint(file_name+' OK!','green') + else: + fail+=1 + cprint(file_name+' FAIL!','red') + if verbose: + print(pd.concat([csv_route, api_csv]).drop_duplicates(keep=False)) + except Exception as ex: + cprint(str(ex)+"\n"+file_name+' FAIL!','red') + fail+=1 + cprint('TOTAL FAIL: '+str(fail),'red') + if (fail > 0): + exit(-1) diff --git a/settings.py b/settings.py index adee7fb1eac65580be7b8549b3282e1c27839161..36f1fdd07faaafaf247019b87eef9a945a1f8d71 100644 --- a/settings.py +++ b/settings.py @@ -226,7 +226,7 @@ SIMCAQ_ROUTE_LIST = [ ['classroom', '&dims=region'], ['classroom', '&dims=state'], ['classroom', '&dims=city'], - ['classroom', '&dims=school'], + #['classroom', '&dims=school'], #### Population ['population', '&dims=state'], @@ -264,43 +264,43 @@ SIMCAQ_ROUTE_LIST = [ ['enrollment_projection','&dims=max_year'], #### Daily Charge Amount - ['daily_charge_amount','&dims=region'], - ['daily_charge_amount','&dims=state'], - ['daily_charge_amount','&dims=location'], + # ['daily_charge_amount','&dims=region'], + # ['daily_charge_amount','&dims=state'], + # ['daily_charge_amount','&dims=location'], - #### Transport - ['transport', '&dims=school'], - ['transport', '&dims=region'], - ['transport', '&dims=state'], - ['transport', '&dims=city'], - ['transport', '&dims=location'], - ['transport', '&dims=rural_location'], - ['transport', '&dims=adm_dependency'], - ['transport', '&dims=adm_dependency_detailed'], - ['transport', '&dims=transportation_manager'], - ['transport', '&dims=service_type'], + # #### Transport + # ['transport', '&dims=school'], + # ['transport', '&dims=region'], + # ['transport', '&dims=state'], + # ['transport', '&dims=city'], + # ['transport', '&dims=location'], + # ['transport', '&dims=rural_location'], + # ['transport', '&dims=adm_dependency'], + # ['transport', '&dims=adm_dependency_detailed'], + # ['transport', '&dims=transportation_manager'], + # ['transport', '&dims=service_type'], - #### Class - ['class', '&dims=education_level_mod'], - ['class', '&dims=adm_dependency'], - ['class', '&dims=adm_dependency_detailed'], - ['class', '&dims=location'], - ['class', '&dims=region'], - ['class', '&dims=state'], - ['class', '&dims=city'], - ['class', '&dims=school'], - ['class', '&dims=period'], - ['class', '&dims=integral_time'], + # #### Class + # ['class', '&dims=education_level_mod'], + # ['class', '&dims=adm_dependency'], + # ['class', '&dims=adm_dependency_detailed'], + # ['class', '&dims=location'], + # ['class', '&dims=region'], + # ['class', '&dims=state'], + # ['class', '&dims=city'], + # ['class', '&dims=school'], + # ['class', '&dims=period'], + # ['class', '&dims=integral_time'], - #### University - ['university', '&dims=state'], - ['university', '&dims=city'], - ['university', '&dims=region'], - ['university', '&dims=min_year'], - ['university', '&dims=max_year'], - ['university', '&dims=upper_adm_dependency'], - ['university', '&dims=academic_organization'], - ['university', '&dims=capital'], + # #### University + # ['university', '&dims=state'], + # ['university', '&dims=city'], + # ['university', '&dims=region'], + # ['university', '&dims=min_year'], + # ['university', '&dims=max_year'], + # ['university', '&dims=upper_adm_dependency'], + # ['university', '&dims=academic_organization'], + # ['university', '&dims=capital'], ['school/count', '&dims=adm_dependency_detailed,location'], ['enrollment', '&dims=adm_dependency_detailed,location'],