Dims done

combinationdims.py

+from itertools import combinations
+
+BASE_URL = 'https://simcaq.c3sl.ufpr.br/api/v1/'
+
+BASE_ROUTE_LIST_COMBINATION = []
+
+#### Enrollments 
+for i in list(combinations(['location','rural_location','school_year','educational_level','adm_dependency','gender','ethnic_group','period','adm_dependency_detailed','educational_level_mod','educational_level_short','region','state','city','school'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['enrollment','&dims='+i[0]+','+i[1]])
+
+#### Course Count
+for i in list(combinations(['academic_level','state','mesoregion','microregion','localOffer','city','course','campi','upper_adm_dependency','upper_education_mod','academic_organization','university','universityLocalOffer','ocde_specific','ocde_geral','ocde_detailed','cine_specific','cine_geral','cine_detailed','is_free','night_time','situation','year'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['course_count','&dims='+i[0]+','+i[1]])
+
+#### Disciplines
+for i in list(combinations(['adm_dependency','adm_dependency_detailed','contract_type','education_level_mod','education_level_short','education_type','region','mesoregion','microregion','state','city','location','rural_location','min_year','max_year','gender','ethnic_group','discipline'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['discipline','&dims='+i[0]+','+i[1]])
+
+#### Employees
+for i in list(combinations(['adm_dependency','adm_dependency_detailed','region','state','city','location','diff_location','rural_locations','function','year'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['employee','&dims='+i[0]+','+i[1]])
+
+#### Distribution Factor
+for i in list(combinations(['region','state','city'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['distribution_factor','&dims='+i[0]+','+i[1]])
+
+#### Financial
+for i in list(combinations(['min_year','max_year','sphere_adm','city','state','financial_data'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['financial','&dims='+i[0]+','+i[1]])
+
+#### IDHM, IDHMR, IDHML, IDHME 
+for i in list(combinations(['state','city'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['idhm','&dims='+i[0]+','+i[1]])
+    BASE_ROUTE_LIST_COMBINATION.append(['idhmr','&dims='+i[0]+','+i[1]])
+    BASE_ROUTE_LIST_COMBINATION.append(['idhml','&dims='+i[0]+','+i[1]])
+    BASE_ROUTE_LIST_COMBINATION.append(['idhme','&dims='+i[0]+','+i[1]])
+
+#### Out of School
+for i in list(combinations(['region','state','ethnic_group','full_age_range','gender','location','extremes_household_income','fifth_household_income','year'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['out_of_school','&dims='+i[0]+','+i[1]])
+
+#### Infrastructure
+for i in list(combinations(['region','state','location','rural_location','adm_dependency','adm_dependency_detailed'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['infrastructure','&dims='+i[0]+','+i[1]])
+
+#### School Location
+for i in list(combinations(['mesoregion','microregion','city'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['school_location','&dims='+i[0]+','+i[1]])
+
+#### Auxiliar Count
+for i in list(combinations(['adm_dependency','adm_dependency_detailed','education_level_mod','education_type','region','state','city','location','rural_location','school','gender','ethnic_group'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['auxiliar/count','&dims='+i[0]+','+i[1]])
+
+#### School Count
+for i in list(combinations(['location','adm_dependency','adm_dependency_detailed','government_agreement','agreement','education_day_care_child','education_preschool_child','education_begin_elementary_school','education_middle_school','education_professional','education_eja'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['school/count','&dims='+i[0]+','+i[1]])
+
+#### Classroom
+for i in list(combinations(['adm_dependency','adm_dependency_detailed','location','region','state','city'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['classroom','&dims='+i[0]+','+i[1]])
+
+#### Population
+for i in list(combinations(['region','state','city','city_size'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['population','&dims='+i[0]+','+i[1]])
+
+#### Pibpercapita
+for i in list(combinations(['region','state','city','income_level'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['pibpercapita','&dims='+i[0]+','+i[1]])
+
+#### Teacher
+for i in list(combinations(['adm_dependency','adm_dependency_detailed','education_level_mod','education_level_short','education_type','location','rural_location','region','state','city','school','gender','contract_type','ethnic_group'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['teacher','&dims='+i[0]+','+i[1]])
+
+#### Enrollment Project
+for i in list(combinations(['adm_dependency','state','city','min_year','max_year'],2)):
+    BASE_ROUTE_LIST_COMBINATION.append(['enrollment_project','&dims='+i[0]+','+i[1]])
+

manage.py

@@ -21,6 +21,7 @@ along with lde-api-regression-test.  If not, see <https://www.gnu.org/licenses/>
 from manager import Manager
 
 from regression_test import RegressionTest
+from regression_test_dims import RegressionTest_dims
 
 manager = Manager()
 
@@ -28,7 +29,8 @@ manager = Manager()
 @manager.command
 def save(route=''):
     '''Save multiple or one route'''
-    rt = RegressionTest(route)
+    #rt = RegressionTest(route)
+    rt = RegressionTest_dims(route)
     rt.save()
 
 @manager.arg('route', help='specify a base route to compare')

regression_test_dims.py

+#!/usr/bin/env python3
+"""
+Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
+Departamento de Informatica - Universidade Federal do Parana - C3SL/UFPR
+
+This file is part of lde-api-regression-test.
+
+lde-api-regression-test is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+lde-api-regression-test is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with lde-api-regression-test.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+import sys
+import pandas as pd
+import numpy as np
+import combinationdims
+from termcolor import colored,cprint
+
+maxYear = 2021
+
+class RegressionTest_dims:
+
+	def __init__(self, route):
+		self.base_url = combinationdims.BASE_URL
+		self.route_list = combinationdims.BASE_ROUTE_LIST_COMBINATION
+		if len(route) > 0:
+			route = [route]
+			self.route_list = filter(lambda k: route[0] in k, self.route_list)
+
+	def save(self):
+		for route in self.route_list:
+			name = route[0]
+			arguments = ''
+			if len(route) > 1:
+				arguments = arguments.join(route[1:])
+			file_name = name+arguments
+			url = self.base_url+name+'?filter=min_year:1991,max_year:' + str(maxYear) + ','+arguments+'&format=csv'
+			url = url.replace(",&format","&format")
+			try:
+				api_csv = pd.read_csv(url,float_precision='round_trip', encoding="utf-8-sig") #get from api
+				file_name = file_name.replace("school/count","school_count")
+				file_name = file_name.replace("class_count/count","class_count_count")
+				file_name = file_name.replace("auxiliar/count", "auxiliar_count")
+				file_name = file_name.replace("simulation/time", "simulation_time")
+				file_name = file_name.replace("simulation/pqr", "simulation_pqr")
+				api_csv.to_csv('route_result/'+file_name+'.csv', encoding="utf-8-sig") #save
+				cprint("Saved "+name+" with arguments: ["+arguments+"] URL: <"+url+">",'green')
+			except Exception as ex:
+				cprint(str(ex)+" Not saved, a problem ocurred at "+file_name+" "+url,'red')
+	
+	def compare(self, verbose):
+		fail = 0
+		for route in self.route_list:
+			name = route[0]
+			arguments = ''
+			if len(route) > 1:
+				arguments = arguments.join(route[1:])
+			file_name = name+arguments
+			url = self.base_url+name+'?filter=min_year:1991,max_year:'+str(maxYear)+','+arguments+'&format=csv'
+			url = url.replace(",&format","&format")
+			try:
+				api_csv = pd.read_csv(url,float_precision='round_trip', encoding="utf-8-sig") #get from api
+				file_name = file_name.replace("school/count","school_count")
+				file_name = file_name.replace("class_count/count","class_count_count")
+				file_name = file_name.replace("auxiliar/count", "auxiliar_count")
+				file_name = file_name.replace("simulation/time", "simulation_time")
+				file_name = file_name.replace("simulation/pqr", "simulation_pqr")
+				csv_route = pd.read_csv('route_result/'+file_name+'.csv', index_col=0, float_precision='round_trip', encoding="utf-8-sig") #get file
+				if csv_route.equals(api_csv): #comparing csv
+					cprint(file_name+' OK!','green')
+				else:
+					fail+=1
+					cprint(file_name+' FAIL!','red')
+					if verbose:
+						print(pd.concat([csv_route, api_csv]).drop_duplicates(keep=False))
+			except Exception as ex:
+					cprint(str(ex)+"\n"+file_name+' FAIL!','red')
+					fail+=1
+		cprint('TOTAL FAIL: '+str(fail),'red')
+		if (fail > 0):
+			exit(-1)

settings.py

@@ -226,7 +226,7 @@ SIMCAQ_ROUTE_LIST = [
     ['classroom', '&dims=region'],
     ['classroom', '&dims=state'],
     ['classroom', '&dims=city'],
-    ['classroom', '&dims=school'],
+    #['classroom', '&dims=school'],
     
 	#### Population 
 	['population', '&dims=state'],
@@ -264,43 +264,43 @@ SIMCAQ_ROUTE_LIST = [
 	['enrollment_projection','&dims=max_year'],
     
 	#### Daily Charge Amount 
-	['daily_charge_amount','&dims=region'],
-	['daily_charge_amount','&dims=state'],
-    ['daily_charge_amount','&dims=location'],
+	# ['daily_charge_amount','&dims=region'],
+	# ['daily_charge_amount','&dims=state'],
+    # ['daily_charge_amount','&dims=location'],
     
-	#### Transport
-	['transport', '&dims=school'],
-    ['transport', '&dims=region'],
-    ['transport', '&dims=state'],
-    ['transport', '&dims=city'],
-    ['transport', '&dims=location'],
-	['transport', '&dims=rural_location'],
-    ['transport', '&dims=adm_dependency'],
-    ['transport', '&dims=adm_dependency_detailed'],
-    ['transport', '&dims=transportation_manager'],
-    ['transport', '&dims=service_type'],
+	# #### Transport
+	# ['transport', '&dims=school'],
+    # ['transport', '&dims=region'],
+    # ['transport', '&dims=state'],
+    # ['transport', '&dims=city'],
+    # ['transport', '&dims=location'],
+	# ['transport', '&dims=rural_location'],
+    # ['transport', '&dims=adm_dependency'],
+    # ['transport', '&dims=adm_dependency_detailed'],
+    # ['transport', '&dims=transportation_manager'],
+    # ['transport', '&dims=service_type'],
     
-	#### Class
-	['class', '&dims=education_level_mod'],
-    ['class', '&dims=adm_dependency'],
-	['class', '&dims=adm_dependency_detailed'],
-	['class', '&dims=location'],
-	['class', '&dims=region'],
-    ['class', '&dims=state'],
-    ['class', '&dims=city'],
-    ['class', '&dims=school'],
-	['class', '&dims=period'],
-	['class', '&dims=integral_time'],
+	# #### Class
+	# ['class', '&dims=education_level_mod'],
+    # ['class', '&dims=adm_dependency'],
+	# ['class', '&dims=adm_dependency_detailed'],
+	# ['class', '&dims=location'],
+	# ['class', '&dims=region'],
+    # ['class', '&dims=state'],
+    # ['class', '&dims=city'],
+    # ['class', '&dims=school'],
+	# ['class', '&dims=period'],
+	# ['class', '&dims=integral_time'],
 
-	#### University
-	['university', '&dims=state'],
-    ['university', '&dims=city'],
-    ['university', '&dims=region'],
-    ['university', '&dims=min_year'],
-    ['university', '&dims=max_year'],
-    ['university', '&dims=upper_adm_dependency'],
-    ['university', '&dims=academic_organization'],
-    ['university', '&dims=capital'],
+	# #### University
+	# ['university', '&dims=state'],
+    # ['university', '&dims=city'],
+    # ['university', '&dims=region'],
+    # ['university', '&dims=min_year'],
+    # ['university', '&dims=max_year'],
+    # ['university', '&dims=upper_adm_dependency'],
+    # ['university', '&dims=academic_organization'],
+    # ['university', '&dims=capital'],
     
 	['school/count', '&dims=adm_dependency_detailed,location'],
 	['enrollment', '&dims=adm_dependency_detailed,location'],