diff --git a/database/database_table.py b/database/database_table.py
index 8483a8526ef43d31f2237b2a6d2f28a364801703..b785db32c1ec698bebf9aa7899bb17de0068cab9 100644
--- a/database/database_table.py
+++ b/database/database_table.py
@@ -818,13 +818,12 @@ class DatabaseTable(Table):
case = case.strip("~") # doesn't need "~" anymore
str_list = re.findall(r'("[\w]"|[\w.]+)', case)
- # str_list = case.split()
for substr in str_list:
- # print('ss', substr)
if '.' in substr: # We have a var from another table
table = substr.split('.')[0]
table = gen_data_table(table, self.metadata)
var_name = substr.split('.')[1]
+ level += 1
else:
table = self
var_name = substr
@@ -833,11 +832,11 @@ class DatabaseTable(Table):
if table._protocol and table._protocol.target_from_dbcolumn(var_name) is not None:
# Prevents lowering level if var is an aggregation.
level = max(level + table._derivative_recursion(var_name, year, recursion_list)['level'], level)
- else:
- var_target = self._get_variable_target(var_name.strip('"'), year)
- if var_target is not None:
- var_db = self._protocol.dbcolumn_from_target(var_target)[0]
- case = case.replace(var_name, var_db)
+
+ var_target = self._get_variable_target(var_name.strip('"'), year)
+ if var_target is not None:
+ var_db = self._protocol.dbcolumn_from_target(var_target)[0]
+ case = case.replace(var_name, var_db)
if table is not self and table not in referred_tables:
referred_tables.append(table)
@@ -854,65 +853,41 @@ class DatabaseTable(Table):
self._derivatives = {}
return self._derivative_recursion(original, year)
- def _get_denormalizations(self, ttable, originals, year, bind):
- '''
- Searches protocol for denormalizations and yields the necessary update queries.
- '''
- exp = r'([a-zA-Z0-9_]+)\.([a-zA-Z0-9_]+)'
- external = {}
- for dst, original in originals:
- original = original.strip(' ~\n\t')
- for match in re.finditer(exp, original):
- table, column = match.groups()
- if table not in external:
- external[table] = []
- external[table].append([dst, text(original)])
-
- for table in external:
- query = update(ttable)
-
- referred_table = gen_data_table(table, self.metadata)
- referred_table.map_from_database(bind)
- try:
- fk_tuples = [(fk_column, fkey) for fk_column, fkey in self.get_relations(referred_table)]
- except MissingForeignKeyError:
- logger.warning("Using relations from " + table + " instead of " + str(self) + " to apply derivative.")
- fk_tuples = [(fk_column, fkey) for fk_column, fkey in referred_table.get_relations(self)]
+ def _apply_denormalization(self, ttable, year, column, denorm_query, referred_tables, bind):
- for dst, src in external[table]:
- query = query.values(**{dst[0]: src})
- for fk_column, fkey in fk_tuples:
- # fk_column = ttable.columns.get(fk_column.name)
- query = query.where(fk_column == fkey)
- if year:
- query = query.where(ttable.columns.get(settings.YEAR_COLUMN) == year)
- yield query
+ # Hack to make pymonetdb be able to work with the columns from 2 tables when one is temporary
+ t_schema = ttable.schema
+ ttable.schema = None
- def _apply_denormalization(self, ttable, year, column, denorm_query, referred_tables, bind):
query = update(ttable)
- print(ttable.name)
query = query.values(**{column: text(denorm_query)})
- print('qc', query, column)
for ref_table in referred_tables:
ref_table.map_from_database(bind)
- try:
- fk_tuples = [(ttable.columns.get(fk_column.name), fkey)
- for fk_column, fkey in self.get_relations(ref_table)]
- except MissingForeignKeyError:
- logger.warning("Using relations from " + str(ref_table) +
+ fk_tuples = [(ttable.columns.get(fk_column.name), fkey)
+ for fk_column, fkey in self.get_relations(ref_table)]
+ if not fk_tuples:
+ logger.warning("Trying to use relations from " + str(ref_table) +
" instead of " + str(self) + " to apply derivative.")
fk_tuples = [(ttable.columns.get(fkey.name),fk_column)
for fk_column, fkey in ref_table.get_relations(self)]
+ if not fk_tuples:
+ logger.error("COULDN'T ESTABLISH " + ref_table.name + " RELATION WITH " + self.name +
+ " IGNORING COLUMN: " + column)
+ ttable.schema = t_schema
+ return
for fk_column, fkey in fk_tuples:
query = query.where(fk_column == fkey)
if year:
query = query.where(ttable.columns.get(settings.YEAR_COLUMN) == year)
+
bind.execute(query)
+ ttable.schema = t_schema
+
def apply_derivatives(self, ttable, columns, year, bind=None, dbonly=False):
'''
Given a list of columns, searches for derivatives and denormalizations and applies them
@@ -933,7 +908,8 @@ class DatabaseTable(Table):
level = [self._derivatives[d] for d in self._derivatives if self._derivatives[d]['level'] == i]
for derivative in level:
if len(derivative['tables']) == 0:
- query[derivative['dbcolumn'][0]] = text(derivative['query'])
+ if not dbonly:
+ query[derivative['dbcolumn'][0]] = text(derivative['query'])
else:
self._apply_denormalization(ttable, year, derivative['dbcolumn'][0], derivative['query'],
derivative['tables'], bind)
@@ -942,35 +918,6 @@ class DatabaseTable(Table):
query = update(ttable).values(**query)
bind.execute(query)
-
-
-
- print(self._derivatives)
-
- # originals = [(self._derivatives[d]['dbcolumn'], self._derivatives[d]['original'])\
- # for d in self._derivatives if self._derivatives[d]['level'] == 0]
- #
- # t_schema = ttable.schema
- # ttable.schema = None
- # for query in self._get_denormalizations(ttable, originals, year, bind):
- # bind.execute(query)
- #
- # ttable.schema = t_schema
- # if len(self._derivatives) > 0:
- # max_level = max([self._derivatives[d]['level'] for d in self._derivatives])
- # for i in range(max_level):
- # i = i+1
- # query = {}
- # level = [self._derivatives[d] for d in self._derivatives if\
- # self._derivatives[d]['level'] == i]
- # for derivative in level:
- # if not dbonly or derivative['dbmapped']:
- # query[derivative['dbcolumn'][0]] = text(derivative['processed'])
- #
- # query = update(ttable).values(**query)
- #
- # bind.execute(query)
-
return self._derivatives
def _get_aggregations(self, year):
@@ -1049,7 +996,8 @@ class DatabaseTable(Table):
foreign_key = fk
break
if not foreign_key:
- raise MissingForeignKeyError(table)
+ logger.warning("Couldn't find foreign key relation between " + self.name + " and " + table.name)
+ return None
for _, fk_column in foreign_key.columns.items():
fkey = list(fk_column.foreign_keys)[0]
fkey = fkey.column.name
diff --git a/settings.py b/settings.py
index b8c1601b483193018e701c5750fc076ef3059c1e..36dd10f504d092e046af72592e81115e2fe7195c 100644
--- a/settings.py
+++ b/settings.py
@@ -33,7 +33,7 @@ DATABASE_USER_PASSWORD = 'monetdb'
DATABASE_HOST = 'localhost'
# Database to connect to
-DATABASE = 'test'
+DATABASE = 'hotmapper_demo'
# Column used to run aggregations and denormalizations
YEAR_COLUMN = 'ano_censo'