diff --git a/database/actions.py b/database/actions.py
index 83ee1edf6265c825c8e4bb41026b7644e7497733..749b26c8d67008c827cfa66557fd92e27479f44b 100644
--- a/database/actions.py
+++ b/database/actions.py
@@ -70,13 +70,13 @@ def insert(file_name, table, year, offset=2, delimiters=[';', '\\n', '"'], null=
trans.commit()
-def create(table):
+def create(table, ignore_definitions=False):
'''Creates table from mapping_protocol metadata'''
table = gen_data_table(table, META)
with ENGINE.connect() as connection:
trans = connection.begin()
- table.create(bind=connection)
+ table.create(bind=connection, ignore_definitions=ignore_definitions)
table.set_source(bind=connection)
table.create_mapping_table(bind=connection)
trans.commit()
@@ -87,12 +87,12 @@ def drop(table):
table.drop()
-def remap(table, auto_confirmation=True):
+def remap(table, auto_confirmation=True, verify_definitions=False):
'''Applies change made in mapping protocols to database'''
table = gen_data_table(table, META)
table.map_from_database()
- table.remap(auto_confirmation)
+ table.remap(auto_confirmation, verify_definitions)
def csv_from_tabbed(table_name, input_file, output_file, year, sep=';'):
table = gen_data_table(table_name, META)
diff --git a/database/database_table.py b/database/database_table.py
index 99fdb2d8fe0fc27bf73c0e53d2b42470fb065e63..d3c0dce175e39374474f1c78672621e38ba95e73 100644
--- a/database/database_table.py
+++ b/database/database_table.py
@@ -321,7 +321,15 @@ class DatabaseTable(Table):
logger.debug("Definitions Updated")
- def get_columns_to_create(self, tdef_columns):
+ def get_columns_dict(self, tdef_columns, ignore_diff=False):
+ '''
+ Get a dictionary of columns, comparing the columns of the associated protocol with those the columns in table
+ definition.
+ :param tdef_columns: column dict from table definitions
+ :param ignore_diff: when set True will ignore differences in table_definition and get the data only from
+ mapping_protocol when both exists
+ :return: {"column_name": ["column_type(str)", "target"]}
+ '''
if self._protocol is None:
if not tdef_columns:
raise MissingProtocolError("You must first load a protocol or add columns to the table definition")
@@ -339,17 +347,42 @@ class DatabaseTable(Table):
column[0] = column[0].strip()
column_dict[column[0]] = [column[1], self._protocol.target_from_dbcolumn(column[0])]
- for c_name, c_type in tdef_columns.items():
- if c_name not in column_dict.keys():
- prompt = input("The column {} is not present on the mapping protocol but is on the table definition,"
- "should it still be created ? (Y/n): ".format(c_name))
- if prompt.upper() in ['', 'Y']:
- print("Column {} will be created, please update the protocol later".format(c_name))
- column_dict[c_name] = c_type
- else:
- print("Column {} will be removed from the table_definitions.".format(c_name))
+ if not ignore_diff:
+ for c_name, c_type in tdef_columns.items():
+ if c_name not in column_dict.keys():
+ prompt = input("The column {} is not present on the mapping protocol but is on the table definition,"
+ " should it exist ? (Y/n): ".format(c_name))
+ if prompt.upper() in ['', 'Y']:
+ print("Column {} will be created, please update the protocol later".format(c_name))
+ column_dict[c_name] = c_type
+ else:
+ print("Column {} will be removed from the table_definitions.".format(c_name))
+
return column_dict
+ def get_targets_from_definitions(self):
+ '''
+ Get a list containing all the targets from table definition
+ '''
+ targets = []
+ definitions = self.get_definitions()
+ for column_name, parameter_list in definitions['columns'].items():
+ targets.append(parameter_list[1])
+
+ return targets
+
+ def get_dbcolumn_from_target_definition(self, target):
+ '''
+ Gets database column from a target column name. Ouput is a list
+ with the column name and type contents.
+ :return: ['column_name','column_type']
+ '''
+ definitions = self.get_definitions()
+ for column_name, parameter_list in definitions['columns'].items():
+ if parameter_list[1] == target:
+ return [column_name, parameter_list[0]]
+
+
def load_protocol(self, protocol):
'''
Takes a Protocol instance and loads it for further use
@@ -426,10 +459,11 @@ class DatabaseTable(Table):
bind.execute(base_query)
- def map_from_protocol(self, create=False, bind=None):
+ def map_from_protocol(self, create=False, bind=None, ignore_defintions=False):
'''
- Uses information from a protocol to generate self columns. Table definitions must also
- be defined to allow primary key and foreign keys addition.
+ Uses information from a protocol or, if protocol is not present, from table definitions
+ to generate self columns.
+ Table definitions must also be defined to allow primary key and foreign keys addition.
Useful for table creation.
'''
if self.columns.keys():
@@ -439,7 +473,7 @@ class DatabaseTable(Table):
bind = self.metadata.bind
definitions = self.get_definitions()
- column_dict = self.get_columns_to_create(definitions.get('columns'))
+ column_dict = self.get_columns_dict(definitions.get('columns'), ignore_defintions)
for c_name, c_type in column_dict.items():
column = Column(c_name, get_type(c_type[0]))
@@ -475,7 +509,7 @@ class DatabaseTable(Table):
self.constraints.add(ForeignKeyConstraint(keys, fkeys))
- def create(self, bind=None, checkfirst=False):
+ def create(self, bind=None, checkfirst=False, ignore_definitions=False):
'''
Overrides sqlalchemy's create method to use map_from_protocol before creating.
'''
@@ -485,7 +519,7 @@ class DatabaseTable(Table):
logger.error("Table %s already exists", self.name)
return
- self.map_from_protocol(create=True, bind=bind)
+ self.map_from_protocol(create=True, bind=bind, ignore_defintions=ignore_definitions)
super().create(bind=bind, checkfirst=checkfirst)
@@ -636,7 +670,7 @@ class DatabaseTable(Table):
def compare_mapping(self):
'''
- Compares contents of mapping table to protocol and returns tuple with differences in
+ Compares contents of mapping table to table definitions and returns tuple with differences in
the following format:
new_columns, to_drop_columns, update_columns
@@ -647,19 +681,17 @@ class DatabaseTable(Table):
The method uses target_names as the criteria to decide if columns are the same or not.
'''
- self.check_protocol()
-
- protocol_target_list = self._protocol.get_targets()
+ target_list = self.get_targets_from_definitions()
query = self._mapping_table.select()
results = self.metadata.bind.execute(query).fetchall()
db_target_list = [t[1] for t in results]
- new_columns = [c for c in protocol_target_list if c not in db_target_list and c != '']
- to_drop_columns = [c for c in db_target_list if c not in protocol_target_list]
+ new_columns = [c for c in target_list if c not in db_target_list and c != '']
+ to_drop_columns = [c for c in db_target_list if c not in target_list]
update_columns = []
- for target in protocol_target_list:
+ for target in target_list:
query = select([self._mapping_table.c.name, self._mapping_table.c.type])\
.where(self._mapping_table.c.target_name == target)
result = self.metadata.bind.execute(query).fetchone()
@@ -667,7 +699,7 @@ class DatabaseTable(Table):
continue
name, field_type = result
try:
- new_name, new_type = self._protocol.dbcolumn_from_target(target)
+ new_name, new_type = self.get_dbcolumn_from_target_definition(target)
except InvalidTargetError:
to_drop_columns.append(target)
continue
@@ -683,20 +715,24 @@ class DatabaseTable(Table):
return new_columns, to_drop_columns, update_columns
- def remap(self, auto_confirmation=True):
+ def remap(self, auto_confirmation=True, verify_definitions=False):
'''
Checks mapping protocol for differences in table structure - then
attempts to apply differences according to what is recorded in the
- mapping table
+ mapping table.
+ If verify_definitions is set it will ask any difference between mapping_protocol and table_definition
'''
if not self.exists():
print("Table {} doesn't exist".format(self.name))
return
- self.check_protocol()
-
mtable = self._mapping_table
+ # Update table definitions
+ definitions = self.get_definitions()
+ definitions['columns'] = self.get_columns_dict(definitions.get('columns'), ignore_diff=not verify_definitions)
+ self.update_defintions(definitions)
+
if not mtable.exists():
print("Mapping table for {} not found.".format(self.name))
print("Creating mapping table...")
diff --git a/manage.py b/manage.py
index cb87c7c46a4cb38fb9247110ae4f8cdc646b9917..efe2f9e5cbfed0d02d83ca0afc6c2f6ad7f8630f 100755
--- a/manage.py
+++ b/manage.py
@@ -35,9 +35,11 @@ def insert(csv_file, table, year, sep=';', null='',notifybackup=None):
if notifybackup:
database.actions.generate_backup()
@manager.command
-def create(table):
- '''Creates table using mapping protocols'''
- database.actions.create(table)
+def create(table, ignore_definitions=False):
+ '''Creates table using mapping protocols
+ If ignore_definitions is set, it will ignore the columns from table definition if both, table_definitions and
+ mapping_protocol, exists (though it will still get primary_key, foreign_key and source information)'''
+ database.actions.create(table, ignore_definitions)
@manager.command
def drop(table):
@@ -45,9 +47,11 @@ def drop(table):
database.actions.drop(table)
@manager.command
-def remap(table, auto_confirmation=False):
- '''Restructures a table to match the mapping protocol.'''
- database.actions.remap(table, auto_confirmation)
+def remap(table, auto_confirmation=False, verify_definitions=False):
+ '''Restructures a table to match the mapping protocol.
+ If auto_confirmation is set it will not ask before doing any operation
+ If verify_definitions is set it will ask any difference between mapping_protocol and table_definition'''
+ database.actions.remap(table, auto_confirmation, verify_definitions)
@manager.command
def update_from_file(csv_file, table, year, columns=None, target_list=None, offset=2, sep=';',
diff --git a/tests/database_test.py b/tests/database_test.py
index b0a34973ccde3515e81c4e1c1df4c10393729ef3..ca4ddd896e30785c9197ffb456c79555c1540702 100755
--- a/tests/database_test.py
+++ b/tests/database_test.py
@@ -91,7 +91,7 @@ def modify_mapping_protocol():
def test_creation():
if not ENGINE.dialect.has_table(ENGINE, 'test_reference'):
database.actions.execute_sql_script('test_reference.sql')
- database.actions.create(table_test)
+ database.actions.create(table_test, ignore_definitions=True)
print("Executing fetchall query:")
with ENGINE.connect() as connection:
table = Table(table_test, META, autoload=True, autoload_with=ENGINE)