From f0e32e371e78bc05a6a142a1b72fb2ed9534c7e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sat, 30 Apr 2016 12:02:15 +0200 Subject: [PATCH 1/4] general: add geolocation to formdata (#10581) --- tests/test_formdef_import.py | 10 +++++++++ tests/test_sql.py | 49 ++++++++++++++++++++++++++++++++++++++++++++ wcs/formdata.py | 10 +++++++++ wcs/formdef.py | 22 ++++++++++++++++++++ wcs/sql.py | 40 +++++++++++++++++++++++++++++++----- 5 files changed, 126 insertions(+), 5 deletions(-) diff --git a/tests/test_formdef_import.py b/tests/test_formdef_import.py index dba874c..20a1171 100644 --- a/tests/test_formdef_import.py +++ b/tests/test_formdef_import.py @@ -340,3 +340,13 @@ def test_workflow_roles(): role.remove_self() fd2 = FormDef.import_from_xml_tree(xml_export, include_id=True) assert fd2.workflow_roles.get('_receiver') is None + +def test_geolocations(): + formdef = FormDef() + formdef.name = 'foo' + formdef.fields = [] + formdef.geolocations = {'base': 'Base'} + fd2 = assert_xml_import_export_works(formdef, include_id=True) + assert fd2.geolocations == formdef.geolocations + fd3 = assert_json_import_export_works(formdef) + assert fd3.geolocations == formdef.geolocations diff --git a/tests/test_sql.py b/tests/test_sql.py index 9daf98c..28992b4 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -178,6 +178,55 @@ def test_sql_field_items(): check_sql_field('6', ['apricot', 'pear']) @postgresql +def test_sql_geoloc(): + test_formdef = FormDef() + test_formdef.name = 'geoloc' + test_formdef.fields = [] + test_formdef.geolocations = {'base': 'Plop'} + test_formdef.store() + data_class = test_formdef.data_class(mode='sql') + formdata = data_class() + formdata.data = {} + formdata.geolocations = {'base': {'lat': 12, 'lon': 21}} + formdata.store() + + formdata2 = data_class.get(formdata.id) + assert formdata2.geolocations == formdata.geolocations + + formdata.geolocations = {} + formdata.store() + formdata2 = data_class.get(formdata.id) + assert formdata2.geolocations == formdata.geolocations + +@postgresql +def test_sql_multi_geoloc(): + test_formdef = FormDef() + test_formdef.name = 'geoloc' + test_formdef.fields = [] + test_formdef.geolocations = {'base': 'Plop'} + test_formdef.store() + data_class = test_formdef.data_class(mode='sql') + formdata = data_class() + formdata.data = {} + formdata.geolocations = {'base': {'lat': 12, 'lon': 21}} + formdata.store() + + formdata2 = data_class.get(formdata.id) + assert formdata2.geolocations == formdata.geolocations + + test_formdef.geolocations = {'base': 'Plop', '2nd': 'XXX'} + test_formdef.store() + formdata.geolocations = {'base': {'lat': 12, 'lon': 21}, '2nd': {'lat': -34, 'lon': -12}} + formdata.store() + formdata2 = data_class.get(formdata.id) + assert formdata2.geolocations == formdata.geolocations + + test_formdef.geolocations = {'base': 'Plop'} + test_formdef.store() + formdata2 = data_class.get(formdata.id) + assert formdata2.geolocations == {'base': {'lat': 12, 'lon': 21}} + +@postgresql def test_sql_change(): data_class = formdef.data_class(mode='sql') formdata = data_class() diff --git a/wcs/formdata.py b/wcs/formdata.py index b78143e..bc77361 100644 --- a/wcs/formdata.py +++ b/wcs/formdata.py @@ -175,6 +175,7 @@ class FormData(StorableObject): workflow_data = None workflow_roles = None + geolocations = None _formdef = None def get_formdef(self): @@ -505,6 +506,10 @@ class FormData(StorableObject): except KeyError: pass + if self.geolocations: + for k, v in self.geolocations.items(): + d['form_geoloc_%s' % k] = v + if self.evolution and self.evolution[-1].comment: d['form_comment'] = self.evolution[-1].comment else: @@ -548,6 +553,11 @@ class FormData(StorableObject): d['form_url'], formvar, fieldvar, self.workflow_data['%s_var_%s' % (formvar, fieldvar)]) + if self.geolocations: + for k, v in self.geolocations.items(): + d['form_geoloc_%s_lat' % k] = v.get('lat') + d['form_geoloc_%s_lon' % k] = v.get('lon') + d = copy.deepcopy(d) flatten_dict(d) diff --git a/wcs/formdef.py b/wcs/formdef.py index b7e0579..c44c39e 100644 --- a/wcs/formdef.py +++ b/wcs/formdef.py @@ -88,6 +88,8 @@ class FormDef(StorableObject): skip_from_360_view = False private_status_and_history = False + geolocations = None + last_modification_time = None last_modification_user_id = None @@ -549,6 +551,9 @@ class FormDef(StorableObject): for field in self.fields: root['fields'].append(field.export_to_json(include_id=include_id)) + if self.geolocations: + root['geolocations'] = self.geolocations.copy() + if self.workflow_options: root['options'] = self.workflow_options @@ -632,6 +637,9 @@ class FormDef(StorableObject): if value.get('options'): formdef.workflow_options = value.get('options') + if value.get('geolocations'): + formdef.geolocations = value.get('geolocations') + return formdef def export_to_xml(self, include_id=False): @@ -711,6 +719,12 @@ class FormDef(StorableObject): else: pass # TODO: extend support to other types + geolocations = ET.SubElement(root, 'geolocations') + for geoloc_key, geoloc_label in (self.geolocations or {}).items(): + element = ET.SubElement(geolocations, 'geolocation') + element.attrib['key'] = geoloc_key + element.text = unicode(geoloc_label, charset) + return root @classmethod @@ -861,6 +875,14 @@ class FormDef(StorableObject): formdef.workflow_roles[role_key] = role_id + if tree.find('geolocations') is not None: + geolocations_node = tree.find('geolocations') + formdef.geolocations = {} + for child in geolocations_node.getchildren(): + geoloc_key = child.attrib['key'] + geoloc_value = child.text.encode(charset) + formdef.geolocations[geoloc_key] = geoloc_value + return formdef def get_detailed_email_form(self, formdata, url): diff --git a/wcs/sql.py b/wcs/sql.py index 392cabc..7bb068d 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -17,6 +17,7 @@ import psycopg2 import datetime import time +import re import cPickle from quixote import get_publisher @@ -399,6 +400,13 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild cur.execute('''ALTER TABLE %s ADD COLUMN %s bytea''' % ( table_name, 'f%s_structured' % field.id)) + for field in (formdef.geolocations or {}).keys(): + column_name = 'geoloc_%s' % field + needed_fields.add(column_name) + if column_name not in existing_fields: + cur.execute('ALTER TABLE %s ADD COLUMN %s %s''' % ( + table_name, column_name, 'POINT')) + # delete obsolete fields for field in (existing_fields - needed_fields): cur.execute('''ALTER TABLE %s DROP COLUMN %s CASCADE''' % (table_name, field)) @@ -922,6 +930,8 @@ class SqlMixin(object): def _row2obdata(cls, row, formdef): obdata = {} i = len(cls._table_static_fields) + if formdef.geolocations: + i += len(formdef.geolocations.keys()) for field in formdef.fields: sql_type = SQL_TYPE_MAPPING.get(field.key, 'varchar') if sql_type is None: @@ -958,6 +968,7 @@ class SqlMixin(object): if obdata['%s_structured' % field.id] is None: del obdata['%s_structured' % field.id] i += 1 + return obdata @classmethod @@ -1128,6 +1139,12 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData): else: sql_dict['submission_context'] = None + for field in (self._formdef.geolocations or {}).keys(): + value = (self.geolocations or {}).get(field) + if value: + value = '(%.6f, %.6f)' % (value.get('lon'), value.get('lat')) + sql_dict['geoloc_%s' % field] = value + sql_dict['concerned_roles_array'] = [str(x) for x in self.concerned_roles if x] sql_dict['actions_roles_array'] = [str(x) for x in self.actions_roles if x] @@ -1242,12 +1259,22 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData): o.workflow_roles = cPickle.loads(str(o.workflow_roles)) if o.submission_context: o.submission_context = cPickle.loads(str(o.submission_context)) + + o.geolocations = {} + for i, field in enumerate((cls._formdef.geolocations or {}).keys()): + value = row[len(cls._table_static_fields)+i] + if not value: + continue + m = re.match(r"\(([^)]+),([^)]+)\)", value) + o.geolocations[field] = {'lon': float(m.group(1)), + 'lat': float(m.group(2))} + o.data = cls._row2obdata(row, cls._formdef) return o @classmethod def get_data_fields(cls): - data_fields = [] + data_fields = ['geoloc_%s' % x for x in (cls._formdef.geolocations or {}).keys()] for field in cls._formdef.fields: sql_type = SQL_TYPE_MAPPING.get(field.key, 'varchar') if sql_type is None: @@ -1274,8 +1301,10 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData): raise KeyError() conn, cur = get_connection_and_cursor() + fields = cls.get_data_fields() + potential_comma = ', ' - if not cls.get_data_fields(): + if not fields: potential_comma = '' sql_statement = '''SELECT %s @@ -1285,7 +1314,7 @@ class SqlFormData(SqlMixin, wcs.formdata.FormData): WHERE id = %%(id)s''' % ( ', '.join([x[0] for x in cls._table_static_fields]), potential_comma, - ', '.join(cls.get_data_fields()), + ', '.join(fields), cls._table_name) cur.execute(sql_statement, {'id': str(id)}) row = cur.fetchone() @@ -1814,7 +1843,7 @@ def get_yearly_totals(period_start=None, period_end=None, criterias=None): return result -SQL_LEVEL = 14 +SQL_LEVEL = 15 def migrate_global_views(conn, cur): cur.execute('''SELECT COUNT(*) FROM information_schema.tables @@ -1848,7 +1877,7 @@ def migrate(): raise RuntimeError() if sql_level < 1: # 1: introduction of tracking_code table do_tracking_code_table() - if sql_level < 14: + if sql_level < 15: # 2: introduction of formdef_id in views # 5: add concerned_roles_array, is_at_endpoint and fts to views # 7: add backoffice_submission to tables @@ -1858,6 +1887,7 @@ def migrate(): # 11: add formdef_name and user_name to views # 13: add backoffice_submission to views # 14: add criticality_level to tables & views + # 15: add geolocation to formdata migrate_views(conn, cur) if sql_level < 12: # 3: introduction of _structured for user fields -- 2.8.1