0003-ignore-fields-with-duplicated-varname-26207.patch

Benjamin Dauvergne, 30 septembre 2019 17:53

Voir les différences: en ligne côte à côte

Subject: [PATCH 3/3] ignore fields with duplicated varname (#26207)

 tests/conftest.py  |  5 ++-
 tests/olap.model   | 10 +++--
 tests/test_wcs.py  |  2 +-
 wcs_olap/feeder.py | 95 +++++++++++++++++++++++++++++-----------------
 4 files changed, 72 insertions(+), 40 deletions(-)

     import utils
     Wcs = namedtuple('Wcs', ['url', 'appdir', 'pid'])
-...
                          items=['foo', 'bar', 'baz'], varname='item'),
         fields.BoolField(id='3', label='3rd field', type='bool', varname='bool'),
         fields.ItemField(id='4', label='4rth field', type='item', varname='itemOpen'),
         fields.StringField(id='5', label='5th field', type='string', anonymise=False, varname='stringCaseSensitiveé'),
         fields.StringField(id='5', label='5th field', type='string', anonymise=False, varname='stringCaseSensitive-é'),
         fields.BoolField(id='6', label='6th field duplicate', type='bool', varname='duplicate'),
         fields.StringField(id='7', label='7th field duplicate', type='string', anonymise=False, varname='duplicate'),
+    ]
     formdef.store()

+                {
                    "filter": true,
                    "label": "5th field",
                    "name": "stringCaseSensitiveé",
                    "name": "stringCaseSensitive-é",
                    "type": "string",
                    "value": "\"field_stringCaseSensitiveé\""
                    "value": "\"field_stringCaseSensitive-é\""
+                }
              ],
              "fact_table" : "\"formdata_demande\"",
-...
                    "type" : "point"
+                }
              ],
              "name" : "formdata_demande"
              "name" : "formdata_demande",
              "warnings": [
                 "duplicate column « 6th field duplicate » with varname « duplicate »",
                 "duplicate column « 7th field duplicate » with varname « duplicate »"
+             ]
+          }
        ],
        "label" : "olap",

             ('formdata_demande', 'field_item'),
             ('formdata_demande', 'field_bool'),
             ('formdata_demande', 'field_itemOpen'),
             ('formdata_demande', 'field_stringCaseSensitive\xe9'),
             ('formdata_demande', 'field_stringCaseSensitive-\xe9'),
             ('formdata_demande', 'function__receiver'),
             ('formdata_demande_field_item', 'id'),
             ('formdata_demande_field_item', 'label'),

     from __future__ import unicode_literals
     from collections import OrderedDict, Counter
     import datetime
     import six
     import copy
-...
                           self.formdef.schema.name])
             self.formdef_sql_id = self.cur.fetchone()[0]
             columns = [['status_id', 'smallint REFERENCES {status_table} (id)']]
             comments = {}
             # compute list of fields
             fields = self.formdef.schema.fields
             if self.formdef.schema.workflow:
                 fields += self.formdef.schema.workflow.fields
             columns = OrderedDict()
             columns['status_id'] = {'sql_col_name': 'status_id', 'sql_col_def': 'smallint REFERENCES {status_table} (id)'}
             # add item fields
             already_seen_varnames = set()
             for field in fields:
                 if field.anonymise is True:
                     continue
                 if not field.varname or '-' in field.varname:
                     continue
                 if field.varname in already_seen_varnames:
                     continue
                 already_seen_varnames.add(field.varname)
             for field in self.good_fields.values():
                 if field.type == 'item':
                     comment = ('valeurs du champ « %s » du formulaire %s'
                                % (field.label, self.formdef.schema.name))
-...
                     field_def = 'varchar'
                 else:
                     continue
                 self.fields.append(field)
                 at = 'field_%s' % field.varname
                 columns.append([at, field_def])
                 comments[at] = field.label
                 columns[field.varname] = {
                     'field': field,
                     'sql_col_name': 'field_%s' % field.varname,
                     'sql_col_def': field_def,
                     'sql_comment': field.label,
+                }
             # keep loaded fields around
             for key in columns:
                 if columns[key].get('field') is not None:
                     self.fields.append(columns[key].get('field'))
             # add geolocation fields
             for geolocation, label in self.formdef.schema.geolocations:
                 at = 'geolocation_%s' % geolocation
                 columns.append([at, 'point'])
                 comments[at] = label
                 columns[at] = {
                     'sql_col_name': at,
                     'sql_col_def': 'point',
                     'comment': 'géoloc « %s »' % label,
+                }
             # add function fields
             for function, name in self.formdef.schema.workflow.functions.iteritems():
                 at = 'function_%s' % slugify(function)
                 columns.append([at, 'smallint REFERENCES {role_table} (id)'])
                 comments[at] = 'fonction « %s »' % name
                 columns[at] = {
                     'sql_col_name': at,
                     'sql_col_def': 'smallint REFERENCES {role_table} (id)',
                     'comment': 'fonction « %s »' % name,
+                }
             self.columns = ([name for name, _type in self.olap_feeder.columns] + [
                 name for name, _type in columns])
             self.columns = [name for name, _type in self.olap_feeder.columns]
             for key in columns:
                 self.columns.append(columns[key]['sql_col_name'])
             self.columns.remove('geolocation_base')
             self.create_table(self.table_name, columns, inherits='{generic_formdata_table}',
             self.create_table(self.table_name,
                               [(columns[key]['sql_col_name'], columns[key]['sql_col_def']) for key in columns],
                               inherits='{generic_formdata_table}',
                               comment='formulaire %s' % self.formdef.schema.name)
             for at, comment in comments.iteritems():
                 self.ex('COMMENT ON COLUMN {formdata_table}.%s IS %%s' % quote(at), vars=(comment,))
             for key in columns:
                 column = columns[key]
                 if column.get('sql_comment'):
                     self.ex('COMMENT ON COLUMN {formdata_table}.%s IS %%s' % quote(column['sql_col_name']),
                             vars=(column['sql_comment'],))
             # Creat index for JSON fields
             if self.has_jsonb:
                 for field in fields:
                     if field.varname and '-' not in field.varname:
                         self.create_formdata_json_index(self.table_name, field.varname)
                 for varname in self.good_fields:
                     self.create_formdata_json_index(self.table_name, varname)
             # PostgreSQL does not propagate foreign key constraints to child tables
             # so we must recreate them manually
-...
             # create cube
             cube = self.cube = copy.deepcopy(self.base_cube)
             def add_warning(message):
                 self.logger.warning('%s', message)
                 cube.setdefault('warnings', []).append(message)
             # remove json field from formdef cubes
             cube.pop('json_field', None)
             cube.update({
-...
             fields = self.formdef.schema.fields
             if self.formdef.schema.workflow:
                 fields += self.formdef.schema.workflow.fields
             # filter duplicates
             duplicate_varnames = set()
             self.good_fields = good_fields = OrderedDict()
             for field in fields:
                 if field.type not in ('item', 'bool', 'string'):
                     continue
                 if field.anonymise is True:
                     continue
                 if not field.varname:
                     add_warning('field « %s » has no varname, ignored' % field.label)
                     continue
                 if '-' in field.varname:
                 if field.varname in good_fields:
                     # duplicate found
                     duplicate_varnames.add(field.varname)
                     add_warning('duplicate column « %s » with varname « %s »' % (
                         good_fields[field.varname].label, field.varname))
                     del self.good_fields[field.varname]
                 if field.varname in duplicate_varnames:
                     add_warning('duplicate column « %s » with varname « %s »' % (field.label, field.varname))
                     continue
                 self.good_fields[field.varname] = field
             for field in good_fields.values():
                 join = None
                 if field.type == 'item':
+    -

Projet

Général

Profil

Produits Entr'ouvert » OLAP / Business Intelligence pour Publik

0003-ignore-fields-with-duplicated-varname-26207.patch