Project

General

Profile

0001-misc-accept-duplicate-fields-with-the-same-type-4242.patch

Benjamin Dauvergne, 25 Sep 2020 08:12 AM

Download (8.28 KB)

View differences:

Subject: [PATCH] misc: accept duplicate fields with the same type (#42429)

 tests/conftest.py  | 10 +++++++++-
 tests/olap.model   | 17 +++++++++++++++--
 tests/test_wcs.py  | 21 +++++++++++++++++++++
 wcs_olap/feeder.py | 26 ++++++++++++++------------
 4 files changed, 59 insertions(+), 15 deletions(-)
tests/conftest.py
102 102
    fields.ItemField(id='4', label='4rth field', type='item', varname='itemOpen'),
103 103
    fields.StringField(id='5', label='5th field', type='string', anonymise=False, varname='stringCaseSensitive-é'),
104 104
    fields.BoolField(id='6', label='6th field duplicate', type='bool', varname='duplicate'),
105
    fields.StringField(id='7', label='7th field duplicate', type='string', anonymise=False, varname='duplicate'),
105
    fields.StringField(id='7', label='7th field bad duplicate', type='string', anonymise=False, varname='duplicate'),
106 106
    fields.StringField(id='8', label='8th field integer', type='string', anonymise=False, varname='integer',
107 107
        validation={'type': 'digits'}),
108
    fields.StringField(id='9', label='8th field duplicate', type='string', anonymise=False,
109
                       required=False, varname='good_duplicate'),
110
    fields.StringField(id='10', label='9th field good duplicate', type='string', anonymise=False,
111
                       required=False, varname='good_duplicate'),
108 112
]
109 113
formdef.store()
110 114

  
......
124 128
        formdata.data['2_display'] = 'foo'
125 129
        formdata.data['4'] = 'open_one'
126 130
        formdata.data['4_display'] = 'open_one'
131
        formdata.data['9'] = 'a'
127 132
    elif i%4 == 1:
128 133
        formdata.data['2'] = 'bar'
129 134
        formdata.data['2_display'] = 'bar'
130 135
        formdata.data['4'] = 'open_two'
131 136
        formdata.data['4_display'] = 'open_two'
137
        formdata.data['10'] = 'b'
132 138
    else:
133 139
        formdata.data['2'] = 'baz'
134 140
        formdata.data['2_display'] = 'baz'
135 141
        formdata.data['4'] = "open'three"
136 142
        formdata.data['4_display'] = "open'three"
143
        formdata.data['9'] = 'a'
144
        formdata.data['10'] = 'b'
137 145

  
138 146
    formdata.data['3'] = bool(i % 2)
139 147
    formdata.data['8'] = str(i % 10)
tests/olap.model
287 287
               "name": "stringCaseSensitive-é",
288 288
               "type": "string",
289 289
               "value": "\"field_stringCaseSensitive-é\""
290
            },
291
            {
292
               "filter": true,
293
               "label": "7th field bad duplicate",
294
               "name": "duplicate",
295
               "type": "string",
296
               "value": "\"field_duplicate\""
297
            },
298
            {
299
               "filter": true,
300
               "label": "9th field good duplicate",
301
               "name": "good_duplicate",
302
               "type": "string",
303
               "value": "\"field_good_duplicate\""
290 304
            }
291 305
         ],
292 306
         "fact_table" : "\"formdata_demande\"",
......
419 433
         ],
420 434
         "name" : "formdata_demande",
421 435
         "warnings": [
422
            "le champ « 6th field duplicate » a un nom de variable dupliqué « duplicate »",
423
            "le champ « 7th field duplicate » a un nom de variable dupliqué « duplicate »"
436
               "Le champ « 7th field bad duplicate » a un nom de variable dupliqué « duplicate » mais pas le même type que « 6th field duplicate », il sera ignoré (string != bool)."
424 437
         ]
425 438
      }
426 439
   ],
tests/test_wcs.py
54 54
    for url_with_limit in url_with_limits:
55 55
        assert 'limit=500&' in url_with_limit
56 56

  
57
    assert (
58
        'Le champ « 7th field bad duplicate » a un nom de variable dupliqué '
59
        '« duplicate » mais pas le même type que « 6th field duplicate », '
60
        'il sera ignoré (string != bool).') in caplog.text
61

  
57 62
    expected_schema = [
58 63
        ('agent', 'id'),
59 64
        ('agent', 'label'),
......
102 107
        ('formdata_demande', 'field_bool'),
103 108
        ('formdata_demande', 'field_itemOpen'),
104 109
        ('formdata_demande', 'field_stringCaseSensitive-\xe9'),
110
        ('formdata_demande', 'field_duplicate'),
105 111
        ('formdata_demande', 'field_integer'),
112
        ('formdata_demande', 'field_good_duplicate'),
106 113
        ('formdata_demande', 'function__receiver'),
107 114
        ('formdata_demande_field_item', 'id'),
108 115
        ('formdata_demande_field_item', 'label'),
......
167 174
        expected_json_schema['pg_dsn'] = postgres_db.dsn
168 175
        assert json_schema == expected_json_schema
169 176

  
177
    # verify data in duplicated columns
178
    with postgres_db.conn() as conn:
179
        with conn.cursor() as c:
180
            c.execute('SET search_path = olap')
181
            c.execute('SELECT field_good_duplicate, count(id) '
182
                      'FROM formdata_demande '
183
                      'GROUP BY field_good_duplicate '
184
                      'ORDER BY field_good_duplicate')
185
            assert dict(c.fetchall()) == {
186
                'a': 37,
187
                'b': 13,
188
            }
189

  
170 190

  
171 191
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
172 192
    @httmock.urlmatch()
......
189 209
            olap_cmd(no_log_errors=False)
190 210
    assert 'invalid signature' in caplog.text
191 211

  
212

  
192 213
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
193 214
    @httmock.urlmatch()
194 215
    def return_invalid_json(url, request):
wcs_olap/feeder.py
1023 1023
            fields += self.formdef.schema.workflow.fields
1024 1024

  
1025 1025
        # filter duplicates
1026
        duplicate_varnames = set()
1027 1026
        self.good_fields = good_fields = OrderedDict()
1028 1027
        for field in fields:
1029 1028
            if field.type not in ('item', 'bool', 'string'):
......
1031 1030
            if field.anonymise is True:
1032 1031
                continue
1033 1032
            if not field.varname:
1034
                add_warning('le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
1033
                add_warning('Le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
1035 1034
                continue
1036
            if field.varname in good_fields:
1037
                # duplicate found
1038
                duplicate_varnames.add(field.varname)
1039
                add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % {
1040
                    'label': good_fields[field.varname].label,
1041
                    'varname': field.varname
1042
                })
1035
            if field.varname in good_fields and good_fields[field.varname].type != field.type:
1036
                # duplicate found, but type is not coherent
1037
                add_warning(
1038
                    'Le champ « %(label)s » a un nom de variable dupliqué « %(varname)s » '
1039
                    'mais pas le même type que « %(label_good)s », il sera ignoré '
1040
                    '(%(type)s != %(type_good)s).' % {
1041
                        'label': field.label,
1042
                        'varname': field.varname,
1043
                        'type': field.type,
1044
                        'label_good': good_fields[field.varname].label,
1045
                        'type_good': good_fields[field.varname].type,
1046
                    }
1047
                )
1043 1048
                del self.good_fields[field.varname]
1044
            if field.varname in duplicate_varnames:
1045
                add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % field.__dict__)
1046
                continue
1047 1049
            self.good_fields[field.varname] = field
1048 1050

  
1049 1051
        for field in good_fields.values():
1050
-