Project

General

Profile

0001-misc-accept-duplicate-fields-with-the-same-type-4242.patch

Benjamin Dauvergne, 04 May 2020 09:42 PM

Download (8.03 KB)

View differences:

Subject: [PATCH] misc: accept duplicate fields with the same type (#42429)

wcs-olap expects w.c.s. API to return only the first value for each
duplicate field.
 tests/conftest.py  | 10 +++++++++-
 tests/olap.model   | 17 +++++++++++++++--
 tests/test_wcs.py  | 21 +++++++++++++++++++++
 wcs_olap/feeder.py | 26 ++++++++++++++------------
 4 files changed, 59 insertions(+), 15 deletions(-)
tests/conftest.py
102 102
    fields.ItemField(id='4', label='4rth field', type='item', varname='itemOpen'),
103 103
    fields.StringField(id='5', label='5th field', type='string', anonymise=False, varname='stringCaseSensitive-é'),
104 104
    fields.BoolField(id='6', label='6th field duplicate', type='bool', varname='duplicate'),
105
    fields.StringField(id='7', label='7th field duplicate', type='string', anonymise=False, varname='duplicate'),
105
    fields.StringField(id='7', label='7th field bad duplicate', type='string', anonymise=False, varname='duplicate'),
106
    fields.StringField(id='8', label='8th field duplicate', type='string', anonymise=False,
107
                       required=False, varname='good_duplicate'),
108
    fields.StringField(id='9', label='9th field good duplicate', type='string', anonymise=False,
109
                       required=False, varname='good_duplicate'),
106 110
]
107 111
formdef.store()
108 112

  
......
122 126
        formdata.data['2_display'] = 'foo'
123 127
        formdata.data['4'] = 'open_one'
124 128
        formdata.data['4_display'] = 'open_one'
129
        formdata.data['8'] = 'a'
125 130
    elif i%4 == 1:
126 131
        formdata.data['2'] = 'bar'
127 132
        formdata.data['2_display'] = 'bar'
128 133
        formdata.data['4'] = 'open_two'
129 134
        formdata.data['4_display'] = 'open_two'
135
        formdata.data['9'] = 'b'
130 136
    else:
131 137
        formdata.data['2'] = 'baz'
132 138
        formdata.data['2_display'] = 'baz'
133 139
        formdata.data['4'] = "open'three"
134 140
        formdata.data['4_display'] = "open'three"
141
        formdata.data['8'] = 'a'
142
        formdata.data['9'] = 'b'
135 143

  
136 144
    formdata.data['3'] = bool(i % 2)
137 145
    if i%3 == 0:
tests/olap.model
287 287
               "name": "stringCaseSensitive-é",
288 288
               "type": "string",
289 289
               "value": "\"field_stringCaseSensitive-é\""
290
            },
291
            {
292
               "filter": true,
293
               "label": "7th field bad duplicate",
294
               "name": "duplicate",
295
               "type": "string",
296
               "value": "\"field_duplicate\""
297
            },
298
            {
299
               "filter": true,
300
               "label": "9th field good duplicate",
301
               "name": "good_duplicate",
302
               "type": "string",
303
               "value": "\"field_good_duplicate\""
290 304
            }
291 305
         ],
292 306
         "fact_table" : "\"formdata_demande\"",
......
413 427
         ],
414 428
         "name" : "formdata_demande",
415 429
         "warnings": [
416
            "le champ « 6th field duplicate » a un nom de variable dupliqué « duplicate »",
417
            "le champ « 7th field duplicate » a un nom de variable dupliqué « duplicate »"
430
               "Le champ « 7th field bad duplicate » a un nom de variable dupliqué « duplicate » mais pas le même type que « 6th field duplicate », il sera ignoré (string != bool)."
418 431
         ]
419 432
      }
420 433
   ],
tests/test_wcs.py
20 20

  
21 21
    olap_cmd()
22 22

  
23
    assert (
24
        'Le champ « 7th field bad duplicate » a un nom de variable dupliqué '
25
        '« duplicate » mais pas le même type que « 6th field duplicate », '
26
        'il sera ignoré (string != bool).') in caplog.text
27

  
23 28
    expected_schema = [
24 29
        ('agent', 'id'),
25 30
        ('agent', 'label'),
......
68 73
        ('formdata_demande', 'field_bool'),
69 74
        ('formdata_demande', 'field_itemOpen'),
70 75
        ('formdata_demande', 'field_stringCaseSensitive-\xe9'),
76
        ('formdata_demande', 'field_duplicate'),
77
        ('formdata_demande', 'field_good_duplicate'),
71 78
        ('formdata_demande', 'function__receiver'),
72 79
        ('formdata_demande_field_item', 'id'),
73 80
        ('formdata_demande_field_item', 'label'),
......
104 111
        expected_json_schema['pg_dsn'] = postgres_db.dsn
105 112
        assert json_schema == expected_json_schema
106 113

  
114
    # verify data in duplicated columns
115
    with postgres_db.conn() as conn:
116
        with conn.cursor() as c:
117
            c.execute('SET search_path = olap')
118
            c.execute('SELECT field_good_duplicate, count(id) '
119
                      'FROM formdata_demande '
120
                      'GROUP BY field_good_duplicate '
121
                      'ORDER BY field_good_duplicate')
122
            assert dict(c.fetchall()) == {
123
                'a': 37,
124
                'b': 13,
125
            }
126

  
107 127

  
108 128
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
109 129
    @httmock.urlmatch()
......
126 146
            olap_cmd(no_log_errors=False)
127 147
    assert 'invalid signature' in caplog.text
128 148

  
149

  
129 150
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
130 151
    @httmock.urlmatch()
131 152
    def return_invalid_json(url, request):
wcs_olap/feeder.py
992 992
            fields += self.formdef.schema.workflow.fields
993 993

  
994 994
        # filter duplicates
995
        duplicate_varnames = set()
996 995
        self.good_fields = good_fields = OrderedDict()
997 996
        for field in fields:
998 997
            if field.type not in ('item', 'bool', 'string'):
......
1000 999
            if field.anonymise is True:
1001 1000
                continue
1002 1001
            if not field.varname:
1003
                add_warning('le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
1002
                add_warning('Le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
1004 1003
                continue
1005
            if field.varname in good_fields:
1006
                # duplicate found
1007
                duplicate_varnames.add(field.varname)
1008
                add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % {
1009
                    'label': good_fields[field.varname].label,
1010
                    'varname': field.varname
1011
                })
1004
            if field.varname in good_fields and good_fields[field.varname].type != field.type:
1005
                # duplicate found, but type is not coherent
1006
                add_warning(
1007
                    'Le champ « %(label)s » a un nom de variable dupliqué « %(varname)s » '
1008
                    'mais pas le même type que « %(label_good)s », il sera ignoré '
1009
                    '(%(type)s != %(type_good)s).' % {
1010
                        'label': field.label,
1011
                        'varname': field.varname,
1012
                        'type': field.type,
1013
                        'label_good': good_fields[field.varname].label,
1014
                        'type_good': good_fields[field.varname].type,
1015
                    }
1016
                )
1012 1017
                del self.good_fields[field.varname]
1013
            if field.varname in duplicate_varnames:
1014
                add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % field.__dict__)
1015
                continue
1016 1018
            self.good_fields[field.varname] = field
1017 1019

  
1018 1020
        for field in good_fields.values():
1019
-