0001-misc-accept-duplicate-fields-with-the-same-type-4242.patch
tests/conftest.py | ||
---|---|---|
102 | 102 |
fields.ItemField(id='4', label='4rth field', type='item', varname='itemOpen'), |
103 | 103 |
fields.StringField(id='5', label='5th field', type='string', anonymise=False, varname='stringCaseSensitive-é'), |
104 | 104 |
fields.BoolField(id='6', label='6th field duplicate', type='bool', varname='duplicate'), |
105 |
fields.StringField(id='7', label='7th field duplicate', type='string', anonymise=False, varname='duplicate'), |
|
105 |
fields.StringField(id='7', label='7th field bad duplicate', type='string', anonymise=False, varname='duplicate'), |
|
106 |
fields.StringField(id='8', label='8th field duplicate', type='string', anonymise=False, |
|
107 |
required=False, varname='good_duplicate'), |
|
108 |
fields.StringField(id='9', label='9th field good duplicate', type='string', anonymise=False, |
|
109 |
required=False, varname='good_duplicate'), |
|
106 | 110 |
] |
107 | 111 |
formdef.store() |
108 | 112 | |
... | ... | |
122 | 126 |
formdata.data['2_display'] = 'foo' |
123 | 127 |
formdata.data['4'] = 'open_one' |
124 | 128 |
formdata.data['4_display'] = 'open_one' |
129 |
formdata.data['8'] = 'a' |
|
125 | 130 |
elif i%4 == 1: |
126 | 131 |
formdata.data['2'] = 'bar' |
127 | 132 |
formdata.data['2_display'] = 'bar' |
128 | 133 |
formdata.data['4'] = 'open_two' |
129 | 134 |
formdata.data['4_display'] = 'open_two' |
135 |
formdata.data['9'] = 'b' |
|
130 | 136 |
else: |
131 | 137 |
formdata.data['2'] = 'baz' |
132 | 138 |
formdata.data['2_display'] = 'baz' |
133 | 139 |
formdata.data['4'] = "open'three" |
134 | 140 |
formdata.data['4_display'] = "open'three" |
141 |
formdata.data['8'] = 'a' |
|
142 |
formdata.data['9'] = 'b' |
|
135 | 143 | |
136 | 144 |
formdata.data['3'] = bool(i % 2) |
137 | 145 |
if i%3 == 0: |
tests/olap.model | ||
---|---|---|
287 | 287 |
"name": "stringCaseSensitive-é", |
288 | 288 |
"type": "string", |
289 | 289 |
"value": "\"field_stringCaseSensitive-é\"" |
290 |
}, |
|
291 |
{ |
|
292 |
"filter": true, |
|
293 |
"label": "7th field bad duplicate", |
|
294 |
"name": "duplicate", |
|
295 |
"type": "string", |
|
296 |
"value": "\"field_duplicate\"" |
|
297 |
}, |
|
298 |
{ |
|
299 |
"filter": true, |
|
300 |
"label": "9th field good duplicate", |
|
301 |
"name": "good_duplicate", |
|
302 |
"type": "string", |
|
303 |
"value": "\"field_good_duplicate\"" |
|
290 | 304 |
} |
291 | 305 |
], |
292 | 306 |
"fact_table" : "\"formdata_demande\"", |
... | ... | |
413 | 427 |
], |
414 | 428 |
"name" : "formdata_demande", |
415 | 429 |
"warnings": [ |
416 |
"le champ « 6th field duplicate » a un nom de variable dupliqué « duplicate »", |
|
417 |
"le champ « 7th field duplicate » a un nom de variable dupliqué « duplicate »" |
|
430 |
"Le champ « 7th field bad duplicate » a un nom de variable dupliqué « duplicate » mais pas le même type que « 6th field duplicate », il sera ignoré (string != bool)." |
|
418 | 431 |
] |
419 | 432 |
} |
420 | 433 |
], |
tests/test_wcs.py | ||
---|---|---|
20 | 20 | |
21 | 21 |
olap_cmd() |
22 | 22 | |
23 |
assert ( |
|
24 |
'Le champ « 7th field bad duplicate » a un nom de variable dupliqué ' |
|
25 |
'« duplicate » mais pas le même type que « 6th field duplicate », ' |
|
26 |
'il sera ignoré (string != bool).') in caplog.text |
|
27 | ||
23 | 28 |
expected_schema = [ |
24 | 29 |
('agent', 'id'), |
25 | 30 |
('agent', 'label'), |
... | ... | |
68 | 73 |
('formdata_demande', 'field_bool'), |
69 | 74 |
('formdata_demande', 'field_itemOpen'), |
70 | 75 |
('formdata_demande', 'field_stringCaseSensitive-\xe9'), |
76 |
('formdata_demande', 'field_duplicate'), |
|
77 |
('formdata_demande', 'field_good_duplicate'), |
|
71 | 78 |
('formdata_demande', 'function__receiver'), |
72 | 79 |
('formdata_demande_field_item', 'id'), |
73 | 80 |
('formdata_demande_field_item', 'label'), |
... | ... | |
104 | 111 |
expected_json_schema['pg_dsn'] = postgres_db.dsn |
105 | 112 |
assert json_schema == expected_json_schema |
106 | 113 | |
114 |
# verify data in duplicated columns |
|
115 |
with postgres_db.conn() as conn: |
|
116 |
with conn.cursor() as c: |
|
117 |
c.execute('SET search_path = olap') |
|
118 |
c.execute('SELECT field_good_duplicate, count(id) ' |
|
119 |
'FROM formdata_demande ' |
|
120 |
'GROUP BY field_good_duplicate ' |
|
121 |
'ORDER BY field_good_duplicate') |
|
122 |
assert dict(c.fetchall()) == { |
|
123 |
'a': 37, |
|
124 |
'b': 13, |
|
125 |
} |
|
126 | ||
107 | 127 | |
108 | 128 |
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
109 | 129 |
@httmock.urlmatch() |
... | ... | |
126 | 146 |
olap_cmd(no_log_errors=False) |
127 | 147 |
assert 'invalid signature' in caplog.text |
128 | 148 | |
149 | ||
129 | 150 |
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
130 | 151 |
@httmock.urlmatch() |
131 | 152 |
def return_invalid_json(url, request): |
wcs_olap/feeder.py | ||
---|---|---|
992 | 992 |
fields += self.formdef.schema.workflow.fields |
993 | 993 | |
994 | 994 |
# filter duplicates |
995 |
duplicate_varnames = set() |
|
996 | 995 |
self.good_fields = good_fields = OrderedDict() |
997 | 996 |
for field in fields: |
998 | 997 |
if field.type not in ('item', 'bool', 'string'): |
... | ... | |
1000 | 999 |
if field.anonymise is True: |
1001 | 1000 |
continue |
1002 | 1001 |
if not field.varname: |
1003 |
add_warning('le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
|
|
1002 |
add_warning('Le champ « %s » n\' a pas de nom de variable, il a été ignoré' % field.label)
|
|
1004 | 1003 |
continue |
1005 |
if field.varname in good_fields: |
|
1006 |
# duplicate found |
|
1007 |
duplicate_varnames.add(field.varname) |
|
1008 |
add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % { |
|
1009 |
'label': good_fields[field.varname].label, |
|
1010 |
'varname': field.varname |
|
1011 |
}) |
|
1004 |
if field.varname in good_fields and good_fields[field.varname].type != field.type: |
|
1005 |
# duplicate found, but type is not coherent |
|
1006 |
add_warning( |
|
1007 |
'Le champ « %(label)s » a un nom de variable dupliqué « %(varname)s » ' |
|
1008 |
'mais pas le même type que « %(label_good)s », il sera ignoré ' |
|
1009 |
'(%(type)s != %(type_good)s).' % { |
|
1010 |
'label': field.label, |
|
1011 |
'varname': field.varname, |
|
1012 |
'type': field.type, |
|
1013 |
'label_good': good_fields[field.varname].label, |
|
1014 |
'type_good': good_fields[field.varname].type, |
|
1015 |
} |
|
1016 |
) |
|
1012 | 1017 |
del self.good_fields[field.varname] |
1013 |
if field.varname in duplicate_varnames: |
|
1014 |
add_warning('le champ « %(label)s » a un nom de variable dupliqué « %(varname)s »' % field.__dict__) |
|
1015 |
continue |
|
1016 | 1018 |
self.good_fields[field.varname] = field |
1017 | 1019 | |
1018 | 1020 |
for field in good_fields.values(): |
1019 |
- |