Projet

Général

Profil

0002-formdata-new-digests-field-45633.patch

Lauréline Guérin, 25 juin 2021 10:48

Télécharger (25,6 ko)

Voir les différences:

Subject: [PATCH 2/4] formdata: new digests field (#45633)

 tests/admin_pages/test_card.py     |  2 +-
 tests/admin_pages/test_form.py     |  2 +-
 tests/api/test_carddef.py          | 10 ++--
 tests/api/test_user.py             |  2 +-
 tests/backoffice_pages/test_all.py |  4 +-
 tests/test_formdata.py             | 12 ++---
 wcs/backoffice/data_management.py  |  2 +-
 wcs/backoffice/management.py       | 28 +++++-----
 wcs/carddata.py                    |  4 +-
 wcs/carddef.py                     |  6 +--
 wcs/formdata.py                    | 39 +++++++++++---
 wcs/qommon/storage.py              | 23 +++++++++
 wcs/sql.py                         | 83 +++++++++++++++++++++---------
 wcs/variables.py                   |  2 +-
 14 files changed, 150 insertions(+), 69 deletions(-)
tests/admin_pages/test_card.py
268 268

  
269 269
    # afterjobs are actually run synchronously during tests; we don't have
270 270
    # to wait to check the digest has been updated:
271
    assert carddef.data_class().get(carddata.id).digest == 'XbarY'
271
    assert carddef.data_class().get(carddata.id).digests['default'] == 'XbarY'
272 272

  
273 273
    carddef.digest_templates = {'default': '{{ form_var_foo }}'}
274 274
    carddef.store()
tests/admin_pages/test_form.py
908 908
    assert 'Existing forms will be updated in the background.' in resp.text
909 909
    # afterjobs are actually run synchronously during tests; we don't have
910 910
    # to wait to check the digest has been updated:
911
    assert formdef.data_class().get(formdata.id).digest == 'XhelloY'
911
    assert formdef.data_class().get(formdata.id).digests['default'] == 'XhelloY'
912 912

  
913 913
    resp = app.get('/backoffice/forms/1/options/templates')
914 914
    resp.form['lateral_template'] = 'X{{form_var_test}}Y'
tests/api/test_carddef.py
148 148
    assert len(resp.json['data']) == 1
149 149
    assert resp.json['data'][0]['display_id'] == formdata.get_display_id()
150 150
    assert resp.json['data'][0]['display_name'] == formdata.get_display_name()
151
    assert resp.json['data'][0]['digest'] == formdata.digest
152
    assert resp.json['data'][0]['text'] == formdata.digest
151
    assert resp.json['data'][0]['digest'] == formdata.digests['default']
152
    assert resp.json['data'][0]['text'] == formdata.digests['default']
153 153
    resp = get_app(pub).get(
154 154
        sign_uri('/api/cards/test/list?NameID=%s&full=on' % local_user.name_identifiers[0])
155 155
    )
156 156
    assert resp.json['data'][0]['fields']['foo'] == 'blah'
157
    assert resp.json['data'][0]['digest'] == formdata.digest
158
    assert resp.json['data'][0]['text'] == formdata.digest
157
    assert resp.json['data'][0]['digest'] == formdata.digests['default']
158
    assert resp.json['data'][0]['text'] == formdata.digests['default']
159 159

  
160 160
    # get single carddata (as signed request without any user specified, so
161 161
    # no check for permissions)
162 162
    resp = get_app(pub).get(sign_uri('/api/cards/test/%s/' % formdata.id))
163
    assert resp.json['text'] == formdata.digest
163
    assert resp.json['text'] == formdata.digests['default']
164 164

  
165 165
    # get schema
166 166
    resp = get_app(pub).get(sign_uri('/api/cards/test/@schema'), status=200)
tests/api/test_user.py
291 291
    # check digest is part of contents
292 292
    formdef.digest_templates = {'default': 'XYZ'}
293 293
    formdef.data_class().get(formdata.id).store()
294
    assert formdef.data_class().get(formdata.id).digest == 'XYZ'
294
    assert formdef.data_class().get(formdata.id).digests['default'] == 'XYZ'
295 295
    resp = get_app(pub).get(sign_uri('/api/user/forms', user=local_user))
296 296
    assert resp.json['data'][0]['form_digest'] == 'XYZ'
297 297

  
tests/backoffice_pages/test_all.py
3058 3058
    )
3059 3059
    formdata.formdef.digest_templates = {'default': 'digest of number <{{form_number}}>'}
3060 3060
    formdata.store()
3061
    assert formdata.get(formdata.id).digest
3061
    assert formdata.get(formdata.id).digests['default']
3062 3062
    resp = app.get('/backoffice/management/listing')
3063 3063
    assert formdata.get_url(backoffice=True) in resp.text
3064 3064
    assert 'digest of number &lt;%s&gt;' % formdata.id_display in resp.text
......
3530 3530
    user_pending_form_url = re.findall('data-async-url="(.*/user-pending-forms)"', resp.text)[0]
3531 3531
    partial_resp = app.get(user_pending_form_url)
3532 3532
    assert number31.get_url(backoffice=True) not in partial_resp.text
3533
    assert number31.digest in partial_resp.text
3533
    assert number31.digests['default'] in partial_resp.text
3534 3534
    assert '<span class="formname">%s</span>' % number31.formdef.name in partial_resp.text
3535 3535

  
3536 3536
    # another item with status = new
tests/test_formdata.py
2310 2310
    formdata = formdef.data_class()()
2311 2311
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
2312 2312
    formdata.store()
2313
    assert formdef.data_class().get(formdata.id).digest == 'plop 2015-05-12 plop'
2313
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015-05-12 plop'
2314 2314

  
2315 2315
    pub.cfg['language'] = {'language': 'fr'}
2316 2316
    pub.write_cfg()
2317 2317
    formdata = formdef.data_class()()
2318 2318
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
2319 2319
    formdata.store()
2320
    assert formdef.data_class().get(formdata.id).digest == 'plop 12/05/2015 plop'
2320
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 12/05/2015 plop'
2321 2321

  
2322 2322
    formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
2323 2323
    formdef.store()
2324 2324
    formdata = formdef.data_class()()
2325 2325
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
2326 2326
    formdata.store()
2327
    assert formdef.data_class().get(formdata.id).digest == 'plop 2015 plop'
2327
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'
2328 2328

  
2329 2329
    formdef.digest_templates = {'default': 'plop {{ form_var_date_raw|date:"Y" }} plop'}
2330 2330
    formdef.store()
2331 2331
    formdata = formdef.data_class()()
2332 2332
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
2333 2333
    formdata.store()
2334
    assert formdef.data_class().get(formdata.id).digest == 'plop 2015 plop'
2334
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop 2015 plop'
2335 2335

  
2336 2336
    formdef.digest_templates = {'default': 'plop {{ form_var_date|date:"Y" }} plop'}
2337 2337
    formdef.store()
2338 2338
    formdata = formdef.data_class()()
2339 2339
    formdata.data = {'0': None}
2340 2340
    formdata.store()
2341
    assert formdef.data_class().get(formdata.id).digest == 'plop  plop'
2341
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop  plop'
2342 2342

  
2343 2343
    # check there's no crash when an invaliad variable is given
2344 2344
    formdef.digest_templates = {'default': 'plop {{ blah|date:"Y" }} plop'}
......
2346 2346
    formdata = formdef.data_class()()
2347 2347
    formdata.data = {'0': time.strptime('2015-05-12', '%Y-%m-%d')}
2348 2348
    formdata.store()
2349
    assert formdef.data_class().get(formdata.id).digest == 'plop  plop'
2349
    assert formdef.data_class().get(formdata.id).digests['default'] == 'plop  plop'
2350 2350

  
2351 2351

  
2352 2352
def test_lazy_formdata_decimal_filter(pub):
wcs/backoffice/data_management.py
308 308
            popup_response_data = json.dumps(
309 309
                {
310 310
                    'value': str(filled.id),
311
                    'obj': str(filled.digest),
311
                    'obj': str(filled.default_digest),
312 312
                }
313 313
            )
314 314
            return template.QommonTemplateResponse(
wcs/backoffice/management.py
985 985
            if include_submission_channel:
986 986
                r += htmltext('<td>%s</td>') % formdata.get_submission_channel_label()
987 987
            r += htmltext('<td>%s') % formdata.formdef.name
988
            if formdata.digest:
989
                r += htmltext(' <small>%s</small>') % formdata.digest
988
            if formdata.default_digest:
989
                r += htmltext(' <small>%s</small>') % formdata.default_digest
990 990
            r += htmltext('</td>')
991 991
            r += htmltext('<td><a href="%s">%s</a></td>') % (
992 992
                formdata.get_url(backoffice=True),
......
2314 2314
        if get_publisher().is_using_postgresql():
2315 2315
            self.formdef.data_class().load_all_evolutions(items)
2316 2316
        if get_request().form.get('full') == 'on':
2317
            output = [
2318
                filled.get_json_export_dict(include_files=False, anonymise=anonymise, user=user)
2319
                for filled in items
2320
            ]
2317
            output = []
2318
            for filled in items:
2319
                data = filled.get_json_export_dict(include_files=False, anonymise=anonymise, user=user)
2320
                data.pop('digests')
2321
                data['digest'] = filled.default_digest
2322
                output.append(data)
2321 2323
        else:
2322 2324
            output = [
2323 2325
                {
2324 2326
                    'id': filled.id,
2325 2327
                    'display_id': filled.get_display_id(),
2326 2328
                    'display_name': filled.get_display_name(),
2327
                    'digest': filled.digest,
2329
                    'digest': filled.default_digest,
2328 2330
                    'text': filled.get_display_label(),
2329 2331
                    'url': filled.get_url(),
2330 2332
                    'receipt_time': datetime.datetime(*filled.receipt_time[:6]),
......
2435 2437
                        formdata.id,
2436 2438
                    )
2437 2439
                    summary = force_text(formdata.get_display_name(), charset)
2438
                    if formdata.digest:
2439
                        summary += ' - %s' % force_text(formdata.digest, charset)
2440
                    if formdata.default_digest:
2441
                        summary += ' - %s' % force_text(formdata.default_digest, charset)
2440 2442
                    vevent.add('summary').value = summary
2441 2443
                    vevent.add('dtstart').value = dtstart
2442 2444
                    if dtend:
......
2447 2449
                    form_name = force_text(formdef.name, charset)
2448 2450
                    status_name = force_text(formdata.get_status_label(), charset)
2449 2451
                    description = '%s | %s | %s\n' % (form_name, formdata.get_display_id(), status_name)
2450
                    if formdata.digest:
2451
                        description += '%s\n' % force_text(formdata.digest, charset)
2452
                    if formdata.default_digest:
2453
                        description += '%s\n' % force_text(formdata.default_digest, charset)
2452 2454
                    description += backoffice_url
2453 2455
                    # TODO: improve performance by loading all users in one
2454 2456
                    # single query before the loop
......
3144 3146
                        submit_date,
3145 3147
                        status_label,
3146 3148
                    )
3147
                    if formdata.digest:
3148
                        r += htmltext('<small>%s</small>') % formdata.digest
3149
                    if formdata.default_digest:
3150
                        r += htmltext('<small>%s</small>') % formdata.default_digest
3149 3151
                    r += htmltext('</li>')
3150 3152
                r += htmltext('</ul>')
3151 3153
            r += htmltext('</div>')
wcs/carddata.py
35 35
    def get_data_source_structured_item(self):
36 36
        item = {
37 37
            'id': self.id,
38
            'text': self.digest,
38
            'text': self.default_digest,
39 39
        }
40 40
        for field in self.formdef.get_all_fields():
41 41
            if not field.varname:
......
46 46
        return item
47 47

  
48 48
    def get_display_label(self):
49
        return self.digest or self.get_display_name()
49
        return self.default_digest or self.get_display_name()
50 50

  
51 51
    def get_author_qualification(self):
52 52
        return None
wcs/carddef.py
24 24
from wcs.formdef import FormDef, get_formdefs_of_all_kinds
25 25

  
26 26
from .qommon import _, force_text, misc
27
from .qommon.storage import Equal, ILike, NotEqual
27
from .qommon.storage import ElementEqual, ElementILike, Equal, NotEqual
28 28
from .qommon.template import Template
29 29

  
30 30
if not hasattr(types, 'ClassType'):
......
215 215
                criteria.value = WorkflowStatusItem.compute(criteria.value)
216 216

  
217 217
        if query:
218
            criterias.append(ILike('digest', query))
218
            criterias.append(ElementILike('digests', 'default', query))
219 219
        if get_by_id:
220 220
            try:
221 221
                if int(get_by_id) >= 2 ** 31:
......
228 228
            else:
229 229
                criterias.append(Equal('id', get_by_id))
230 230
        if get_by_text:
231
            criterias.append(Equal('digest', get_by_text))
231
            criterias.append(ElementEqual('digests', 'default', get_by_text))
232 232

  
233 233
        items = [
234 234
            x.get_data_source_structured_item()
wcs/formdata.py
245 245
    submission_context = None
246 246
    submission_channel = None
247 247
    criticality_level = 0
248
    digest = None
248
    digests = None
249 249

  
250 250
    prefilling_data = None
251 251
    workflow_data = None
......
290 290
            self.submission_agent_id = str(self.submission_context.get('agent_id'))
291 291
            changed = True
292 292

  
293
        if 'digest' in self.__dict__:
294
            # migration from a simple digest to digests
295
            if not self.digests:
296
                self.digests = {}
297
            self.digests['default'] = self.__dict__['digest']
298
            del self.__dict__['digest']
299
            changed = True
300

  
293 301
        if changed:
294 302
            self.store()
295 303

  
......
423 431
        self.evolution = [evo]
424 432

  
425 433
    def set_auto_fields(self, *args, **kwargs):
426
        fields = {'digest': self.formdef.default_digest_template}
434
        fields = {}
435
        for key, value in (self.formdef.digest_templates or {}).items():
436
            fields['template:%s' % key] = value
427 437
        if not self.id_display:
428 438
            # only set id_display once as it may have been set automatically
429 439
            # by interpreting a webservice response.
......
450 460
        if any(fields.values()):
451 461
            context = self.get_substitution_variables()
452 462
            context['formdef_id'] = self.formdef.id
463
            digests = self.digests or {}
453 464
            for attribute, template in fields.items():
454 465
                if template is None:
455 466
                    new_value = None
456 467
                else:
457 468
                    new_value = Template(template, autoescape=False).render(context)
458
                if new_value != getattr(self, attribute, None):
459
                    setattr(self, attribute, new_value)
469
                if attribute.startswith('template:'):
470
                    key = attribute[9:]
471
                    if new_value != (self.digests or {}).get(key):
472
                        digests[key] = new_value
473
                else:
474
                    if new_value != getattr(self, attribute, None):
475
                        setattr(self, attribute, new_value)
476
                        changed = True
477
                if digests:
478
                    self.digests = digests
460 479
                    changed = True
461 480
        return changed
462 481

  
......
768 787
                    'form_url_backoffice': self.get_url(backoffice=True),
769 788
                    'form_uri': '%s/%s/' % (self.formdef.url_name, self.id),
770 789
                    'form_criticality_level': self.criticality_level,
771
                    'form_digest': self.digest,
790
                    'form_digest': self.default_digest,
772 791
                    'form_display_name': self.get_display_name(),
773 792
                }
774 793
            )
......
1093 1112
    def get_display_name(self):
1094 1113
        return _('%(name)s #%(id)s') % {'name': self.formdef.name, 'id': self.get_display_id()}
1095 1114

  
1115
    @property
1116
    def default_digest(self):
1117
        return (self.digests or {}).get('default')
1118

  
1096 1119
    def get_display_label(self):
1097
        if self.digest:
1098
            return '%s (%s)' % (self.get_display_name(), self.digest)
1120
        if self.default_digest:
1121
            return '%s (%s)' % (self.get_display_name(), self.default_digest)
1099 1122
        return self.get_display_name()
1100 1123

  
1101 1124
    def get_auto_geoloc(self):
......
1149 1172
    def get_json_export_dict(self, include_files=True, anonymise=False, user=None):
1150 1173
        data = {}
1151 1174
        data['id'] = str(self.id)
1152
        data['digest'] = self.digest
1175
        data['digests'] = self.digests
1153 1176
        data['display_id'] = self.get_display_id()
1154 1177
        data['display_name'] = self.get_display_name()
1155 1178
        data['text'] = self.get_display_label()
wcs/qommon/storage.py
272 272
        return lambda x: getattr(x, self.attribute, None) is None
273 273

  
274 274

  
275
class ElementEqual(Criteria):
276
    def __init__(self, attribute, key, value):
277
        self.attribute = attribute
278
        self.key = key
279
        self.value = value
280

  
281
    def build_lambda(self):
282
        return lambda x: (getattr(x, self.attribute, None) or {}).get(self.key) == self.value
283

  
284

  
285
class ElementILike(Criteria):
286
    def __init__(self, attribute, key, value):
287
        self.attribute = attribute
288
        self.key = key
289
        self.value = value
290

  
291
    def build_lambda(self):
292
        return (
293
            lambda x: self.value.lower()
294
            in ((getattr(x, self.attribute, None) or {}).get(self.key) or '').lower()
295
        )
296

  
297

  
275 298
def parse_clause(clause):
276 299
    # creates a callable out of a clause
277 300
    #  (attribute, operator, value)
wcs/sql.py
279 279
        return 'fts @@ plainto_tsquery(%%(c%s)s)' % id(self.value)
280 280

  
281 281

  
282
class ElementEqual(Criteria):
283
    def __init__(self, attribute, key, value):
284
        super().__init__(attribute, value)
285
        self.key = key
286

  
287
    def as_sql(self):
288
        return "%s->>'%s' = %%(c%s)s" % (self.attribute, self.key, id(self.value))
289

  
290

  
291
class ElementILike(ElementEqual):
292
    def __init__(self, *args, **kwargs):
293
        super().__init__(*args, **kwargs)
294
        self.value = '%' + self.value + '%'
295

  
296
    def as_sql(self):
297
        return "%s->>'%s' ILIKE %%(c%s)s" % (self.attribute, self.key, id(self.value))
298

  
299

  
282 300
def get_name_as_sql_identifier(name):
283 301
    name = qommon.misc.simplify(name)
284 302
    for char in '<>|{}!?^*+/=\'':  # forbidden chars
......
563 581
            'submission_channel',
564 582
            'criticality_level',
565 583
            'last_update_time',
566
            'digest',
584
            'digests',
567 585
            'user_label',
568 586
            'prefilling_data',
569 587
        ]
......
612 630
    if 'last_update_time' not in existing_fields:
613 631
        cur.execute('''ALTER TABLE %s ADD COLUMN last_update_time timestamp''' % table_name)
614 632

  
615
    if 'digest' not in existing_fields:
616
        cur.execute('''ALTER TABLE %s ADD COLUMN digest varchar''' % table_name)
633
    if 'digests' not in existing_fields:
634
        cur.execute('''ALTER TABLE %s ADD COLUMN digests jsonb''' % table_name)
617 635

  
618 636
    if 'user_label' not in existing_fields:
619 637
        cur.execute('''ALTER TABLE %s ADD COLUMN user_label varchar''' % table_name)
......
1221 1239
        'submission_channel',
1222 1240
        'backoffice_submission',
1223 1241
        'last_update_time',
1224
        'digest',
1242
        'digests',
1225 1243
        'user_label',
1226 1244
    ):
1227 1245
        view_fields.append((field, field))
......
1862 1880
        ('submission_channel', 'varchar'),
1863 1881
        ('criticality_level', 'int'),
1864 1882
        ('last_update_time', 'timestamp'),
1865
        ('digest', 'varchar'),
1883
        ('digests', 'jsonb'),
1866 1884
        ('user_label', 'varchar'),
1867 1885
    ]
1868 1886

  
......
1953 1971
        conn.commit()
1954 1972
        cur.close()
1955 1973

  
1974
    @guard_postgres
1975
    def _set_auto_fields(self, cur):
1976
        if self.set_auto_fields():
1977
            sql_statement = (
1978
                '''UPDATE %s
1979
                                  SET id_display = %%(id_display)s,
1980
                                      digests = %%(digests)s,
1981
                                      user_label = %%(user_label)s
1982
                                WHERE id = %%(id)s'''
1983
                % self._table_name
1984
            )
1985
            cur.execute(
1986
                sql_statement,
1987
                {
1988
                    'id': self.id,
1989
                    'id_display': self.id_display,
1990
                    'digests': self.digests,
1991
                    'user_label': self.user_label,
1992
                },
1993
            )
1994

  
1956 1995
    @guard_postgres
1957 1996
    @invalidate_substitution_cache
1958 1997
    def store(self):
......
2029 2068
                cur.execute(sql_statement, sql_dict)
2030 2069
                self.id = cur.fetchone()[0]
2031 2070

  
2032
        if self.set_auto_fields():
2033
            sql_statement = (
2034
                '''UPDATE %s
2035
                                  SET id_display = %%(id_display)s,
2036
                                      digest = %%(digest)s,
2037
                                      user_label = %%(user_label)s
2038
                                WHERE id = %%(id)s'''
2039
                % self._table_name
2040
            )
2041
            cur.execute(
2042
                sql_statement,
2043
                {
2044
                    'id': self.id,
2045
                    'id_display': self.id_display,
2046
                    'digest': self.digest,
2047
                    'user_label': self.user_label,
2048
                },
2049
            )
2071
        self._set_auto_fields(cur)
2050 2072

  
2051 2073
        if self._evolution:
2052 2074
            for evo in self._evolution:
......
3393 3415
# latest migration, number + description (description is not used
3394 3416
# programmaticaly but will make sure git conflicts if two migrations are
3395 3417
# separately added with the same number)
3396
SQL_LEVEL = (51, 'add index on formdata blockdef fields')
3418
SQL_LEVEL = (52, 'store digests on formdata and carddata')
3397 3419

  
3398 3420

  
3399 3421
def migrate_global_views(conn, cur):
......
3476 3498
        raise RuntimeError()
3477 3499
    if sql_level < 1:  # 1: introduction of tracking_code table
3478 3500
        do_tracking_code_table()
3479
    if sql_level < 43:
3501
    if sql_level < 52:
3480 3502
        # 2: introduction of formdef_id in views
3481 3503
        # 5: add concerned_roles_array, is_at_endpoint and fts to views
3482 3504
        # 7: add backoffice_submission to tables
......
3496 3518
        # 33: add anonymised field to global view
3497 3519
        # 38: extract submission_agent_id to its own column
3498 3520
        # 43: add prefilling_data to formdata
3521
        # 52: store digests on formdata and carddata
3499 3522
        migrate_views(conn, cur)
3500 3523
    if sql_level < 40:
3501 3524
        # 3: introduction of _structured for user fields
......
3558 3581
                    for evo in formdata.evolution:
3559 3582
                        evo.who = None
3560 3583
                    formdata.store()
3584
    if sql_level < 52:
3585
        # 52: store digests on formdata and carddata
3586
        from wcs.carddef import CardDef
3587
        from wcs.formdef import FormDef
3588

  
3589
        for formdef in FormDef.select() + CardDef.select():
3590
            if not formdef.digest_templates:
3591
                continue
3592
            for formdata in formdef.data_class().select_iterator():
3593
                formdata._set_auto_fields(cur)  # build digests
3561 3594
    if sql_level < 42:
3562 3595
        # 42: create snapshots table
3563 3596
        do_snapshots_table()
wcs/variables.py
408 408

  
409 409
    @property
410 410
    def digest(self):
411
        return self._formdata.digest
411
        return self._formdata.default_digest
412 412

  
413 413
    @property
414 414
    def display_name(self):
415
-