Projet

Général

Profil

0002-sql-add-blockdef-fields-to-fulltext-indexation-53284.patch

Nicolas Roche, 30 avril 2021 18:25

Télécharger (7,33 ko)

Voir les différences:

Subject: [PATCH 2/2] sql: add blockdef fields to fulltext indexation (#53284)

 tests/test_sql.py | 43 +++++++++++++++++++++++++++++++++++++++++++
 wcs/sql.py        | 23 ++++++++++++++++++-----
 2 files changed, 61 insertions(+), 5 deletions(-)
tests/test_sql.py
6 6
import time
7 7

  
8 8
import psycopg2
9 9
import pytest
10 10
from quixote import cleanup
11 11

  
12 12
import wcs.qommon.storage as st
13 13
from wcs import fields, sql
14
from wcs.blocks import BlockDef
14 15
from wcs.formdata import Evolution
15 16
from wcs.formdef import FormDef
16 17
from wcs.qommon import force_str
17 18
from wcs.wf.jump import JumpWorkflowStatusItem
18 19
from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
19 20
from wcs.workflows import CommentableWorkflowStatusItem, Workflow, WorkflowCriticalityLevel
20 21

  
21 22
from .utilities import clean_temporary_pub, create_temporary_pub
......
23 24

  
24 25
def setup_module(module):
25 26
    global pub, formdef
26 27

  
27 28
    cleanup()
28 29

  
29 30
    pub = create_temporary_pub(sql_mode=True)
30 31

  
32
    block = BlockDef()
33
    block.name = 'fooblock'
34
    block.fields = [
35
        fields.StringField(id='1', label='string', type='string'),
36
        fields.ItemField(id='2', label='item', items=('boat', 'plane', 'kick scooter')),
37
    ]
38
    block.store()
39

  
31 40
    formdef = FormDef()
32 41
    formdef.name = 'tests'
33 42
    formdef.fields = [
34 43
        fields.StringField(id='0', label='string'),
35 44
        fields.EmailField(id='1', label='email'),
36 45
        fields.TextField(id='2', label='text'),
37 46
        fields.BoolField(id='3', label='bool'),
38 47
        fields.ItemField(id='4', label='item', items=('apple', 'pear', 'peach', 'apricot')),
39 48
        fields.DateField(id='5', label='date'),
40 49
        fields.ItemsField(id='6', label='items', items=('apple', 'pear', 'peach', 'apricot')),
50
        fields.BlockField(id='7', label='block', type='block:fooblock'),
41 51
    ]
42 52
    formdef.store()
43 53

  
44 54

  
45 55
def teardown_module(module):
46 56
    clean_temporary_pub()
47 57

  
48 58

  
......
159 169

  
160 170

  
161 171
def test_sql_field_items():
162 172
    check_sql_field('6', ['apricot'], display=True)
163 173
    check_sql_field('6', ['apricot', 'pear'], display=True)
164 174
    check_sql_field('6', ['pomme', 'poire', 'pêche'], display=True)
165 175

  
166 176

  
177
def test_sql_block_field_text():
178
    check_sql_field('7', {'data': [{'1': 'foo'}, {'1': 'bar'}]})
179

  
180

  
181
def test_sql_block_field_item():
182
    check_sql_field(
183
        '7',
184
        {
185
            'data': [
186
                {'2': 'boat', '2_display': 'Yacht'},
187
                {'2': 'plane', '2_display': 'Cessna'},
188
            ]
189
        },
190
    )
191

  
192

  
167 193
def test_sql_geoloc():
168 194
    test_formdef = FormDef()
169 195
    test_formdef.name = 'geoloc'
170 196
    test_formdef.fields = []
171 197
    test_formdef.geolocations = {'base': 'Plop'}
172 198
    test_formdef.store()
173 199
    data_class = test_formdef.data_class(mode='sql')
174 200
    formdata = data_class()
......
366 392
    formdata.store()
367 393
    id2 = formdata.id
368 394

  
369 395
    formdata = data_class()
370 396
    formdata.data = {'2': 'you would think other ideas of text would emerge'}
371 397
    formdata.store()
372 398
    id3 = formdata.id
373 399

  
400
    formdata = data_class()
401
    formdata.data = {
402
        '7': {
403
            'data': [
404
                {'1': 'some other example having foo', '2': 'boat', '2_display': 'Yatch'},
405
                {'1': 'bar', '2': 'plane', '2_display': 'Cessna'},
406
            ]
407
        }
408
    }
409
    formdata.store()
410
    id4 = formdata.id
411

  
374 412
    ids = data_class.get_ids_from_query('text')
375 413
    assert set(ids) == set([id1, id3])
376 414

  
377 415
    ids = data_class.get_ids_from_query('classical')
378 416
    assert set(ids) == set([id2])
379 417

  
418
    ids = data_class.get_ids_from_query('FOO')
419
    assert set(ids) == set([id4])
420
    ids = data_class.get_ids_from_query('cessna')
421
    assert set(ids) == set([id4])
422

  
380 423

  
381 424
def test_sql_rollback_on_error():
382 425
    data_class = formdef.data_class(mode='sql')
383 426
    data_class.wipe()
384 427
    with pytest.raises(psycopg2.Error):
385 428
        # this will raise a psycopg2.ProgrammingError as there's no FOOBAR
386 429
        # column in the table.
387 430
        data_class.get_ids_with_indexed_value('FOOBAR', '2')
wcs/sql.py
2060 2060
                    sql_dict['parts'] = None
2061 2061
                cur.execute(sql_statement, sql_dict)
2062 2062
                evo._sql_id = cur.fetchone()[0]
2063 2063

  
2064 2064
        fts_strings = [str(self.id), self.get_display_id()]
2065 2065
        fts_strings.append(self._formdef.name)
2066 2066
        if self.tracking_code:
2067 2067
            fts_strings.append(self.tracking_code)
2068
        for field in self._formdef.get_all_fields():
2069
            if not self.data.get(field.id):
2068

  
2069
        def get_all_fields():
2070
            for field in self._formdef.get_all_fields():
2071
                if field.key == 'block' and self.data.get(field.id):
2072
                    for data in self.data[field.id].get('data'):
2073
                        for subfield in field.block.fields:
2074
                            yield subfield, data
2075
                else:
2076
                    yield field, self.data
2077

  
2078
        for field, data in get_all_fields():
2079
            if not data.get(field.id):
2070 2080
                continue
2071 2081
            value = None
2072 2082
            if field.key in ('string', 'text', 'email'):
2073
                value = self.data.get(field.id)
2083
                value = data.get(field.id)
2074 2084
            elif field.key in ('item', 'items'):
2075
                value = self.data.get('%s_display' % field.id)
2085
                value = data.get('%s_display' % field.id)
2076 2086
            if value:
2077 2087
                if isinstance(value, str):
2078 2088
                    fts_strings.append(value)
2079 2089
                elif type(value) in (tuple, list):
2080 2090
                    fts_strings.extend(value)
2081 2091
        if self._evolution:
2082 2092
            for evo in self._evolution:
2083 2093
                if evo.comment:
......
3308 3318
    cur.close()
3309 3319

  
3310 3320
    return result
3311 3321

  
3312 3322

  
3313 3323
# latest migration, number + description (description is not used
3314 3324
# programmaticaly but will make sure git conflicts if two migrations are
3315 3325
# separately added with the same number)
3316
SQL_LEVEL = (50, 'switch role uuid column to varchar')
3326
SQL_LEVEL = (51, 'add index on formdata blockdef fields')
3317 3327

  
3318 3328

  
3319 3329
def migrate_global_views(conn, cur):
3320 3330
    cur.execute(
3321 3331
        '''SELECT COUNT(*) FROM information_schema.tables
3322 3332
                    WHERE table_schema = 'public'
3323 3333
                      AND table_name = %s''',
3324 3334
        ('wcs_all_forms',),
......
3484 3494
        # 47: store LoggedErrors in SQL
3485 3495
        # 48: remove acked attribute from LoggedError
3486 3496
        do_loggederrors_table()
3487 3497
    if sql_level < 50:
3488 3498
        # 49: store Role in SQL
3489 3499
        # 50: switch role uuid column to varchar
3490 3500
        do_role_table()
3491 3501
        migrate_legacy_roles()
3502
    if sql_level < 51:
3503
        # 51: add index on formdata blockdef fields
3504
        set_reindex('formdata', 'needed', conn=conn, cur=cur)
3492 3505

  
3493 3506
    cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', (str(SQL_LEVEL[0]), 'sql_level'))
3494 3507

  
3495 3508
    conn.commit()
3496 3509
    cur.close()
3497 3510

  
3498 3511

  
3499 3512
@guard_postgres
3500
-