From 15632b054153a820a9d1425fd1b295effc36b329 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Tue, 18 May 2021 09:28:37 +0200
Subject: [PATCH 2/2] sql: add blockdef fields to fulltext indexation (#53284)

---
 tests/test_sql.py | 43 +++++++++++++++++++++++++++++++++++++++++++
 wcs/sql.py        | 24 ++++++++++++++++++------
 2 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/tests/test_sql.py b/tests/test_sql.py
index 7fcefc19..e08317a9 100644
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -4,16 +4,17 @@ import string
 import time
 
 import psycopg2
 import pytest
 from quixote import cleanup
 
 import wcs.qommon.storage as st
 from wcs import fields, sql
+from wcs.blocks import BlockDef
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon import force_str
 from wcs.wf.jump import JumpWorkflowStatusItem
 from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
 from wcs.workflows import CommentableWorkflowStatusItem, Workflow, WorkflowCriticalityLevel
 
 from .utilities import clean_temporary_pub, create_temporary_pub
@@ -21,26 +22,35 @@ from .utilities import clean_temporary_pub, create_temporary_pub
 
 def setup_module(module):
     global pub, formdef
 
     cleanup()
 
     pub = create_temporary_pub(sql_mode=True)
 
+    block = BlockDef()
+    block.name = 'fooblock'
+    block.fields = [
+        fields.StringField(id='1', label='string', type='string'),
+        fields.ItemField(id='2', label='item', items=('boat', 'plane', 'kick scooter')),
+    ]
+    block.store()
+
     formdef = FormDef()
     formdef.name = 'tests'
     formdef.fields = [
         fields.StringField(id='0', label='string'),
         fields.EmailField(id='1', label='email'),
         fields.TextField(id='2', label='text'),
         fields.BoolField(id='3', label='bool'),
         fields.ItemField(id='4', label='item', items=('apple', 'pear', 'peach', 'apricot')),
         fields.DateField(id='5', label='date'),
         fields.ItemsField(id='6', label='items', items=('apple', 'pear', 'peach', 'apricot')),
+        fields.BlockField(id='7', label='block', type='block:fooblock'),
     ]
     formdef.store()
 
 
 def teardown_module(module):
     clean_temporary_pub()
 
 
@@ -157,16 +167,32 @@ def test_sql_field_date():
 
 
 def test_sql_field_items():
     check_sql_field('6', ['apricot'], display=True)
     check_sql_field('6', ['apricot', 'pear'], display=True)
     check_sql_field('6', ['pomme', 'poire', 'pêche'], display=True)
 
 
+def test_sql_block_field_text():
+    check_sql_field('7', {'data': [{'1': 'foo'}, {'1': 'bar'}]})
+
+
+def test_sql_block_field_item():
+    check_sql_field(
+        '7',
+        {
+            'data': [
+                {'2': 'boat', '2_display': 'Yacht'},
+                {'2': 'plane', '2_display': 'Cessna'},
+            ]
+        },
+    )
+
+
 def test_sql_geoloc():
     test_formdef = FormDef()
     test_formdef.name = 'geoloc'
     test_formdef.fields = []
     test_formdef.geolocations = {'base': 'Plop'}
     test_formdef.store()
     data_class = test_formdef.data_class(mode='sql')
     formdata = data_class()
@@ -364,22 +390,39 @@ def test_sql_get_ids_from_query():
     formdata.store()
     id2 = formdata.id
 
     formdata = data_class()
     formdata.data = {'2': 'you would think other ideas of text would emerge'}
     formdata.store()
     id3 = formdata.id
 
+    formdata = data_class()
+    formdata.data = {
+        '7': {
+            'data': [
+                {'1': 'some other example having foo', '2': 'boat', '2_display': 'Yatch'},
+                {'1': 'bar', '2': 'plane', '2_display': 'Cessna'},
+            ]
+        }
+    }
+    formdata.store()
+    id4 = formdata.id
+
     ids = data_class.get_ids_from_query('text')
     assert set(ids) == set([id1, id3])
 
     ids = data_class.get_ids_from_query('classical')
     assert set(ids) == set([id2])
 
+    ids = data_class.get_ids_from_query('FOO')
+    assert set(ids) == set([id4])
+    ids = data_class.get_ids_from_query('cessna')
+    assert set(ids) == set([id4])
+
 
 def test_sql_rollback_on_error():
     data_class = formdef.data_class(mode='sql')
     data_class.wipe()
     with pytest.raises(psycopg2.Error):
         # this will raise a psycopg2.ProgrammingError as there's no FOOBAR
         # column in the table.
         data_class.get_ids_with_indexed_value('FOOBAR', '2')
diff --git a/wcs/sql.py b/wcs/sql.py
index 793af668..d180a864 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -2091,24 +2091,35 @@ class SqlDataMixin(SqlMixin):
                     sql_dict['parts'] = None
                 cur.execute(sql_statement, sql_dict)
                 evo._sql_id = cur.fetchone()[0]
 
         fts_strings = [str(self.id), self.get_display_id()]
         fts_strings.append(self._formdef.name)
         if self.tracking_code:
             fts_strings.append(self.tracking_code)
-        for field in self._formdef.get_all_fields():
-            if not self.data.get(field.id):
+
+        def get_all_fields():
+            for field in self._formdef.get_all_fields():
+                if field.key == 'block' and self.data.get(field.id):
+                    for data in self.data[field.id].get('data'):
+                        for subfield in field.block.fields:
+                            yield subfield, data
+                else:
+                    data = self.data
+                    yield field, self.data
+
+        for field, data in get_all_fields():
+            if not data.get(field.id):
                 continue
             value = None
             if field.key in ('string', 'text', 'email'):
-                value = self.data.get(field.id)
+                value = data.get(field.id)
             elif field.key in ('item', 'items'):
-                value = self.data.get('%s_display' % field.id)
+                value = data.get('%s_display' % field.id)
             if value:
                 if isinstance(value, str):
                     fts_strings.append(value)
                 elif type(value) in (tuple, list):
                     fts_strings.extend(value)
         if self._evolution:
             for evo in self._evolution:
                 if evo.comment:
@@ -3342,17 +3353,17 @@ def get_period_total(
     cur.close()
 
     return result
 
 
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (50, 'switch role uuid column to varchar')
+SQL_LEVEL = (51, 'add index on formdata blockdef fields')
 
 
 def migrate_global_views(conn, cur):
     cur.execute(
         '''SELECT COUNT(*) FROM information_schema.tables
                     WHERE table_schema = 'public'
                       AND table_name = %s''',
         ('wcs_all_forms',),
@@ -3467,26 +3478,27 @@ def migrate():
         migrate_views(conn, cur)
         for formdef in FormDef.select():
             formdef.data_class().rebuild_security()
     if sql_level < 23:
         # 12: (second part), store fts in existing rows
         # 21: (second part), store ascii_name of users
         # 23: (first part), use misc.simplify() over full text queries
         set_reindex('user', 'needed', conn=conn, cur=cur)
-    if sql_level < 41:
+    if sql_level < 51:
         # 17: store last_update_time in tables
         # 18: add user name to full-text search index
         # 21: (third part), add user ascii_names to full-text index
         # 23: (second part) use misc.simplify() over full text queries
         # 28: add display id and formdef name to full-text index
         # 29: add evolution parts to full-text index
         # 31: add user_label to formdata
         # 38: extract submission_agent_id to its own column
         # 41: update full text normalization
+        # 51: add index on formdata blockdef fields
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 46:
         from wcs.carddef import CardDef
         from wcs.formdef import FormDef
 
         # 24: add index on evolution(formdata_id)
         # 35: add indexes on formdata(receipt_time) and formdata(anonymised)
         # 36: add index on formdata(user_id)
-- 
2.30.2

