From 9daa29670964de96267326667e98c89d67aaca6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Tue, 15 Mar 2022 15:55:09 +0100
Subject: [PATCH] sql: use setweight on fts strings (#16252)

---
 wcs/sql.py | 80 ++++++++++++++++++++++++++++++++----------------------
 1 file changed, 47 insertions(+), 33 deletions(-)

diff --git a/wcs/sql.py b/wcs/sql.py
index 60a4afa05..763534127 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -2357,10 +2357,12 @@ class SqlDataMixin(SqlMixin):
                 cur.execute(sql_statement, sql_dict)
                 evo._sql_id = cur.fetchone()[0]
 
-        fts_strings = [str(self.id), self.get_display_id()]
-        fts_strings.append(self._formdef.name)
+        fts_strings = []
+        fts_strings.append(('A', str(self.id)))
+        fts_strings.append(('A', self.get_display_id()))
+        fts_strings.append(('C', self._formdef.name))
         if self.tracking_code:
-            fts_strings.append(self.tracking_code)
+            fts_strings.append(('A', self.tracking_code))
 
         def get_all_fields():
             for field in self._formdef.get_all_fields():
@@ -2385,33 +2387,36 @@ class SqlDataMixin(SqlMixin):
             elif field.key in ('item', 'items'):
                 value = data.get('%s_display' % field.id)
             if value:
+                weight = 'C'
+                if field.include_in_listing:
+                    weight = 'B'
                 if isinstance(value, str) and len(value) < 10000:
                     # avoid overlong strings, typically base64-encoded values
-                    fts_strings.append(value)
+                    fts_strings.append((weight, value))
                 elif type(value) in (tuple, list):
-                    fts_strings.extend(value)
+                    for val in value:
+                        fts_strings.append((weight, val))
         if self._evolution:
             for evo in self._evolution:
                 if evo.comment:
-                    fts_strings.append(evo.comment)
+                    fts_strings.append(('D', evo.comment))
                 for part in evo.parts or []:
-                    fts_strings.append(part.render_for_fts() if part.render_for_fts else '')
+                    fts_strings.append(('D', part.render_for_fts() if part.render_for_fts else ''))
         user = self.get_user()
         if user:
-            fts_strings.append(user.get_display_name())
-
-        sql_statement = (
-            '''UPDATE %s SET fts = to_tsvector( %%(fts)s)
-                            WHERE id = %%(id)s'''
-            % self._table_name
-        )
-        cur.execute(
-            sql_statement,
-            {
-                'id': self.id,
-                'fts': FtsMatch.get_fts_value(' '.join(str(fts_string) for fts_string in fts_strings)),
-            },
+            fts_strings.append(('A', user.get_display_name()))
+
+        fts_parts = []
+        parameters = {'id': self.id}
+        for i, (weight, value) in enumerate(fts_strings):
+            fts_parts.append("setweight(to_tsvector(%%(fts%s)s), '%s')" % (i, weight))
+            parameters['fts%s' % i] = FtsMatch.get_fts_value(value)
+        sql_statement = '''UPDATE %s SET fts = %s
+                            WHERE id = %%(id)s''' % (
+            self._table_name,
+            ' || '.join(fts_parts),
         )
+        cur.execute(sql_statement, parameters)
 
         conn.commit()
         cur.close()
@@ -2688,10 +2693,10 @@ class SqlUser(SqlMixin, wcs.users.User):
 
         fts_strings = []
         if self.name:
-            fts_strings.append(self.name)
-            fts_strings.append(self.ascii_name)
+            fts_strings.append(('A', self.name))
+            fts_strings.append(('A', self.ascii_name))
         if self.email:
-            fts_strings.append(self.email)
+            fts_strings.append(('B', self.email))
         if user_formdef and user_formdef.fields:
             for field in user_formdef.fields:
                 if not self.form_data.get(field.id):
@@ -2703,15 +2708,22 @@ class SqlUser(SqlMixin, wcs.users.User):
                     value = self.form_data.get('%s_display' % field.id)
                 if value:
                     if isinstance(value, str):
-                        fts_strings.append(value)
+                        fts_strings.append(('B', value))
                     elif type(value) in (tuple, list):
-                        fts_strings.extend(value)
-        sql_statement = (
-            '''UPDATE %s SET fts = to_tsvector( %%(fts)s)
-                            WHERE id = %%(id)s'''
-            % self._table_name
+                        for val in value:
+                            fts_strings.append(('B', val))
+
+        fts_parts = []
+        parameters = {'id': self.id}
+        for i, (weight, value) in enumerate(fts_strings):
+            fts_parts.append("setweight(to_tsvector(%%(fts%s)s), '%s')" % (i, weight))
+            parameters['fts%s' % i] = FtsMatch.get_fts_value(value)
+        sql_statement = '''UPDATE %s SET fts = %s
+                            WHERE id = %%(id)s''' % (
+            self._table_name,
+            ' || '.join(fts_parts),
         )
-        cur.execute(sql_statement, {'id': self.id, 'fts': ' '.join(fts_strings)})
+        cur.execute(sql_statement, parameters)
 
         conn.commit()
         cur.close()
@@ -3758,7 +3770,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (58, 'add workflow_merged_roles_dict')
+SQL_LEVEL = (59, 'use setweight on formdata é user indexation')
 
 
 def migrate_global_views(conn, cur):
@@ -3879,12 +3891,13 @@ def migrate():
         migrate_views(conn, cur)
         for formdef in FormDef.select():
             formdef.data_class().rebuild_security()
-    if sql_level < 23:
+    if sql_level < 59:
         # 12: (second part), store fts in existing rows
         # 21: (second part), store ascii_name of users
         # 23: (first part), use misc.simplify() over full text queries
+        # 59: use setweight on formdata é user indexation
         set_reindex('user', 'needed', conn=conn, cur=cur)
-    if sql_level < 58:
+    if sql_level < 59:
         # 17: store last_update_time in tables
         # 18: add user name to full-text search index
         # 21: (third part), add user ascii_names to full-text index
@@ -3896,6 +3909,7 @@ def migrate():
         # 41: update full text normalization
         # 51: add index on formdata blockdef fields
         # 55: update full text normalisation (switch to unidecode)
+        # 59: use setweight on formdata é user indexation
         set_reindex('formdata', 'needed', conn=conn, cur=cur)
     if sql_level < 56:
         from wcs.carddef import CardDef
-- 
2.34.1

