From d214f306ed4c2883f991235ba1d3a5dbfa05d79c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Sat, 19 Mar 2022 15:27:32 +0100
Subject: [PATCH 8/9] sql: do not include carddata in wcs_all_forms (#60552)

---
 wcs/sql.py | 113 +++++++++++++++++++++++++----------------------------
 1 file changed, 53 insertions(+), 60 deletions(-)

diff --git a/wcs/sql.py b/wcs/sql.py
index fd1b586e2..7b65e6eed 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -574,7 +574,7 @@ def formdef_wipe():
     row = cur.fetchone()
     # only do the delete if wcs_all_forms is a table and not still a view
     if row is not None and row[0] == 'r':
-        cur.execute("DELETE FROM wcs_all_forms WHERE object_type = 'formdata'")
+        cur.execute('TRUNCATE wcs_all_forms')
     conn.commit()
     cur.close()
 
@@ -792,7 +792,49 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
     for field in existing_fields - needed_fields:
         cur.execute('''ALTER TABLE %s DROP COLUMN %s CASCADE''' % (table_name, field))
 
+    if formdef.data_sql_prefix == 'formdata':
+        recreate_trigger(formdef, cur, conn)
+
+    # migrations on _evolutions table
+    cur.execute(
+        '''SELECT column_name FROM information_schema.columns
+                    WHERE table_schema = 'public'
+                      AND table_name = '%s_evolutions'
+                '''
+        % table_name
+    )
+    evo_existing_fields = {x[0] for x in cur.fetchall()}
+    if 'last_jump_datetime' not in evo_existing_fields:
+        cur.execute('''ALTER TABLE %s_evolutions ADD COLUMN last_jump_datetime timestamp''' % table_name)
+
+    if rebuild_views or len(existing_fields - needed_fields):
+        # views may have been dropped when dropping columns, so we recreate
+        # them even if not asked to.
+        redo_views(conn, cur, formdef, rebuild_global_views=rebuild_global_views)
+
+    if new_table:
+        do_formdef_indexes(formdef, created=True, conn=conn, cur=cur)
+
+    if own_conn:
+        conn.commit()
+        cur.close()
+
+    actions = []
+    if 'concerned_roles_array' not in existing_fields:
+        actions.append('rebuild_security')
+    elif 'actions_roles_array' not in existing_fields:
+        actions.append('rebuild_security')
+    if 'tracking_code' not in existing_fields:
+        # if tracking code has just been added to the table we need to make
+        # sure the tracking code table does exist.
+        actions.append('do_tracking_code_table')
+
+    return actions
+
+
+def recreate_trigger(formdef, cur, conn):
     # recreate the trigger function, just so it's uptodate
+    table_name = get_formdef_table_name(formdef)
     category_value = formdef.category_id
     geoloc_base_x_query = "NULL"
     geoloc_base_y_query = "NULL"
@@ -810,7 +852,6 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild
     if endpoint_status_filter == "":
         # not the prettiest in town, but will do fine for now.
         endpoint_status_filter = "'xxxx'"
-    object_type = formdef.data_sql_prefix  # shortcut.
     formed_name_quotedstring = psycopg2.extensions.QuotedString(formdef.name)
     formed_name_quotedstring.encoding = 'utf8'
     formdef_name = formed_name_quotedstring.getquoted().decode()
@@ -823,12 +864,11 @@ AS $$
 BEGIN
     -- TODO : sync back from users change !
     IF TG_OP = 'DELETE' THEN
-        DELETE FROM wcs_all_forms WHERE formdef_id = {formdef_id} AND id = OLD.id AND object_type = '{object_type}';
+        DELETE FROM wcs_all_forms WHERE formdef_id = {formdef_id} AND id = OLD.id;
         RETURN OLD;
     ELSEIF TG_OP = 'INSERT' THEN
         INSERT INTO wcs_all_forms VALUES (
             {category_id},
-            '{object_type}',
             {formdef_id},
             NEW.id,
             NEW.user_id,
@@ -874,7 +914,7 @@ BEGIN
                 geoloc_base_x = {geoloc_base_x},
                 geoloc_base_y = {geoloc_base_y},
                 anonymised = NEW.anonymised
-            WHERE formdef_id = {formdef_id}  AND id = OLD.id AND object_type = '{object_type}';
+            WHERE formdef_id = {formdef_id} AND id = OLD.id;
         RETURN NEW;
     END IF;
 END;
@@ -888,7 +928,6 @@ $$;
             formdef_name=formdef_name,
             criticality_levels=criticality_levels,
             endpoint_status=endpoint_status_filter,
-            object_type=object_type,
         )
     )
 
@@ -912,42 +951,6 @@ $$;
             )
         )
 
-    # migrations on _evolutions table
-    cur.execute(
-        '''SELECT column_name FROM information_schema.columns
-                    WHERE table_schema = 'public'
-                      AND table_name = '%s_evolutions'
-                '''
-        % table_name
-    )
-    evo_existing_fields = {x[0] for x in cur.fetchall()}
-    if 'last_jump_datetime' not in evo_existing_fields:
-        cur.execute('''ALTER TABLE %s_evolutions ADD COLUMN last_jump_datetime timestamp''' % table_name)
-
-    if rebuild_views or len(existing_fields - needed_fields):
-        # views may have been dropped when dropping columns, so we recreate
-        # them even if not asked to.
-        redo_views(conn, cur, formdef, rebuild_global_views=rebuild_global_views)
-
-    if new_table:
-        do_formdef_indexes(formdef, created=True, conn=conn, cur=cur)
-
-    if own_conn:
-        conn.commit()
-        cur.close()
-
-    actions = []
-    if 'concerned_roles_array' not in existing_fields:
-        actions.append('rebuild_security')
-    elif 'actions_roles_array' not in existing_fields:
-        actions.append('rebuild_security')
-    if 'tracking_code' not in existing_fields:
-        # if tracking code has just been added to the table we need to make
-        # sure the tracking code table does exist.
-        actions.append('do_tracking_code_table')
-
-    return actions
-
 
 def do_formdef_indexes(formdef, created, conn, cur, concurrently=False):
     table_name = get_formdef_table_name(formdef)
@@ -1664,14 +1667,12 @@ def drop_global_views(conn, cur):
 
 def do_global_views(conn, cur):
     # recreate global views
-    from .carddef import CardDef
     from .formdef import FormDef
 
     # XXX TODO: make me dynamic, please ?
     cur.execute(
         """CREATE TABLE IF NOT EXISTS wcs_all_forms (
         category_id integer,
-        object_type character varying NOT NULL, -- formdef or carddef
         formdef_id integer NOT NULL,
         id integer NOT NULL,
         user_id character varying,
@@ -1694,7 +1695,7 @@ def do_global_views(conn, cur):
         geoloc_base_x double precision,
         geoloc_base_y double precision,
         anonymised timestamp with time zone
-        , PRIMARY KEY(object_type, formdef_id, id)
+        , PRIMARY KEY(formdef_id, id)
     )"""
     )
     cur.execute(
@@ -1712,17 +1713,13 @@ def do_global_views(conn, cur):
         )
 
     # Purge of any dead data
-    valid_data = {FormDef.data_sql_prefix: set(), CardDef.data_sql_prefix: set()}
+    valid_ids = set()
     for formdef in FormDef.select(ignore_migration=True):
-        valid_data[formdef.data_sql_prefix].add(formdef.id)
-    for object_type, valid_ids in valid_data.items():
-        if valid_ids:
-            cur.execute(
-                "DELETE FROM wcs_all_forms WHERE object_type = '%s' AND formdef_id NOT IN (%s)"
-                % (object_type, ", ".join(valid_ids))
-            )
-        else:
-            cur.execute("DELETE FROM wcs_all_forms WHERE object_type = '%s'" % object_type)
+        valid_ids.add(int(formdef.id))
+    if valid_ids:
+        cur.execute('DELETE FROM wcs_all_forms WHERE NOT formdef_id = ANY(%s)', (list(valid_ids),))
+    else:
+        cur.execute('TRUNCATE wcs_all_forms')
 
     for category in wcs.categories.Category.select():
         name = get_name_as_sql_identifier(category.url_name)[:40]
@@ -1734,7 +1731,6 @@ def do_global_views(conn, cur):
 
 
 def init_global_table(conn=None, cur=None):
-    from .carddef import CardDef
     from .formdef import FormDef
 
     own_conn = False
@@ -1755,7 +1751,7 @@ def init_global_table(conn=None, cur=None):
     do_global_views(conn, cur)
 
     # now copy all data into the table
-    for formdef in FormDef.select() + CardDef.select():
+    for formdef in FormDef.select():
         category_value = formdef.category_id
         if formdef.category_id is None:
             category_value = "NULL"
@@ -1773,7 +1769,6 @@ def init_global_table(conn=None, cur=None):
         if endpoint_status_filter == "":
             # not the prettiest in town, but will do fine for now.
             endpoint_status_filter = "'xxxx'"
-        object_type = formdef.data_sql_prefix  # shortcut.
         formed_name_quotedstring = psycopg2.extensions.QuotedString(formdef.name)
         formed_name_quotedstring.encoding = 'utf8'
         formdef_name = formed_name_quotedstring.getquoted().decode()
@@ -1782,7 +1777,6 @@ def init_global_table(conn=None, cur=None):
             INSERT INTO wcs_all_forms
             SELECT
                 {category_id},
-                '{object_type}',
                 {formdef_id},
                 id,
                 user_id,
@@ -1816,7 +1810,6 @@ def init_global_table(conn=None, cur=None):
                 formdef_name=formdef_name,
                 criticality_levels=criticality_levels,
                 endpoint_status=endpoint_status_filter,
-                object_type=object_type,
             )
         )
 
-- 
2.35.1

