From 0efb07dd8752522f3e39a02d8813cea47ae8720a Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Fri, 20 Apr 2018 12:17:55 +0200 Subject: [PATCH 2/2] sql: delay re-index operations on first cron (#22383) --- tests/test_sql.py | 20 +++++++++++++++++ wcs/publisher.py | 4 ++++ wcs/qommon/cron.py | 5 +++++ wcs/sql.py | 55 ++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 77 insertions(+), 7 deletions(-) diff --git a/tests/test_sql.py b/tests/test_sql.py index fd9a84aa..2ec226cb 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -8,6 +8,8 @@ import string import sys import time +from django.core.management import call_command + from quixote import cleanup from wcs import formdef, publisher, fields @@ -1102,6 +1104,15 @@ def test_migration_12_users_fts(): assert column_exists_in_table(cur, 'users', 'fts') assert migration_level(cur) >= 12 + # no fts, migration only prepare re-index + assert len(sql.SqlUser.get_ids_from_query('pierre')) == 0 + + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True + call_command('cron') # first cron = reindex + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False + # make sure the fts is filled after the migration assert len(sql.SqlUser.get_ids_from_query('pierre')) == 1 @@ -1127,6 +1138,15 @@ def test_migration_21_users_ascii_name(): assert column_exists_in_table(cur, 'users', 'ascii_name') assert migration_level(cur) >= 21 + # no fts, migration only prepare re-index + assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 0 + + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True + call_command('cron') # first cron = reindex + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False + # make sure the ascii_name is filled after the migration assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 1 diff --git a/wcs/publisher.py b/wcs/publisher.py index 1ebcbea0..2a1e49c6 100644 --- a/wcs/publisher.py +++ b/wcs/publisher.py @@ -298,6 +298,10 @@ class WcsPublisher(StubWcsPublisher): import sql sql.migrate() + def reindex_sql(self): + import sql + sql.reindex() + def cleanup(self): if self.is_using_postgresql(): import sql diff --git a/wcs/qommon/cron.py b/wcs/qommon/cron.py index 15a46366..6af321ba 100644 --- a/wcs/qommon/cron.py +++ b/wcs/qommon/cron.py @@ -35,6 +35,11 @@ def cron_worker(publisher, now): publisher.set_config() except: return + + # reindex user and formdata if needed (should only be run once) + if publisher.is_using_postgresql(): + publisher.reindex_sql() + for job in publisher.cronjobs: if job.days and now[2] not in job.days: continue diff --git a/wcs/sql.py b/wcs/sql.py index ae115c3d..2a1459df 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -2123,6 +2123,31 @@ def get_sql_level(conn, cur): sql_level = int(cur.fetchone()[0]) return sql_level +@guard_postgres +def is_reindex_needed(index, conn, cur): + do_meta_table(conn, cur, insert_current_sql_level=False) + key_name = 'reindex_%s' % index + cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, )) + row = cur.fetchone() + if row is None: + cur.execute('''INSERT INTO wcs_meta (id, key, value) + VALUES (DEFAULT, %s, %s)''', (key_name, 'no')) + return False + return row[0] == 'needed' + +@guard_postgres +def set_reindex(index, value, conn, cur): + do_meta_table(conn, cur, insert_current_sql_level=False) + key_name = 'reindex_%s' % index + cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, )) + row = cur.fetchone() + if row is None: + cur.execute('''INSERT INTO wcs_meta (id, key, value) + VALUES (DEFAULT, %s, %s)''', (key_name, value)) + else: + cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', ( + value, key_name)) + def migrate_views(conn, cur): drop_views(None, conn, cur) from wcs.formdef import FormDef @@ -2172,18 +2197,13 @@ def migrate(): # 12: (second part), store fts in existing rows # 21: (second part), store ascii_name of users # 23: (first part), use misc.simplify() over full text queries - for user_id in SqlUser.keys(): - SqlUser.get(user_id).store() + set_reindex('user', 'needed', conn=conn, cur=cur) if sql_level < 23: # 17: store last_update_time in tables # 18: add user name to full-text search index # 21: (third part), add user ascii_names to full-text index # 23: (second part) use misc.simplify() over full text queries - # load and store all formdatas - from wcs.formdef import FormDef - for formdef in FormDef.select(): - for formdata in formdef.data_class().select(): - formdata.store() + set_reindex('formdata', 'needed', conn=conn, cur=cur) if sql_level < 24: from wcs.formdef import FormDef # 24: add index on evolution(formdata_id) @@ -2197,3 +2217,24 @@ def migrate(): conn.commit() cur.close() + + +@guard_postgres +def reindex(): + conn, cur = get_connection_and_cursor() + + if is_reindex_needed('user', conn=conn, cur=cur): + for user in SqlUser.select(iterator=True): + user.store() + set_reindex('user', 'done', conn=conn, cur=cur) + + if is_reindex_needed('formdata', conn=conn, cur=cur): + # load and store all formdatas + from wcs.formdef import FormDef + for formdef in FormDef.select(): + for formdata in formdef.data_class().select(iterator=True): + formdata.store() + set_reindex('formdata', 'done', conn=conn, cur=cur) + + conn.commit() + cur.close() -- 2.17.0