From fadccbbcd858ad3a93faf2200192173870b40848 Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Mon, 9 Apr 2018 20:50:12 +0200 Subject: [PATCH] sql: delay re-index operations on first cron (#22383) --- tests/test_sql.py | 44 ++++++++++++++----------------------- wcs/publisher.py | 4 ++++ wcs/qommon/cron.py | 5 +++++ wcs/sql.py | 55 ++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 74 insertions(+), 34 deletions(-) diff --git a/tests/test_sql.py b/tests/test_sql.py index 3a423a3e..4e046e32 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -8,6 +8,8 @@ import string import sys import time +from django.core.management import call_command + from quixote import cleanup from wcs import formdef, publisher, fields @@ -19,7 +21,7 @@ from wcs.wf.jump import JumpWorkflowStatusItem from wcs import sql import wcs.qommon.storage as st -from utilities import create_temporary_pub +from utilities import create_temporary_pub, clean_temporary_pub import pytest postgresql = pytest.mark.postgresql @@ -34,19 +36,7 @@ def setup_module(module): cleanup() - pub = create_temporary_pub() - pub.user_class = sql.SqlUser - pub.is_using_postgresql = lambda: True - - conn = psycopg2.connect(user=os.environ['USER']) - conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = conn.cursor() - dbname = 'wcstests%d' % random.randint(0, 100000) - cur.execute('CREATE DATABASE %s' % dbname) - cur.close() - - pub.cfg['postgresql'] = {'database': dbname, 'user': os.environ['USER']} - pub.initialize_sql() + pub = create_temporary_pub(sql_mode=True) formdef = formdef.FormDef() formdef.name = 'tests' @@ -61,20 +51,8 @@ def setup_module(module): ] formdef.store() - conn.close() - - def teardown_module(module): - shutil.rmtree(pub.APP_DIR) - - if hasattr(pub, 'pgconn') and pub.pgconn: - pub.pgconn.close() - - conn = psycopg2.connect(user=os.environ['USER']) - conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = conn.cursor() - cur.execute('DROP DATABASE %s' % pub.cfg['postgresql']['database']) - cur.close() + clean_temporary_pub() @postgresql def test_sql_table_name_invalid_chars(): @@ -1098,6 +1076,12 @@ def test_migration_12_users_fts(): assert column_exists_in_table(cur, 'users', 'fts') assert migration_level(cur) >= 12 + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True + call_command('cron') # first cron = reindex + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False + # make sure the fts is filled after the migration assert len(sql.SqlUser.get_ids_from_query('pierre')) == 1 @@ -1123,6 +1107,12 @@ def test_migration_21_users_ascii_name(): assert column_exists_in_table(cur, 'users', 'ascii_name') assert migration_level(cur) >= 21 + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True + call_command('cron') # first cron = reindex + assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False + assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False + # make sure the ascii_name is filled after the migration assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 1 diff --git a/wcs/publisher.py b/wcs/publisher.py index 1ebcbea0..2a1e49c6 100644 --- a/wcs/publisher.py +++ b/wcs/publisher.py @@ -298,6 +298,10 @@ class WcsPublisher(StubWcsPublisher): import sql sql.migrate() + def reindex_sql(self): + import sql + sql.reindex() + def cleanup(self): if self.is_using_postgresql(): import sql diff --git a/wcs/qommon/cron.py b/wcs/qommon/cron.py index 15a46366..6af321ba 100644 --- a/wcs/qommon/cron.py +++ b/wcs/qommon/cron.py @@ -35,6 +35,11 @@ def cron_worker(publisher, now): publisher.set_config() except: return + + # reindex user and formdata if needed (should only be run once) + if publisher.is_using_postgresql(): + publisher.reindex_sql() + for job in publisher.cronjobs: if job.days and now[2] not in job.days: continue diff --git a/wcs/sql.py b/wcs/sql.py index 69e42a8a..10fa737f 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -2102,6 +2102,31 @@ def get_sql_level(conn, cur): sql_level = int(cur.fetchone()[0]) return sql_level +@guard_postgres +def is_reindex_needed(index, conn, cur): + do_meta_table(conn, cur, insert_current_sql_level=False) + key_name = 'reindex_%s' % index + cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, )) + row = cur.fetchone() + if row is None: + cur.execute('''INSERT INTO wcs_meta (id, key, value) + VALUES (DEFAULT, %s, %s)''', (key_name, 'no')) + return False + return row[0] == 'needed' + +@guard_postgres +def set_reindex(index, value, conn, cur): + do_meta_table(conn, cur, insert_current_sql_level=False) + key_name = 'reindex_%s' % index + cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, )) + row = cur.fetchone() + if row is None: + cur.execute('''INSERT INTO wcs_meta (id, key, value) + VALUES (DEFAULT, %s, %s)''', (key_name, value)) + else: + cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', ( + value, key_name)) + def migrate_views(conn, cur): drop_views(None, conn, cur) from wcs.formdef import FormDef @@ -2151,18 +2176,13 @@ def migrate(): # 12: (second part), store fts in existing rows # 21: (second part), store ascii_name of users # 23: (first part), use misc.simplify() over full text queries - for user_id in SqlUser.keys(): - SqlUser.get(user_id).store() + set_reindex('user', 'needed', conn=conn, cur=cur) if sql_level < 23: # 17: store last_update_time in tables # 18: add user name to full-text search index # 21: (third part), add user ascii_names to full-text index # 23: (second part) use misc.simplify() over full text queries - # load and store all formdatas - from wcs.formdef import FormDef - for formdef in FormDef.select(): - for formdata in formdef.data_class().select(): - formdata.store() + set_reindex('formdata', 'needed', conn=conn, cur=cur) if sql_level < 24: from wcs.formdef import FormDef # 24: add index on evolution(formdata_id) @@ -2176,3 +2196,24 @@ def migrate(): conn.commit() cur.close() + + +@guard_postgres +def reindex(): + conn, cur = get_connection_and_cursor() + + if is_reindex_needed('user', conn=conn, cur=cur): + for user_id in SqlUser.keys(): + SqlUser.get(user_id).store() + set_reindex('user', 'done', conn=conn, cur=cur) + + if is_reindex_needed('formdata', conn=conn, cur=cur): + # load and store all formdatas + from wcs.formdef import FormDef + for formdef in FormDef.select(): + for formdata in formdef.data_class().select(): + formdata.store() + set_reindex('formdata', 'done', conn=conn, cur=cur) + + conn.commit() + cur.close() -- 2.17.0