0001-sql-delay-re-index-operations-on-first-cron-22383.patch

Thomas Noël, 20 avril 2018 11:32

Voir les différences: en ligne côte à côte

Subject: [PATCH] sql: delay re-index operations on first cron (#22383)

 tests/test_sql.py  | 44 ++++++++++++++-----------------------
 wcs/publisher.py   |  4 ++++
 wcs/qommon/cron.py |  5 +++++
 wcs/sql.py         | 55 ++++++++++++++++++++++++++++++++++++++++------
 4 files changed, 74 insertions(+), 34 deletions(-)

     import sys
     import time
     from django.core.management import call_command
     from quixote import cleanup
     from wcs import formdef, publisher, fields
-...
     from wcs import sql
     import wcs.qommon.storage as st
     from utilities import create_temporary_pub
     from utilities import create_temporary_pub, clean_temporary_pub
     import pytest
     postgresql = pytest.mark.postgresql
-...
         cleanup()
         pub = create_temporary_pub()
         pub.user_class = sql.SqlUser
         pub.is_using_postgresql = lambda: True
         conn = psycopg2.connect(user=os.environ['USER'])
         conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
         cur = conn.cursor()
         dbname = 'wcstests%d' % random.randint(0, 100000)
         cur.execute('CREATE DATABASE %s' % dbname)
         cur.close()
         pub.cfg['postgresql'] = {'database': dbname, 'user': os.environ['USER']}
         pub.initialize_sql()
         pub = create_temporary_pub(sql_mode=True)
         formdef = formdef.FormDef()
         formdef.name = 'tests'
-...
+        ]
         formdef.store()
         conn.close()
     def teardown_module(module):
         shutil.rmtree(pub.APP_DIR)
         if hasattr(pub, 'pgconn') and pub.pgconn:
             pub.pgconn.close()
         conn = psycopg2.connect(user=os.environ['USER'])
         conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
         cur = conn.cursor()
         cur.execute('DROP DATABASE %s' % pub.cfg['postgresql']['database'])
         cur.close()
         clean_temporary_pub()
     @postgresql
     def test_sql_table_name_invalid_chars():
-...
         assert column_exists_in_table(cur, 'users', 'fts')
         assert migration_level(cur) >= 12
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
         call_command('cron')  # first cron = reindex
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
         # make sure the fts is filled after the migration
         assert len(sql.SqlUser.get_ids_from_query('pierre')) == 1
-...
         assert column_exists_in_table(cur, 'users', 'ascii_name')
         assert migration_level(cur) >= 21
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
         call_command('cron')  # first cron = reindex
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
         # make sure the ascii_name is filled after the migration
         assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 1

             import sql
             sql.migrate()
         def reindex_sql(self):
             import sql
             sql.reindex()
         def cleanup(self):
             if self.is_using_postgresql():
                 import sql

             publisher.set_config()
         except:
             return
         # reindex user and formdata if needed (should only be run once)
         if publisher.is_using_postgresql():
             publisher.reindex_sql()
         for job in publisher.cronjobs:
             if job.days and now[2] not in job.days:
                 continue

         sql_level = int(cur.fetchone()[0])
         return sql_level
     @guard_postgres
     def is_reindex_needed(index, conn, cur):
         do_meta_table(conn, cur, insert_current_sql_level=False)
         key_name = 'reindex_%s' % index
         cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
         row = cur.fetchone()
         if row is None:
             cur.execute('''INSERT INTO wcs_meta (id, key, value)
                            VALUES (DEFAULT, %s, %s)''', (key_name, 'no'))
             return False
         return row[0] == 'needed'
     @guard_postgres
     def set_reindex(index, value, conn, cur):
         do_meta_table(conn, cur, insert_current_sql_level=False)
         key_name = 'reindex_%s' % index
         cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
         row = cur.fetchone()
         if row is None:
             cur.execute('''INSERT INTO wcs_meta (id, key, value)
                            VALUES (DEFAULT, %s, %s)''', (key_name, value))
         else:
             cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', (
                         value, key_name))
     def migrate_views(conn, cur):
         drop_views(None, conn, cur)
         from wcs.formdef import FormDef
-...
             # 12: (second part), store fts in existing rows
             # 21: (second part), store ascii_name of users
             # 23: (first part), use misc.simplify() over full text queries
             for user_id in SqlUser.keys():
                 SqlUser.get(user_id).store()
             set_reindex('user', 'needed', conn=conn, cur=cur)
         if sql_level < 23:
             # 17: store last_update_time in tables
             # 18: add user name to full-text search index
             # 21: (third part), add user ascii_names to full-text index
             # 23: (second part) use misc.simplify() over full text queries
             # load and store all formdatas
             from wcs.formdef import FormDef
             for formdef in FormDef.select():
                 for formdata in formdef.data_class().select():
                     formdata.store()
             set_reindex('formdata', 'needed', conn=conn, cur=cur)
         if sql_level < 24:
             from wcs.formdef import FormDef
             # 24: add index on evolution(formdata_id)
-...
         conn.commit()
         cur.close()
     @guard_postgres
     def reindex():
         conn, cur = get_connection_and_cursor()
         if is_reindex_needed('user', conn=conn, cur=cur):
             for user_id in SqlUser.keys():
                 SqlUser.get(user_id).store()
             set_reindex('user', 'done', conn=conn, cur=cur)
         if is_reindex_needed('formdata', conn=conn, cur=cur):
             # load and store all formdatas
             from wcs.formdef import FormDef
             for formdef in FormDef.select():
                 for formdata in formdef.data_class().select():
                     formdata.store()
             set_reindex('formdata', 'done', conn=conn, cur=cur)
         conn.commit()
         cur.close()
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0001-sql-delay-re-index-operations-on-first-cron-22383.patch