0002-sql-delay-re-index-operations-on-first-cron-22383.patch

Thomas Noël, 20 avril 2018 14:04

Voir les différences: en ligne côte à côte

Subject: [PATCH 2/2] sql: delay re-index operations on first cron (#22383)

 tests/test_sql.py  | 20 +++++++++++++++++
 wcs/publisher.py   |  4 ++++
 wcs/qommon/cron.py |  5 +++++
 wcs/sql.py         | 55 ++++++++++++++++++++++++++++++++++++++++------
 4 files changed, 77 insertions(+), 7 deletions(-)

     import sys
     import time
     from django.core.management import call_command
     from quixote import cleanup
     from wcs import formdef, publisher, fields
-...
         assert column_exists_in_table(cur, 'users', 'fts')
         assert migration_level(cur) >= 12
         # no fts, migration only prepare re-index
         assert len(sql.SqlUser.get_ids_from_query('pierre')) == 0
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
         call_command('cron')  # first cron = reindex
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
         # make sure the fts is filled after the migration
         assert len(sql.SqlUser.get_ids_from_query('pierre')) == 1
-...
         assert column_exists_in_table(cur, 'users', 'ascii_name')
         assert migration_level(cur) >= 21
         # no fts, migration only prepare re-index
         assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 0
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is True
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is True
         call_command('cron')  # first cron = reindex
         assert sql.is_reindex_needed('user', conn=conn, cur=cur) is False
         assert sql.is_reindex_needed('formdata', conn=conn, cur=cur) is False
         # make sure the ascii_name is filled after the migration
         assert sql.SqlUser.count([st.Equal('ascii_name', 'jean senisme')]) == 1

             import sql
             sql.migrate()
         def reindex_sql(self):
             import sql
             sql.reindex()
         def cleanup(self):
             if self.is_using_postgresql():
                 import sql

             publisher.set_config()
         except:
             return
         # reindex user and formdata if needed (should only be run once)
         if publisher.is_using_postgresql():
             publisher.reindex_sql()
         for job in publisher.cronjobs:
             if job.days and now[2] not in job.days:
                 continue

         sql_level = int(cur.fetchone()[0])
         return sql_level
     @guard_postgres
     def is_reindex_needed(index, conn, cur):
         do_meta_table(conn, cur, insert_current_sql_level=False)
         key_name = 'reindex_%s' % index
         cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
         row = cur.fetchone()
         if row is None:
             cur.execute('''INSERT INTO wcs_meta (id, key, value)
                            VALUES (DEFAULT, %s, %s)''', (key_name, 'no'))
             return False
         return row[0] == 'needed'
     @guard_postgres
     def set_reindex(index, value, conn, cur):
         do_meta_table(conn, cur, insert_current_sql_level=False)
         key_name = 'reindex_%s' % index
         cur.execute('''SELECT value FROM wcs_meta WHERE key = %s''', (key_name, ))
         row = cur.fetchone()
         if row is None:
             cur.execute('''INSERT INTO wcs_meta (id, key, value)
                            VALUES (DEFAULT, %s, %s)''', (key_name, value))
         else:
             cur.execute('''UPDATE wcs_meta SET value = %s WHERE key = %s''', (
                         value, key_name))
     def migrate_views(conn, cur):
         drop_views(None, conn, cur)
         from wcs.formdef import FormDef
-...
             # 12: (second part), store fts in existing rows
             # 21: (second part), store ascii_name of users
             # 23: (first part), use misc.simplify() over full text queries
             for user_id in SqlUser.keys():
                 SqlUser.get(user_id).store()
             set_reindex('user', 'needed', conn=conn, cur=cur)
         if sql_level < 23:
             # 17: store last_update_time in tables
             # 18: add user name to full-text search index
             # 21: (third part), add user ascii_names to full-text index
             # 23: (second part) use misc.simplify() over full text queries
             # load and store all formdatas
             from wcs.formdef import FormDef
             for formdef in FormDef.select():
                 for formdata in formdef.data_class().select():
                     formdata.store()
             set_reindex('formdata', 'needed', conn=conn, cur=cur)
         if sql_level < 24:
             from wcs.formdef import FormDef
             # 24: add index on evolution(formdata_id)
-...
         conn.commit()
         cur.close()
     @guard_postgres
     def reindex():
         conn, cur = get_connection_and_cursor()
         if is_reindex_needed('user', conn=conn, cur=cur):
             for user in SqlUser.select(iterator=True):
                 user.store()
             set_reindex('user', 'done', conn=conn, cur=cur)
         if is_reindex_needed('formdata', conn=conn, cur=cur):
             # load and store all formdatas
             from wcs.formdef import FormDef
             for formdef in FormDef.select():
                 for formdata in formdef.data_class().select(iterator=True):
                     formdata.store()
             set_reindex('formdata', 'done', conn=conn, cur=cur)
         conn.commit()
         cur.close()
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0002-sql-delay-re-index-operations-on-first-cron-22383.patch