0004-misc-migration-to-convert-python-datasources-72096.patch

Lauréline Guérin, 16 décembre 2022 18:35

Voir les différences: en ligne côte à côte

Subject: [PATCH 4/4] misc: migration to convert python datasources (#72096)

 tests/test_sql.py | 173 +++++++++++++++++++++++++++++++++++++++++++++-
 wcs/sql.py        |  77 ++++++++++++++++++++-
 2 files changed, 248 insertions(+), 2 deletions(-)

     import wcs.qommon.storage as st
     from wcs import fields, sql
     from wcs.blocks import BlockDef
     from wcs.carddef import CardDef
     from wcs.data_sources import NamedDataSource
     from wcs.formdata import Evolution
     from wcs.formdef import FormDef
     from wcs.qommon import force_str
     from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
     from wcs.workflows import Workflow, WorkflowCriticalityLevel
     from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowCriticalityLevel
     from .utilities import clean_temporary_pub, create_temporary_pub
-...
         lazy = sql.LazyEvolutionList(dump)
         assert list(pickle.loads(pickle.dumps(lazy))) == list(lazy)
     def test_python_datasource_migration(pub):
         FormDef.wipe()
         CardDef.wipe()
         Workflow.wipe()
         NamedDataSource.wipe()
         ds1 = {'type': 'formula', 'value': repr([('un'), ('deux')])}
         ds2 = {'type': 'formula', 'value': repr([('1', 'un'), ('2', 'deux')])}
         ds3 = {'type': 'formula', 'value': repr([('1', 'un', 'foo'), ('2', 'deux', 'bar')])}
         ds4 = {'type': 'formula', 'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])}
         failing_ds1 = {'type': 'formula', 'value': repr([{'id': '', 'text': 'un'}, {'id': '2', 'text': 'deux'}])}
         failing_ds2 = {'type': 'formula', 'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': ''}])}
         failing_ds3 = {'type': 'formula', 'value': 'xxx()'}
         for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
             formdef = FormDef()
             formdef.name = 'foobar %s' % i
             formdef.fields = [
                 fields.ItemField(id='1', label='item', data_source=ds),
+            ]
             formdef.store()
         for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
             carddef = CardDef()
             carddef.name = 'foobar %s' % i
             carddef.fields = [
                 fields.ItemField(id='1', label='item', data_source=ds),
+            ]
             carddef.store()
         for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
             wf = Workflow()
             wf.name = 'foobar %s' % i
             wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
             wf.backoffice_fields_formdef.fields = [
                 fields.ItemField(id='1', label='item', data_source=ds),
+            ]
             wf.store()
         for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
             data_source = NamedDataSource(name='foobar %s' % i)
             data_source.data_source = ds
             data_source.store()
             conn, cur = sql.get_connection_and_cursor()
         cur.execute('UPDATE wcs_meta SET value = 70 WHERE key = %s', ('sql_level',))
         conn.commit()
         cur.close()
         conn, cur = sql.get_connection_and_cursor()
         sql.migrate()
         assert sql.is_reindex_needed('python_ds_migration', conn=conn, cur=cur) is True
         sql.reindex()
         formdef = FormDef.get(1)
         assert formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+        }
         formdef = FormDef.get(2)
         assert formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         formdef = FormDef.get(3)
         assert formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+        }
         formdef = FormDef.get(4)
         assert formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         formdef = FormDef.get(5)
         assert formdef.fields[0].data_source == failing_ds1
         formdef = FormDef.get(6)
         assert formdef.fields[0].data_source == failing_ds2
         formdef = FormDef.get(7)
         assert formdef.fields[0].data_source == failing_ds3
         carddef = CardDef.get(1)
         assert carddef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+        }
         carddef = CardDef.get(2)
         assert carddef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         carddef = CardDef.get(3)
         assert carddef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+        }
         carddef = CardDef.get(4)
         assert carddef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         carddef = CardDef.get(5)
         assert carddef.fields[0].data_source == failing_ds1
         carddef = CardDef.get(6)
         assert carddef.fields[0].data_source == failing_ds2
         carddef = CardDef.get(7)
         assert carddef.fields[0].data_source == failing_ds3
         wf = Workflow.get(1)
         assert wf.backoffice_fields_formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+        }
         wf = Workflow.get(2)
         assert wf.backoffice_fields_formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         wf = Workflow.get(3)
         assert wf.backoffice_fields_formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+        }
         wf = Workflow.get(4)
         assert wf.backoffice_fields_formdef.fields[0].data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         wf = Workflow.get(5)
         assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds1
         wf = Workflow.get(6)
         assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds2
         wf = Workflow.get(7)
         assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds3
         data_source = NamedDataSource.get(1)
         assert data_source.data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+        }
         data_source = NamedDataSource.get(2)
         assert data_source.data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         data_source = NamedDataSource.get(3)
         assert data_source.data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+        }
         data_source = NamedDataSource.get(4)
         assert data_source.data_source == {
             'type': 'jsonvalue',
             'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+        }
         data_source = NamedDataSource.get(5)
         assert data_source.data_source == failing_ds1
         data_source = NamedDataSource.get(6)
         assert data_source.data_source == failing_ds2
         data_source = NamedDataSource.get(7)
         assert data_source.data_source == failing_ds3
         # check it's no longer needed afterwards
         sql.migrate()
         assert sql.is_reindex_needed('python_ds_migration', conn=conn, cur=cur) is False

     import copy
     import datetime
     import io
     import itertools
     import json
     import re
     import time
-...
     # latest migration, number + description (description is not used
     # programmaticaly but will make sure git conflicts if two migrations are
     # separately added with the same number)
     SQL_LEVEL = (70, 'repair anonymisation')
     SQL_LEVEL = (71, 'python datasource migration')
     def migrate_global_views(conn, cur):
-...
             for formdef in FormDef.select():
                 do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         if sql_level < 71:
             # 71: python datasource migration
             set_reindex('python_ds_migration', 'needed', conn=conn, cur=cur)
         if sql_level != SQL_LEVEL[0]:
             cur.execute(
                 '''UPDATE wcs_meta SET value = %s, updated_at=NOW() WHERE key = %s''',
-...
                         print('error reindexing %s (%r)' % (formdata, e))
             set_reindex('formdata', 'done', conn=conn, cur=cur)
         if is_reindex_needed('python_ds_migration', conn=conn, cur=cur):
             # migrate python datasource
             def migrate_value(value):
                 try:
                     value = eval(value)
                 except Exception:
                     return
                 if len(value) == 0:
                     return []
                 if isinstance(value[0], (list, tuple)):
                     if len(value[0]) >= 3:
                         return [{'id': x[0], 'text': x[1], 'key': x[2]} for x in value]
                     elif len(value[0]) == 2:
                         return [{'id': x[0], 'text': x[1]} for x in value]
                     elif len(value[0]) == 1:
                         return [{'id': x[0], 'text': x[0]} for x in value]
                     return value
                 elif isinstance(value[0], str):
                     return [{'id': x, 'text': x} for x in value]
                 elif isinstance(value[0], dict):
                     if all(str(x.get('id', '')) and x.get('text') for x in value):
                         return value
             def migrate_field(field):
                 if not getattr(field, 'data_source', None):
                     return
                 data_source_type = field.data_source.get('type')
                 if data_source_type != 'formula':
                     return
                 value = migrate_value(field.data_source.get('value'))
                 if value is not None:
                     field.data_source['type'] = 'jsonvalue'
                     field.data_source['value'] = json.dumps(value)
                     return True
             for formdef in itertools.chain(FormDef.select(), CardDef.select()):
                 changed = False
                 for field in formdef.fields or []:
                     changed |= bool(migrate_field(field))
                 if changed:
                     formdef.store(comment=_('Automatic update'), snapshot_store_user=False)
             from wcs.workflows import Workflow
             for workflow in Workflow.select():
                 changed = False
                 if workflow.backoffice_fields_formdef and workflow.backoffice_fields_formdef.fields:
                     for field in workflow.backoffice_fields_formdef.fields:
                         changed |= bool(migrate_field(field))
                 if changed:
                     workflow.store(
                         migration_update=True, comment=_('Automatic update'), snapshot_store_user=False
+                    )
             from wcs.data_sources import NamedDataSource
             for datasource in NamedDataSource.select():
                 data_source = datasource.data_source
                 if not data_source.get('type') == 'formula':
                     continue
                 value = migrate_value(data_source.get('value'))
                 if value is not None:
                     datasource.data_source['type'] = 'jsonvalue'
                     datasource.data_source['value'] = json.dumps(value)
                     datasource.store(comment=_('Automatic update'), snapshot_store_user=False)
             set_reindex('python_ds_migration', 'done', conn=conn, cur=cur)
         conn.commit()
         cur.close()
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0004-misc-migration-to-convert-python-datasources-72096.patch