From 8e66078eacb753d5d97942bdef624e1b8a5afa0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 16 Dec 2022 16:48:35 +0100
Subject: [PATCH 4/4] misc: migration to convert python datasources (#72096)

---
 tests/test_sql.py | 173 +++++++++++++++++++++++++++++++++++++++++++++-
 wcs/sql.py        |  77 ++++++++++++++++++++-
 2 files changed, 248 insertions(+), 2 deletions(-)

diff --git a/tests/test_sql.py b/tests/test_sql.py
index f80a82b6f..526dcfba9 100644
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -12,11 +12,13 @@ import pytest
 import wcs.qommon.storage as st
 from wcs import fields, sql
 from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
+from wcs.data_sources import NamedDataSource
 from wcs.formdata import Evolution
 from wcs.formdef import FormDef
 from wcs.qommon import force_str
 from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
-from wcs.workflows import Workflow, WorkflowCriticalityLevel
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef, WorkflowCriticalityLevel
 
 from .utilities import clean_temporary_pub, create_temporary_pub
 
@@ -2278,3 +2280,172 @@ def test_lazyevolutionlist():
 
     lazy = sql.LazyEvolutionList(dump)
     assert list(pickle.loads(pickle.dumps(lazy))) == list(lazy)
+
+
+def test_python_datasource_migration(pub):
+    FormDef.wipe()
+    CardDef.wipe()
+    Workflow.wipe()
+    NamedDataSource.wipe()
+
+    ds1 = {'type': 'formula', 'value': repr([('un'), ('deux')])}
+    ds2 = {'type': 'formula', 'value': repr([('1', 'un'), ('2', 'deux')])}
+    ds3 = {'type': 'formula', 'value': repr([('1', 'un', 'foo'), ('2', 'deux', 'bar')])}
+    ds4 = {'type': 'formula', 'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': 'deux'}])}
+    failing_ds1 = {'type': 'formula', 'value': repr([{'id': '', 'text': 'un'}, {'id': '2', 'text': 'deux'}])}
+    failing_ds2 = {'type': 'formula', 'value': repr([{'id': '1', 'text': 'un'}, {'id': '2', 'text': ''}])}
+    failing_ds3 = {'type': 'formula', 'value': 'xxx()'}
+
+    for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
+        formdef = FormDef()
+        formdef.name = 'foobar %s' % i
+        formdef.fields = [
+            fields.ItemField(id='1', label='item', data_source=ds),
+        ]
+        formdef.store()
+
+    for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
+        carddef = CardDef()
+        carddef.name = 'foobar %s' % i
+        carddef.fields = [
+            fields.ItemField(id='1', label='item', data_source=ds),
+        ]
+        carddef.store()
+
+    for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
+        wf = Workflow()
+        wf.name = 'foobar %s' % i
+        wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+        wf.backoffice_fields_formdef.fields = [
+            fields.ItemField(id='1', label='item', data_source=ds),
+        ]
+        wf.store()
+
+    for i, ds in enumerate([ds1, ds2, ds3, ds4, failing_ds1, failing_ds2, failing_ds3]):
+        data_source = NamedDataSource(name='foobar %s' % i)
+        data_source.data_source = ds
+        data_source.store()
+
+        conn, cur = sql.get_connection_and_cursor()
+
+    cur.execute('UPDATE wcs_meta SET value = 70 WHERE key = %s', ('sql_level',))
+    conn.commit()
+    cur.close()
+
+    conn, cur = sql.get_connection_and_cursor()
+    sql.migrate()
+
+    assert sql.is_reindex_needed('python_ds_migration', conn=conn, cur=cur) is True
+    sql.reindex()
+
+    formdef = FormDef.get(1)
+    assert formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+    }
+    formdef = FormDef.get(2)
+    assert formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    formdef = FormDef.get(3)
+    assert formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+    }
+    formdef = FormDef.get(4)
+    assert formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    formdef = FormDef.get(5)
+    assert formdef.fields[0].data_source == failing_ds1
+    formdef = FormDef.get(6)
+    assert formdef.fields[0].data_source == failing_ds2
+    formdef = FormDef.get(7)
+    assert formdef.fields[0].data_source == failing_ds3
+
+    carddef = CardDef.get(1)
+    assert carddef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+    }
+    carddef = CardDef.get(2)
+    assert carddef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    carddef = CardDef.get(3)
+    assert carddef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+    }
+    carddef = CardDef.get(4)
+    assert carddef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    carddef = CardDef.get(5)
+    assert carddef.fields[0].data_source == failing_ds1
+    carddef = CardDef.get(6)
+    assert carddef.fields[0].data_source == failing_ds2
+    carddef = CardDef.get(7)
+    assert carddef.fields[0].data_source == failing_ds3
+
+    wf = Workflow.get(1)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+    }
+    wf = Workflow.get(2)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    wf = Workflow.get(3)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+    }
+    wf = Workflow.get(4)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    wf = Workflow.get(5)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds1
+    wf = Workflow.get(6)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds2
+    wf = Workflow.get(7)
+    assert wf.backoffice_fields_formdef.fields[0].data_source == failing_ds3
+
+    data_source = NamedDataSource.get(1)
+    assert data_source.data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "un", "text": "un"}, {"id": "deux", "text": "deux"}]',
+    }
+    data_source = NamedDataSource.get(2)
+    assert data_source.data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    data_source = NamedDataSource.get(3)
+    assert data_source.data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un", "key": "foo"}, {"id": "2", "text": "deux", "key": "bar"}]',
+    }
+    data_source = NamedDataSource.get(4)
+    assert data_source.data_source == {
+        'type': 'jsonvalue',
+        'value': '[{"id": "1", "text": "un"}, {"id": "2", "text": "deux"}]',
+    }
+    data_source = NamedDataSource.get(5)
+    assert data_source.data_source == failing_ds1
+    data_source = NamedDataSource.get(6)
+    assert data_source.data_source == failing_ds2
+    data_source = NamedDataSource.get(7)
+    assert data_source.data_source == failing_ds3
+
+    # check it's no longer needed afterwards
+    sql.migrate()
+    assert sql.is_reindex_needed('python_ds_migration', conn=conn, cur=cur) is False
diff --git a/wcs/sql.py b/wcs/sql.py
index 8ac8477a6..ea8c018aa 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -17,6 +17,7 @@
 import copy
 import datetime
 import io
+import itertools
 import json
 import re
 import time
@@ -4659,7 +4660,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (70, 'repair anonymisation')
+SQL_LEVEL = (71, 'python datasource migration')
 
 
 def migrate_global_views(conn, cur):
@@ -4943,6 +4944,10 @@ def migrate():
         for formdef in FormDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
 
+    if sql_level < 71:
+        # 71: python datasource migration
+        set_reindex('python_ds_migration', 'needed', conn=conn, cur=cur)
+
     if sql_level != SQL_LEVEL[0]:
         cur.execute(
             '''UPDATE wcs_meta SET value = %s, updated_at=NOW() WHERE key = %s''',
@@ -4976,6 +4981,76 @@ def reindex():
                     print('error reindexing %s (%r)' % (formdata, e))
         set_reindex('formdata', 'done', conn=conn, cur=cur)
 
+    if is_reindex_needed('python_ds_migration', conn=conn, cur=cur):
+        # migrate python datasource
+
+        def migrate_value(value):
+            try:
+                value = eval(value)
+            except Exception:
+                return
+            if len(value) == 0:
+                return []
+            if isinstance(value[0], (list, tuple)):
+                if len(value[0]) >= 3:
+                    return [{'id': x[0], 'text': x[1], 'key': x[2]} for x in value]
+                elif len(value[0]) == 2:
+                    return [{'id': x[0], 'text': x[1]} for x in value]
+                elif len(value[0]) == 1:
+                    return [{'id': x[0], 'text': x[0]} for x in value]
+                return value
+            elif isinstance(value[0], str):
+                return [{'id': x, 'text': x} for x in value]
+            elif isinstance(value[0], dict):
+                if all(str(x.get('id', '')) and x.get('text') for x in value):
+                    return value
+
+        def migrate_field(field):
+            if not getattr(field, 'data_source', None):
+                return
+            data_source_type = field.data_source.get('type')
+            if data_source_type != 'formula':
+                return
+            value = migrate_value(field.data_source.get('value'))
+
+            if value is not None:
+                field.data_source['type'] = 'jsonvalue'
+                field.data_source['value'] = json.dumps(value)
+                return True
+
+        for formdef in itertools.chain(FormDef.select(), CardDef.select()):
+            changed = False
+            for field in formdef.fields or []:
+                changed |= bool(migrate_field(field))
+            if changed:
+                formdef.store(comment=_('Automatic update'), snapshot_store_user=False)
+
+        from wcs.workflows import Workflow
+
+        for workflow in Workflow.select():
+            changed = False
+            if workflow.backoffice_fields_formdef and workflow.backoffice_fields_formdef.fields:
+                for field in workflow.backoffice_fields_formdef.fields:
+                    changed |= bool(migrate_field(field))
+            if changed:
+                workflow.store(
+                    migration_update=True, comment=_('Automatic update'), snapshot_store_user=False
+                )
+
+        from wcs.data_sources import NamedDataSource
+
+        for datasource in NamedDataSource.select():
+            data_source = datasource.data_source
+            if not data_source.get('type') == 'formula':
+                continue
+            value = migrate_value(data_source.get('value'))
+            if value is not None:
+                datasource.data_source['type'] = 'jsonvalue'
+                datasource.data_source['value'] = json.dumps(value)
+                datasource.store(comment=_('Automatic update'), snapshot_store_user=False)
+
+        set_reindex('python_ds_migration', 'done', conn=conn, cur=cur)
+
     conn.commit()
     cur.close()
 
-- 
2.35.1

