From 8c0161edabf54274a1844a0958dccc80751032ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 16 Dec 2022 11:43:11 +0100
Subject: [PATCH 1/4] datasources: new datasource type, jsonvalue, to replace
 formula (#72096)

---
 tests/admin_pages/test_datasource.py          |   2 +
 tests/admin_pages/test_form.py                |   7 +
 tests/test_datasource.py                      | 144 ++++++++++++++++++
 wcs/admin/data_sources.py                     |   4 +-
 wcs/data_sources.py                           |  44 +++++-
 wcs/fields.py                                 |   1 +
 wcs/templates/wcs/backoffice/data-source.html |   2 +
 7 files changed, 196 insertions(+), 8 deletions(-)

diff --git a/tests/admin_pages/test_datasource.py b/tests/admin_pages/test_datasource.py
index 9a3dee7d4..4d87500bc 100644
--- a/tests/admin_pages/test_datasource.py
+++ b/tests/admin_pages/test_datasource.py
@@ -235,6 +235,7 @@ def test_data_sources_type_options(pub):
         ('jsonp', False, 'JSONP URL'),
         ('geojson', False, 'GeoJSON URL'),
         ('python', False, 'Python Expression'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
 
     pub.site_options.set('options', 'disable-python-expressions', 'true')
@@ -247,6 +248,7 @@ def test_data_sources_type_options(pub):
         ('json', False, 'JSON URL'),
         ('jsonp', False, 'JSONP URL'),
         ('geojson', False, 'GeoJSON URL'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
 
 
diff --git a/tests/admin_pages/test_form.py b/tests/admin_pages/test_form.py
index b5da58d73..7765429bb 100644
--- a/tests/admin_pages/test_form.py
+++ b/tests/admin_pages/test_form.py
@@ -2101,6 +2101,7 @@ def test_form_edit_item_field_data_source(pub):
     create_superuser(pub)
     create_role(pub)
 
+    NamedDataSource.wipe()
     FormDef.wipe()
     formdef = FormDef()
     formdef.name = 'form title'
@@ -2114,6 +2115,7 @@ def test_form_edit_item_field_data_source(pub):
         ('json', False, 'JSON URL'),
         ('jsonp', False, 'JSONP URL'),
         ('python', False, 'Python Expression'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
     resp = resp.form.submit('submit').follow()
 
@@ -2130,6 +2132,7 @@ def test_form_edit_item_field_data_source(pub):
         ('json', False, 'JSON URL'),
         ('jsonp', False, 'JSONP URL'),
         ('python', False, 'Python Expression'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
     resp.form['data_mode'].value = 'data-source'
     resp.form['data_source$type'].value = 'foobar'
@@ -2149,6 +2152,7 @@ def test_form_edit_item_field_data_source(pub):
         ('json', False, 'JSON URL'),
         ('jsonp', False, 'JSONP URL'),
         ('python', False, 'Python Expression'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
 
     carddef.digest_templates = {'default': 'plop'}
@@ -2168,6 +2172,7 @@ def test_form_edit_item_field_data_source(pub):
         ('json', False, 'JSON URL'),
         ('jsonp', False, 'JSONP URL'),
         ('python', False, 'Python Expression'),
+        ('jsonvalue', False, 'JSON Expression'),
     ]
     assert (
         resp.pyquery('select#form_data_source__type option')[1].attrib['data-goto-url']
@@ -2231,6 +2236,7 @@ def test_form_edit_item_field_data_source_with_categories(pub):
         'json',
         'jsonp',
         'python',
+        'jsonvalue',
     ]
 
     cat_b = DataSourceCategory(name='Cat B')
@@ -2257,6 +2263,7 @@ def test_form_edit_item_field_data_source_with_categories(pub):
         'json',
         'jsonp',
         'python',
+        'jsonvalue',
     ]
 
 
diff --git a/tests/test_datasource.py b/tests/test_datasource.py
index 0e7cf17a9..7ae1d04be 100644
--- a/tests/test_datasource.py
+++ b/tests/test_datasource.py
@@ -1,4 +1,5 @@
 import codecs
+import json
 import os
 import urllib.parse
 import xml.etree.ElementTree as ET
@@ -216,6 +217,149 @@ def test_python_datasource_with_evalutils(pub):
     ]
 
 
+def test_item_field_jsonvalue_datasource(requests_pub):
+    req = get_request()
+    req.environ['REQUEST_METHOD'] = 'POST'
+    field = fields.ItemField()
+    field.id = 1
+    field.data_source = {
+        'type': 'jsonvalue',
+        'value': json.dumps([{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}]),
+    }
+    form = Form()
+    field.add_to_form(form)
+    widget = form.get_widget('f1')
+    assert widget is not None
+    assert widget.options == [('1', 'un', '1'), ('2', 'deux', '2')]
+
+    form = MockHtmlForm(widget)
+    mock_form_submission(req, widget, {'f1': ['1']})
+    assert widget.parse() == '1'
+
+    form = Form()
+    field.add_to_view_form(form, value='1')
+    widget = form.get_widget('f1')
+
+    form = MockHtmlForm(widget)
+    mock_form_submission(req, widget)
+    assert widget.parse() == '1'
+
+
+def test_jsonvalue_datasource(pub):
+    plain_list = [{"id": "1", "text": "foo"}, {"id": "2", "text": "bar"}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list)}
+    assert data_sources.get_items(datasource) == [
+        ('1', 'foo', '1', {'id': '1', 'text': 'foo'}),
+        ('2', 'bar', '2', {'id': '2', 'text': 'bar'}),
+    ]
+    assert data_sources.get_structured_items(datasource) == [
+        {'id': '1', 'text': 'foo'},
+        {'id': '2', 'text': 'bar'},
+    ]
+
+    # with key
+    plain_list = [{"id": "1", "text": "foo", "key": "a"}, {"id": "2", "text": "bar", "key": "b"}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list)}
+    assert data_sources.get_items(datasource) == [
+        ('1', 'foo', 'a', {'id': '1', 'key': 'a', 'text': 'foo'}),
+        ('2', 'bar', 'b', {'id': '2', 'key': 'b', 'text': 'bar'}),
+    ]
+
+
+def test_jsonvalue_datasource_errors(pub):
+    # not a list
+    datasource = {'type': 'jsonvalue', 'value': 'foobar', 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert logged_error.summary == "[DATASOURCE] JSON data source ('foobar') gave a non usable result"
+
+    pub.loggederror_class.wipe()
+    plain_list = {'foo': 'bar'}
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('{\"foo\": \"bar\"}') gave a non usable result"
+    )
+
+    # not a list of dict
+    pub.loggederror_class.wipe()
+    plain_list = ["foobar"]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert logged_error.summary == "[DATASOURCE] JSON data source ('[\"foobar\"]') gave a non usable result"
+
+    pub.loggederror_class.wipe()
+    plain_list = [{'foo': 'bar'}, "foobar"]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('[{\"foo\": \"bar\"}, \"foobar\"]') gave a non usable result"
+    )
+
+    # no id found
+    pub.loggederror_class.wipe()
+    plain_list = [{"text": "foo"}, {"id": "2", "text": "bar"}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('[{\"text\": \"foo\"}, {\"id\": \"2\", \"text\": \"bar\"}]') gave a non usable result"
+    )
+
+    pub.loggederror_class.wipe()
+    plain_list = [{"id": "1", "text": "foo"}, {"id": "", "text": "bar"}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('[{\"id\": \"1\", \"text\": \"foo\"}, {\"id\": \"\", \"text\": \"bar\"}]') gave a non usable result"
+    )
+
+    # no text found
+    pub.loggederror_class.wipe()
+    plain_list = [{"id": "1"}, {"id": "2", "text": "bar"}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('[{\"id\": \"1\"}, {\"id\": \"2\", \"text\": \"bar\"}]') gave a non usable result"
+    )
+
+    pub.loggederror_class.wipe()
+    plain_list = [{"id": "1", "text": "foo"}, {"id": "2", "text": ""}]
+    datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
+    assert data_sources.get_items(datasource) == []
+    assert pub.loggederror_class.count() == 1
+    logged_error = pub.loggederror_class.select()[0]
+    assert logged_error.workflow_id is None
+    assert (
+        logged_error.summary
+        == "[DATASOURCE] JSON data source ('[{\"id\": \"1\", \"text\": \"foo\"}, {\"id\": \"2\", \"text\": \"\"}]') gave a non usable result"
+    )
+
+
 def test_json_datasource(pub, requests_pub):
     get_request().datasources_cache = {}
     datasource = {'type': 'json', 'value': ''}
diff --git a/wcs/admin/data_sources.py b/wcs/admin/data_sources.py
index 05fff8827..9fd9c61b9 100644
--- a/wcs/admin/data_sources.py
+++ b/wcs/admin/data_sources.py
@@ -88,7 +88,7 @@ class NamedDataSourceUI:
                 'data_source',
                 value=self.datasource.data_source,
                 title=_('Data Source'),
-                allowed_source_types={'json', 'jsonp', 'geojson', 'python'},
+                allowed_source_types={'json', 'jsonp', 'geojson', 'python', 'jsonvalue'},
                 required=True,
             )
         form.add(
@@ -371,7 +371,7 @@ class NamedDataSourcePage(Directory):
             data_source = self.datasource.extended_data_source
         finally:
             get_request().disable_error_notifications = False
-        if data_source.get('type') not in ('json', 'geojson', 'formula', 'wcs:users'):
+        if data_source.get('type') not in ('json', 'geojson', 'formula', 'jsonvalue', 'wcs:users'):
             return ''
         try:
             items = get_structured_items(data_source)
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index 2b708a528..da31071c1 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -17,6 +17,7 @@
 import collections
 import collections.abc
 import hashlib
+import json
 import urllib.parse
 import xml.etree.ElementTree as ET
 
@@ -57,7 +58,7 @@ def register_data_source_function(function, function_name=None):
 class DataSourceSelectionWidget(CompositeWidget):
     def __init__(self, name, value=None, allowed_source_types=None, disallowed_source_types=None, **kwargs):
         if allowed_source_types is None:
-            allowed_source_types = {'json', 'jsonp', 'geojson', 'named', 'cards', 'python'}
+            allowed_source_types = {'json', 'jsonp', 'geojson', 'named', 'cards', 'python', 'jsonvalue'}
         if get_publisher().has_site_option('disable-python-expressions') and 'python' in allowed_source_types:
             allowed_source_types.remove('python')
         if get_publisher().has_site_option('disable-jsonp-sources') and 'jsonp' in allowed_source_types:
@@ -161,6 +162,8 @@ class DataSourceSelectionWidget(CompositeWidget):
             generic_options.append(('formula', _('Python Expression'), 'python'))
         elif value.get('type') == 'formula':
             generic_options.append(('formula', _('Python Expression (deprecated)'), 'python'))
+        if 'jsonvalue' in allowed_source_types:
+            generic_options.append(('jsonvalue', _('JSON Expression'), 'jsonvalue'))
 
         if len(options) > 1 and generic_options:
             options.append(OptGroup(_('Generic Data Sources')))
@@ -185,7 +188,7 @@ class DataSourceSelectionWidget(CompositeWidget):
             size=80,
             attrs={
                 'data-dynamic-display-child-of': 'data_source$type',
-                'data-dynamic-display-value-in': 'json|jsonp|geojson|python',
+                'data-dynamic-display-value-in': 'json|jsonp|geojson|python|jsonvalue',
             },
         )
 
@@ -368,7 +371,7 @@ def _get_structured_items(data_source, mode=None, raise_on_error=False):
 
         return CardDef.get_data_source_items(data_source['type'])
 
-    if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula', 'wcs:users'):
+    if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue', 'wcs:users'):
         # named data source
         named_data_source = NamedDataSource.get_by_slug(data_source['type'])
         if named_data_source.cache_duration:
@@ -385,6 +388,34 @@ def _get_structured_items(data_source, mode=None, raise_on_error=False):
         include_disabled_users = data_source.get('include_disabled_users')
         return [{'id': u.id, 'text': u.name} for u in users if u.is_active or include_disabled_users]
 
+    if data_source.get('type') == 'jsonvalue':
+        try:
+            value = json.loads(data_source.get('value'))
+        except json.JSONDecodeError:
+            get_publisher().record_error(
+                'JSON data source (%r) gave a non usable result' % data_source.get('value'),
+                context='[DATASOURCE]',
+                notify=data_source.get('notify_on_errors'),
+                record=data_source.get('record_on_errors'),
+            )
+            return []
+        try:
+            if not isinstance(value, list):
+                raise ValueError
+            for item in value:
+                if not isinstance(item, dict):
+                    raise ValueError
+            if all(str(x.get('id', '')) and x.get('text') for x in value):
+                return value
+            raise ValueError
+        except ValueError:
+            get_publisher().record_error(
+                'JSON data source (%r) gave a non usable result' % data_source.get('value'),
+                context='[DATASOURCE]',
+                notify=data_source.get('notify_on_errors'),
+                record=data_source.get('record_on_errors'),
+            )
+            return []
     if data_source.get('type') == 'formula':
         # the result of a python expression, it must be a list.
         # - of strings
@@ -510,7 +541,7 @@ def get_real(data_source):
     if not data_source:
         return None
     ds_type = data_source.get('type')
-    if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
+    if ds_type in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue'):
         return data_source
     if ds_type and ds_type.startswith('carddef:'):
         return data_source
@@ -523,7 +554,7 @@ def get_object(data_source, ignore_errors=True):
     ds_type = data_source.get('type')
     if ds_type is None:
         return None
-    if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
+    if ds_type in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue'):
         named_data_source = NamedDataSource()
         named_data_source.data_source = data_source
         return named_data_source
@@ -966,6 +997,7 @@ class NamedDataSource(XmlStorableObject):
             'jsonp': _('JSONP'),
             'geojson': _('GeoJSON'),
             'formula': _('Python Expression'),
+            'jsonvalue': _('JSON Expression'),
         }
         data_source_type = self.data_source.get('type')
         return data_source_labels.get(data_source_type)
@@ -1011,7 +1043,7 @@ class NamedDataSource(XmlStorableObject):
 
 class StubNamedDataSource(NamedDataSource):
     type = 'formula'
-    data_source = {'type': 'formula', 'value': []}
+    data_source = {'type': 'jsonvalue', 'value': []}
     cache_duration = None
 
     def __init__(self, name=None):
diff --git a/wcs/fields.py b/wcs/fields.py
index fe5065fad..e7d411178 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -827,6 +827,7 @@ class Field:
             'jsonp': _('JSONP URL'),
             'geojson': _('GeoJSON URL'),
             'formula': _('Python Expression'),
+            'jsonvalue': _('JSON Expression'),
         }
         if value.get('type') in data_source_types:
             return '%s - %s' % (data_source_types[value.get('type')], value.get('value'))
diff --git a/wcs/templates/wcs/backoffice/data-source.html b/wcs/templates/wcs/backoffice/data-source.html
index b98ad1838..74a1e301d 100644
--- a/wcs/templates/wcs/backoffice/data-source.html
+++ b/wcs/templates/wcs/backoffice/data-source.html
@@ -38,6 +38,8 @@
             <li>{% trans "URL:" %} <a href="{{ url }}">{{ datasource.data_source.value }}</a></li>
           {% elif datasource.data_source.type == 'formula' %}
             <li>{% trans "Python Expression:" %} {{ datasource.data_source.value }}</li>
+          {% elif datasource.data_source.type == 'jsonvalue' %}
+            <li>{% trans "JSON Expression:" %} {{ datasource.data_source.value }}</li>
           {% elif datasource.data_source.type == 'wcs:users' %}
             {% spaceless %}
               <li>{% trans "Users with roles:" %}
-- 
2.35.1

