From be52fbf13bb0c37e5f3b2085b644978ac7045387 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Fri, 9 Oct 2020 15:35:00 +0200
Subject: [PATCH] datasources: json datasource attributes (#47218)

---
 tests/test_datasource.py              |  40 +++++++++
 tests/test_datasources_admin_pages.py |  17 ++++
 wcs/admin/data_sources.py             |  33 +++++++
 wcs/data_sources.py                   | 122 ++++++++++++++++----------
 4 files changed, 167 insertions(+), 45 deletions(-)

diff --git a/tests/test_datasource.py b/tests/test_datasource.py
index 2abda1ac..2ed83541 100644
--- a/tests/test_datasource.py
+++ b/tests/test_datasource.py
@@ -283,6 +283,46 @@ def test_json_datasource(requests_pub, http_requests):
     assert data_sources.get_items(datasource) == []
     assert data_sources.get_structured_items(datasource) == []
 
+    # specify data_attribute
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'results'}
+    get_request().datasources_cache = {}
+    json_file = open(json_file_path, 'w')
+    json.dump({'results': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file)
+    json_file.close()
+    assert data_sources.get_structured_items(datasource) == [
+            {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
+
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'data'}
+    get_request().datasources_cache = {}
+    assert data_sources.get_structured_items(datasource) == []
+
+    # specify id_attribute
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'pk'}
+    get_request().datasources_cache = {}
+    json_file = open(json_file_path, 'w')
+    json.dump({'data': [{'pk': '1', 'text': 'foo'}, {'pk': '2', 'text': 'bar'}]}, json_file)
+    json_file.close()
+    assert data_sources.get_structured_items(datasource) == [
+            {'id': '1', 'text': 'foo', 'pk': '1'}, {'id': '2', 'text': 'bar', 'pk': '2'}]
+
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'id'}
+    get_request().datasources_cache = {}
+    assert data_sources.get_structured_items(datasource) == []
+
+    # specify text_attribute
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'label'}
+    get_request().datasources_cache = {}
+    json_file = open(json_file_path, 'w')
+    json.dump({'data': [{'id': '1', 'label': 'foo'}, {'id': '2', 'label': 'bar'}]}, json_file)
+    json_file.close()
+    assert data_sources.get_structured_items(datasource) == [
+            {'id': '1', 'text': 'foo', 'label': 'foo'}, {'id': '2', 'text': 'bar', 'label': 'bar'}]
+
+    datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'text'}
+    get_request().datasources_cache = {}
+    assert data_sources.get_structured_items(datasource) == [
+            {'id': '1', 'text': '1', 'label': 'foo'}, {'id': '2', 'text': '2', 'label': 'bar'}]
+
 
 def test_json_datasource_bad_url(http_requests, caplog):
     datasource = {'type': 'json', 'value': 'http://remote.example.net/404'}
diff --git a/tests/test_datasources_admin_pages.py b/tests/test_datasources_admin_pages.py
index 0bfaa80c..58010a34 100644
--- a/tests/test_datasources_admin_pages.py
+++ b/tests/test_datasources_admin_pages.py
@@ -161,7 +161,24 @@ def test_data_sources_view(pub):
     assert 'Preview' in resp.text
     assert 'foo' in resp.text
 
+    # with other attributes
+    json_file = open(json_file_path, 'w')
+    json.dump({'results': [{'pk': '1', 'label': 'foo'}, {'pk': '2'}]}, json_file)
+    json_file.close()
+
+    data_source.data_attribute = 'results'
+    data_source.id_attribute = 'pk'
+    data_source.text_attribute = 'label'
+    data_source.store()
+    with HttpRequestsMocking():
+        resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
+    assert 'Preview' in resp.text
+    assert '<tt>1</tt>: foo</li>' in resp.text
+    assert '<tt>2</tt>: 2</li>' in resp.text
+    assert '<p>Additional keys are available: label, pk</p>' in resp.text
+
     # variadic url
+    data_source.data_attribute = None
     data_source.data_source = {'type': 'json', 'value': '{{ site_url }}/foo/bar'}
     data_source.store()
     with HttpRequestsMocking():
diff --git a/wcs/admin/data_sources.py b/wcs/admin/data_sources.py
index f0aefeae..b4c1fc79 100644
--- a/wcs/admin/data_sources.py
+++ b/wcs/admin/data_sources.py
@@ -104,6 +104,36 @@ class NamedDataSourceUI(object):
                      'data-dynamic-display-child-of': 'data_source$type',
                      'data-dynamic-display-value': 'geojson',
                  })
+        form.add(StringWidget, 'data_attribute',
+                 value=self.datasource.data_attribute,
+                 title=_('Data Attribute'),
+                 hint=_('Name of the attribute containing the list of results (default: data)'),
+                 required=False,
+                 advanced=False,
+                 attrs={
+                     'data-dynamic-display-child-of': 'data_source$type',
+                     'data-dynamic-display-value': 'json',
+                 })
+        form.add(StringWidget, 'id_attribute',
+                 value=self.datasource.id_attribute,
+                 title=_('Id Attribute'),
+                 hint=_('Name of the attribute containing the identifier of an entry (default: id)'),
+                 required=False,
+                 advanced=False,
+                 attrs={
+                     'data-dynamic-display-child-of': 'data_source$type',
+                     'data-dynamic-display-value-in': 'json|geojson',
+                 })
+        form.add(StringWidget, 'text_attribute',
+                 value=self.datasource.text_attribute,
+                 title=_('Text Attribute'),
+                 hint=_('Name of the attribute containing the label of an entry (default: text)'),
+                 required=False,
+                 advanced=False,
+                 attrs={
+                     'data-dynamic-display-child-of': 'data_source$type',
+                     'data-dynamic-display-value': 'json',
+                 })
         form.add(StringWidget, 'label_template_property',
                  value=self.datasource.label_template_property,
                  title=_('Label template'),
@@ -148,6 +178,9 @@ class NamedDataSourceUI(object):
         self.datasource.cache_duration = form.get_widget('cache_duration').parse()
         self.datasource.query_parameter = form.get_widget('query_parameter').parse()
         self.datasource.id_parameter = form.get_widget('id_parameter').parse()
+        self.datasource.data_attribute = form.get_widget('data_attribute').parse()
+        self.datasource.id_attribute = form.get_widget('id_attribute').parse()
+        self.datasource.text_attribute = form.get_widget('text_attribute').parse()
         self.datasource.id_property = form.get_widget('id_property').parse()
         self.datasource.label_template_property = form.get_widget('label_template_property').parse()
         if slug_widget:
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index 5e9d2b1b..fadc1784 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -118,54 +118,69 @@ def get_items(data_source, include_disabled=False, mode=None):
 
 def request_json_items(url, data_source):
     url = sign_url_auto_orig(url)
-    geojson = data_source.get('type') == 'geojson'
+    data_key = data_source.get('data_attribute') or 'data'
     try:
         entries = misc.json_loads(misc.urlopen(url).read())
         if not isinstance(entries, dict):
             raise ValueError('not a json dict')
         if entries.get('err') not in (None, 0, "0"):
             raise ValueError('err %s' % entries['err'])
-        if not geojson and not isinstance(entries.get('data'), list):
-            raise ValueError('not a json dict with a data list attribute')
-        if geojson and not isinstance(entries.get('features'), list):
-            raise ValueError('bad geojson format')
+        if not isinstance(entries.get(data_key), list):
+            raise ValueError('not a json dict with a %s list attribute' % data_key)
     except misc.ConnectionError as e:
-        if geojson:
-            get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
-        else:
-            get_logger().warning('Error loading JSON data source (%s)' % str(e))
+        get_logger().warning('Error loading JSON data source (%s)' % str(e))
         return None
     except (ValueError, TypeError) as e:
-        if geojson:
-            get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
+        get_logger().warning('Error reading JSON data source output (%s)' % str(e))
+        return None
+    items = []
+    id_attribute = data_source.get('id_attribute') or 'id'
+    text_attribute = data_source.get('text_attribute') or 'text'
+    for item in entries.get(data_key):
+        # skip malformed items
+        if not isinstance(item, dict):
+            continue
+        if item.get(id_attribute) is None or item.get(id_attribute) == '':
+            continue
+        item['id'] = item[id_attribute]
+        if text_attribute not in item:
+            item['text'] = item['id']
         else:
-            get_logger().warning('Error reading JSON data source output (%s)' % str(e))
+            item['text'] = item[text_attribute]
+        items.append(item)
+    return items
+
+
+def request_geojson_items(url, data_source):
+    url = sign_url_auto_orig(url)
+    try:
+        entries = misc.json_loads(misc.urlopen(url).read())
+        if not isinstance(entries, dict):
+            raise ValueError('not a json dict')
+        if entries.get('err') not in (None, 0, "0"):
+            raise ValueError('err %s' % entries['err'])
+        if not isinstance(entries.get('features'), list):
+            raise ValueError('bad geojson format')
+    except misc.ConnectionError as e:
+        get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
+        return None
+    except (ValueError, TypeError) as e:
+        get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
         return None
     items = []
-    if geojson:
-        id_property = data_source.get('id_property') or 'id'
-        for item in entries.get('features'):
-            if not item.get('properties', {}).get(id_property):
-                continue
-            item['id'] = item['properties'][id_property]
-            try:
-                item['text'] = Template(
-                    data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
-            except (TemplateSyntaxError, VariableDoesNotExist):
-                pass
-            if not item.get('text'):
-                item['text'] = item['id']
-            items.append(item)
-    else:
-        for item in entries.get('data'):
-            # skip malformed items
-            if not isinstance(item, dict):
-                continue
-            if item.get('id') is None or item.get('id') == '':
-                continue
-            if 'text' not in item:
-                item['text'] = item['id']
-            items.append(item)
+    id_property = data_source.get('id_property') or 'id'
+    for item in entries.get('features'):
+        if not item.get('properties', {}).get(id_property):
+            continue
+        item['id'] = item['properties'][id_property]
+        try:
+            item['text'] = Template(
+                data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
+        except (TemplateSyntaxError, VariableDoesNotExist):
+            pass
+        if not item.get('text'):
+            item['text'] = item['id']
+        items.append(item)
     return items
 
 
@@ -247,7 +262,10 @@ def get_structured_items(data_source, mode=None):
             if items is not None:
                 return items
 
-        items = request_json_items(url, data_source)
+        if geojson:
+            items = request_geojson_items(url, data_source)
+        else:
+            items = request_json_items(url, data_source)
         if items is None:
             return []
         if hasattr(request, 'datasources_cache'):
@@ -298,6 +316,9 @@ class NamedDataSource(XmlStorableObject):
     cache_duration = None
     query_parameter = None
     id_parameter = None
+    data_attribute = None
+    id_attribute = None
+    text_attribute = None
     id_property = None
     label_template_property = None
 
@@ -306,6 +327,9 @@ class NamedDataSource(XmlStorableObject):
             ('cache_duration', 'str'),
             ('query_parameter', 'str'),
             ('id_parameter', 'str'),
+            ('data_attribute', 'str'),
+            ('id_attribute', 'str'),
+            ('text_attribute', 'str'),
             ('id_property', 'str'),
             ('label_template_property', 'str'),
             ('data_source', 'data_source'),
@@ -321,14 +345,22 @@ class NamedDataSource(XmlStorableObject):
 
     @property
     def extended_data_source(self):
-        if self.type != 'geojson':
-            return self.data_source
-        data_source = self.data_source.copy()
-        data_source.update({
-            'id_property': self.id_property,
-            'label_template_property': self.label_template_property,
-        })
-        return data_source
+        if self.type == 'geojson':
+            data_source = self.data_source.copy()
+            data_source.update({
+                'id_property': self.id_property,
+                'label_template_property': self.label_template_property,
+            })
+            return data_source
+        if self.type == 'json':
+            data_source = self.data_source.copy()
+            data_source.update({
+                'data_attribute': self.data_attribute,
+                'id_attribute': self.id_attribute,
+                'text_attribute': self.text_attribute,
+            })
+            return data_source
+        return self.data_source
 
     def can_jsonp(self):
         if self.type == 'jsonp':
-- 
2.28.0

