From c2a6a60ed2c56baa2a7723eca18bdabd88096e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 9 Nov 2020 09:41:01 +0100
Subject: [PATCH 1/3] api: add endpoint for geojson data sources (#47066)

---
 tests/test_datasource.py | 12 +++---
 wcs/api.py               | 14 +++++-
 wcs/data_sources.py      | 93 ++++++++++++++++++++++++++++++----------
 3 files changed, 90 insertions(+), 29 deletions(-)

diff --git a/tests/test_datasource.py b/tests/test_datasource.py
index 7871670fb..22dc9e633 100644
--- a/tests/test_datasource.py
+++ b/tests/test_datasource.py
@@ -588,22 +588,22 @@ def test_geojson_datasource(requests_pub, http_requests):
 def test_geojson_datasource_bad_url(http_requests, caplog):
     datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error loading GeoJSON data source' in caplog.records[-1].message
+    assert 'Error loading JSON data source' in caplog.records[-1].message
     assert 'status: 404' in caplog.records[-1].message
 
     datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error reading GeoJSON data source output' in caplog.records[-1].message
+    assert 'Error reading JSON data source output' in caplog.records[-1].message
     assert 'Expecting value:' in caplog.records[-1].message
 
     datasource = {'type': 'geojson', 'value': 'http://remote.example.net/connection-error'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error loading GeoJSON data source' in caplog.records[-1].message
+    assert 'Error loading JSON data source' in caplog.records[-1].message
     assert 'error' in caplog.records[-1].message
 
     datasource = {'type': 'geojson', 'value': 'http://remote.example.net/json-list-err1'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error reading GeoJSON data source output (err 1)' in caplog.records[-1].message
+    assert 'Error reading JSON data source output (err 1)' in caplog.records[-1].message
 
 
 def test_geojson_datasource_bad_url_scheme(caplog):
@@ -613,12 +613,12 @@ def test_geojson_datasource_bad_url_scheme(caplog):
 
     datasource = {'type': 'geojson', 'value': 'foo://bar'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error loading GeoJSON data source' in caplog.records[-1].message
+    assert 'Error loading JSON data source' in caplog.records[-1].message
     assert 'invalid scheme in URL' in caplog.records[-1].message
 
     datasource = {'type': 'geojson', 'value': '/bla/blo'}
     assert data_sources.get_items(datasource) == []
-    assert 'Error loading GeoJSON data source' in caplog.records[-1].message
+    assert 'Error loading JSON data source' in caplog.records[-1].message
     assert 'invalid scheme in URL' in caplog.records[-1].message
 
 
diff --git a/wcs/api.py b/wcs/api.py
index 2763f3852..e7af0b5cf 100644
--- a/wcs/api.py
+++ b/wcs/api.py
@@ -36,6 +36,7 @@ from wcs.categories import Category
 from wcs.conditions import Condition, ValidationError
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
+from wcs.data_sources import get_object as get_data_source_object
 from wcs.roles import Role, logged_users_role
 from wcs.forms.common import FormStatusPage
 import wcs.qommon.storage as st
@@ -912,10 +913,20 @@ class AutocompleteDirectory(Directory):
         return json.dumps({'data': [{'id': x['id'], 'text': x['text']} for x in values]})
 
 
+class GeoJsonDirectory(Directory):
+    def _q_lookup(self, component):
+        try:
+            data_source = get_data_source_object({'type': component}, ignore_errors=False)
+        except KeyError:
+            raise TraversalError()
+        get_response().set_content_type('application/json')
+        return json.dumps(data_source.get_geojson_data())
+
+
 class ApiDirectory(Directory):
     _q_exports = ['forms', 'roles', ('reverse-geocoding', 'reverse_geocoding'),
             'formdefs', 'categories', 'user', 'users', 'code', 'autocomplete',
-            'cards']
+            'cards', 'geojson']
 
     cards = ApiCardsDirectory()
     forms = ApiFormsDirectory()
@@ -925,6 +936,7 @@ class ApiDirectory(Directory):
     users = ApiUsersDirectory()
     code = ApiTrackingCodeDirectory()
     autocomplete = AutocompleteDirectory()
+    geojson = GeoJsonDirectory()
 
     def roles(self):
         get_response().set_content_type('application/json')
diff --git a/wcs/data_sources.py b/wcs/data_sources.py
index 2afc5f6b9..d31dfeff7 100644
--- a/wcs/data_sources.py
+++ b/wcs/data_sources.py
@@ -126,26 +126,39 @@ def get_items(data_source, include_disabled=False, mode=None):
     return tupled_items
 
 
-def request_json_items(url, data_source):
+def get_json_from_url(url, data_source):
     url = sign_url_auto_orig(url)
     data_key = data_source.get('data_attribute') or 'data'
+    geojson = data_source.get('type') == 'geojson'
     try:
         entries = misc.json_loads(misc.urlopen(url).read())
         if not isinstance(entries, dict):
             raise ValueError('not a json dict')
         if entries.get('err') not in (None, 0, "0"):
             raise ValueError('err %s' % entries['err'])
-        if not isinstance(entries.get(data_key), list):
-            raise ValueError('not a json dict with a %s list attribute' % data_key)
+        if geojson:
+            if not isinstance(entries.get('features'), list):
+                raise ValueError('bad geojson format')
+        else:
+            if not isinstance(entries.get(data_key), list):
+                raise ValueError('not a json dict with a %s list attribute' % data_key)
     except misc.ConnectionError as e:
         get_logger().warning('Error loading JSON data source (%s)' % str(e))
         return None
     except (ValueError, TypeError) as e:
         get_logger().warning('Error reading JSON data source output (%s)' % str(e))
         return None
-    items = []
+    return entries
+
+
+def request_json_items(url, data_source):
+    entries = get_json_from_url(url, data_source)
+    if entries is None:
+        return None
+    data_key = data_source.get('data_attribute') or 'data'
     id_attribute = data_source.get('id_attribute') or 'id'
     text_attribute = data_source.get('text_attribute') or 'text'
+    items = []
     for item in entries.get(data_key):
         # skip malformed items
         if not isinstance(item, dict):
@@ -162,20 +175,8 @@ def request_json_items(url, data_source):
 
 
 def request_geojson_items(url, data_source):
-    url = sign_url_auto_orig(url)
-    try:
-        entries = misc.json_loads(misc.urlopen(url).read())
-        if not isinstance(entries, dict):
-            raise ValueError('not a json dict')
-        if entries.get('err') not in (None, 0, "0"):
-            raise ValueError('err %s' % entries['err'])
-        if not isinstance(entries.get('features'), list):
-            raise ValueError('bad geojson format')
-    except misc.ConnectionError as e:
-        get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
-        return None
-    except (ValueError, TypeError) as e:
-        get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
+    entries = get_json_from_url(url, data_source)
+    if entries is None:
         return None
     items = []
     id_property = data_source.get('id_property') or 'id'
@@ -306,7 +307,7 @@ def get_real(data_source):
     return NamedDataSource.get_by_slug(ds_type).data_source
 
 
-def get_object(data_source):
+def get_object(data_source, ignore_errors=True):
     if not data_source:
         return None
     ds_type = data_source.get('type')
@@ -320,7 +321,7 @@ def get_object(data_source):
         named_data_source = NamedDataSource()
         named_data_source.data_source = data_source
         return named_data_source
-    return NamedDataSource.get_by_slug(ds_type)
+    return NamedDataSource.get_by_slug(ds_type, ignore_errors=ignore_errors)
 
 
 class NamedDataSource(XmlStorableObject):
@@ -444,10 +445,12 @@ class NamedDataSource(XmlStorableObject):
             }
 
     @classmethod
-    def get_by_slug(cls, slug):
+    def get_by_slug(cls, slug, ignore_errors=True):
         objects = [x for x in cls.select() if x.slug == slug]
         if objects:
             return objects[0]
+        if not ignore_errors:
+            raise KeyError(slug)
         get_logger().warning("data source '%s' does not exist" % slug)
         return StubNamedDataSource(name=slug)
 
@@ -499,6 +502,52 @@ class NamedDataSource(XmlStorableObject):
                     }))
         return None
 
+    def get_geojson_url(self):
+        assert self.type == 'geojson'
+        return '/api/geojson/%s' % self.slug
+
+    def get_geojson_data(self):
+        url = self.data_source.get('value').strip()
+        if Template.is_template_string(url):
+            vars = get_publisher().substitutions.get_context_variables(mode='lazy')
+            url = get_variadic_url(url, vars)
+
+        request = get_request()
+        if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
+            return request.datasources_cache[url]
+
+        cache_duration = 0
+        if self.cache_duration:
+            cache_duration = int(self.cache_duration)
+
+        if cache_duration:
+            cache_key = 'geojson-data-source-%s' % force_str(hashlib.md5(force_bytes(url)).hexdigest())
+            from django.core.cache import cache
+            data = cache.get(cache_key)
+            if data is not None:
+                return data
+
+        data = get_json_from_url(url, self.data_source)
+        id_property = self.id_property or 'id'
+        label_template_property = self.label_template_property or '{{ text }}'
+
+        for feature in data['features']:
+            feature['properties']['_id'] = feature['properties'][id_property]
+            try:
+                feature['properties']['_text'] = Template(
+                        label_template_property).render(feature['properties'])
+            except (TemplateSyntaxError, VariableDoesNotExist):
+                pass
+            if not feature['properties'].get('_text'):
+                feature['properties']['_text'] = feature['properties']['_id']
+
+        if hasattr(request, 'datasources_cache'):
+            request.datasources_cache[url] = data
+        if cache_duration:
+            cache.set(cache_key, data, cache_duration)
+
+        return data
+
     def get_value_by_id(self, param_name, param_value):
         url = self.data_source.get('value').strip()
         if Template.is_template_string(url):
@@ -562,7 +611,7 @@ class NamedDataSource(XmlStorableObject):
         elif self.type == 'json' and self.id_parameter:
             value = self.get_value_by_id(self.id_parameter, option_id)
         else:
-            structured_items = get_structured_items(self.data_source, mode='lazy')
+            structured_items = get_structured_items(self.extended_data_source, mode='lazy')
             for item in structured_items:
                 if str(item['id']) == str(option_id):
                     value = item
-- 
2.29.2

