0001-api-add-endpoint-for-geojson-data-sources-47066.patch

Frédéric Péters, 24 novembre 2020 16:19

Voir les différences: en ligne côte à côte

Subject: [PATCH 1/3] api: add endpoint for geojson data sources (#47066)

 tests/test_datasource.py | 12 +++---
 wcs/api.py               | 14 +++++-
 wcs/data_sources.py      | 93 ++++++++++++++++++++++++++++++----------
 3 files changed, 90 insertions(+), 29 deletions(-)

     def test_geojson_datasource_bad_url(http_requests, caplog):
         datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404'}
         assert data_sources.get_items(datasource) == []
         assert 'Error loading GeoJSON data source' in caplog.records[-1].message
         assert 'Error loading JSON data source' in caplog.records[-1].message
         assert 'status: 404' in caplog.records[-1].message
         datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml'}
         assert data_sources.get_items(datasource) == []
         assert 'Error reading GeoJSON data source output' in caplog.records[-1].message
         assert 'Error reading JSON data source output' in caplog.records[-1].message
         assert 'Expecting value:' in caplog.records[-1].message
         datasource = {'type': 'geojson', 'value': 'http://remote.example.net/connection-error'}
         assert data_sources.get_items(datasource) == []
         assert 'Error loading GeoJSON data source' in caplog.records[-1].message
         assert 'Error loading JSON data source' in caplog.records[-1].message
         assert 'error' in caplog.records[-1].message
         datasource = {'type': 'geojson', 'value': 'http://remote.example.net/json-list-err1'}
         assert data_sources.get_items(datasource) == []
         assert 'Error reading GeoJSON data source output (err 1)' in caplog.records[-1].message
         assert 'Error reading JSON data source output (err 1)' in caplog.records[-1].message
     def test_geojson_datasource_bad_url_scheme(caplog):
-...
         datasource = {'type': 'geojson', 'value': 'foo://bar'}
         assert data_sources.get_items(datasource) == []
         assert 'Error loading GeoJSON data source' in caplog.records[-1].message
         assert 'Error loading JSON data source' in caplog.records[-1].message
         assert 'invalid scheme in URL' in caplog.records[-1].message
         datasource = {'type': 'geojson', 'value': '/bla/blo'}
         assert data_sources.get_items(datasource) == []
         assert 'Error loading GeoJSON data source' in caplog.records[-1].message
         assert 'Error loading JSON data source' in caplog.records[-1].message
         assert 'invalid scheme in URL' in caplog.records[-1].message

     from wcs.conditions import Condition, ValidationError
     from wcs.carddef import CardDef
     from wcs.formdef import FormDef
     from wcs.data_sources import get_object as get_data_source_object
     from wcs.roles import Role, logged_users_role
     from wcs.forms.common import FormStatusPage
     import wcs.qommon.storage as st
-...
             return json.dumps({'data': [{'id': x['id'], 'text': x['text']} for x in values]})
     class GeoJsonDirectory(Directory):
         def _q_lookup(self, component):
             try:
                 data_source = get_data_source_object({'type': component}, ignore_errors=False)
             except KeyError:
                 raise TraversalError()
             get_response().set_content_type('application/json')
             return json.dumps(data_source.get_geojson_data())
     class ApiDirectory(Directory):
         _q_exports = ['forms', 'roles', ('reverse-geocoding', 'reverse_geocoding'),
                 'formdefs', 'categories', 'user', 'users', 'code', 'autocomplete',
                 'cards']
                 'cards', 'geojson']
         cards = ApiCardsDirectory()
         forms = ApiFormsDirectory()
-...
         users = ApiUsersDirectory()
         code = ApiTrackingCodeDirectory()
         autocomplete = AutocompleteDirectory()
         geojson = GeoJsonDirectory()
         def roles(self):
             get_response().set_content_type('application/json')

         return tupled_items
     def request_json_items(url, data_source):
     def get_json_from_url(url, data_source):
         url = sign_url_auto_orig(url)
         data_key = data_source.get('data_attribute') or 'data'
         geojson = data_source.get('type') == 'geojson'
         try:
             entries = misc.json_loads(misc.urlopen(url).read())
             if not isinstance(entries, dict):
                 raise ValueError('not a json dict')
             if entries.get('err') not in (None, 0, "0"):
                 raise ValueError('err %s' % entries['err'])
             if not isinstance(entries.get(data_key), list):
                 raise ValueError('not a json dict with a %s list attribute' % data_key)
             if geojson:
                 if not isinstance(entries.get('features'), list):
                     raise ValueError('bad geojson format')
             else:
                 if not isinstance(entries.get(data_key), list):
                     raise ValueError('not a json dict with a %s list attribute' % data_key)
         except misc.ConnectionError as e:
             get_logger().warning('Error loading JSON data source (%s)' % str(e))
             return None
         except (ValueError, TypeError) as e:
             get_logger().warning('Error reading JSON data source output (%s)' % str(e))
             return None
         items = []
         return entries
     def request_json_items(url, data_source):
         entries = get_json_from_url(url, data_source)
         if entries is None:
             return None
         data_key = data_source.get('data_attribute') or 'data'
         id_attribute = data_source.get('id_attribute') or 'id'
         text_attribute = data_source.get('text_attribute') or 'text'
         items = []
         for item in entries.get(data_key):
             # skip malformed items
             if not isinstance(item, dict):
-...
     def request_geojson_items(url, data_source):
         url = sign_url_auto_orig(url)
         try:
             entries = misc.json_loads(misc.urlopen(url).read())
             if not isinstance(entries, dict):
                 raise ValueError('not a json dict')
             if entries.get('err') not in (None, 0, "0"):
                 raise ValueError('err %s' % entries['err'])
             if not isinstance(entries.get('features'), list):
                 raise ValueError('bad geojson format')
         except misc.ConnectionError as e:
             get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
             return None
         except (ValueError, TypeError) as e:
             get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
         entries = get_json_from_url(url, data_source)
         if entries is None:
             return None
         items = []
         id_property = data_source.get('id_property') or 'id'
-...
         return NamedDataSource.get_by_slug(ds_type).data_source
     def get_object(data_source):
     def get_object(data_source, ignore_errors=True):
         if not data_source:
             return None
         ds_type = data_source.get('type')
-...
             named_data_source = NamedDataSource()
             named_data_source.data_source = data_source
             return named_data_source
         return NamedDataSource.get_by_slug(ds_type)
         return NamedDataSource.get_by_slug(ds_type, ignore_errors=ignore_errors)
     class NamedDataSource(XmlStorableObject):
-...
+                }
         @classmethod
         def get_by_slug(cls, slug):
         def get_by_slug(cls, slug, ignore_errors=True):
             objects = [x for x in cls.select() if x.slug == slug]
             if objects:
                 return objects[0]
             if not ignore_errors:
                 raise KeyError(slug)
             get_logger().warning("data source '%s' does not exist" % slug)
             return StubNamedDataSource(name=slug)
-...
                         }))
             return None
         def get_geojson_url(self):
             assert self.type == 'geojson'
             return '/api/geojson/%s' % self.slug
         def get_geojson_data(self):
             url = self.data_source.get('value').strip()
             if Template.is_template_string(url):
                 vars = get_publisher().substitutions.get_context_variables(mode='lazy')
                 url = get_variadic_url(url, vars)
             request = get_request()
             if hasattr(request, 'datasources_cache') and url in request.datasources_cache:
                 return request.datasources_cache[url]
             cache_duration = 0
             if self.cache_duration:
                 cache_duration = int(self.cache_duration)
             if cache_duration:
                 cache_key = 'geojson-data-source-%s' % force_str(hashlib.md5(force_bytes(url)).hexdigest())
                 from django.core.cache import cache
                 data = cache.get(cache_key)
                 if data is not None:
                     return data
             data = get_json_from_url(url, self.data_source)
             id_property = self.id_property or 'id'
             label_template_property = self.label_template_property or '{{ text }}'
             for feature in data['features']:
                 feature['properties']['_id'] = feature['properties'][id_property]
                 try:
                     feature['properties']['_text'] = Template(
                             label_template_property).render(feature['properties'])
                 except (TemplateSyntaxError, VariableDoesNotExist):
                     pass
                 if not feature['properties'].get('_text'):
                     feature['properties']['_text'] = feature['properties']['_id']
             if hasattr(request, 'datasources_cache'):
                 request.datasources_cache[url] = data
             if cache_duration:
                 cache.set(cache_key, data, cache_duration)
             return data
         def get_value_by_id(self, param_name, param_value):
             url = self.data_source.get('value').strip()
             if Template.is_template_string(url):
-...
             elif self.type == 'json' and self.id_parameter:
                 value = self.get_value_by_id(self.id_parameter, option_id)
             else:
                 structured_items = get_structured_items(self.data_source, mode='lazy')
                 structured_items = get_structured_items(self.extended_data_source, mode='lazy')
                 for item in structured_items:
                     if str(item['id']) == str(option_id):
                         value = item
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0001-api-add-endpoint-for-geojson-data-sources-47066.patch