From 8a60890a1dc4ce3e02fe612c1986e53d85819919 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 7 Feb 2020 08:37:16 +0100 Subject: [PATCH 3/3] arcgis: add query system (#27782) --- .../migrations/0005_auto_20200207_0524.py | 41 +++ passerelle/apps/arcgis/models.py | 259 +++++++++++++++++- .../templates/arcgis/arcgis_detail.html | 38 +++ .../arcgis/query_confirm_delete.html | 1 + .../arcgis/templates/arcgis/query_form.html | 1 + passerelle/apps/arcgis/urls.py | 28 ++ passerelle/apps/arcgis/views.py | 50 ++++ passerelle/utils/templates.py | 50 ++++ tests/test_arcgis.py | 114 +++++++- 9 files changed, 574 insertions(+), 8 deletions(-) create mode 100644 passerelle/apps/arcgis/migrations/0005_auto_20200207_0524.py create mode 100644 passerelle/apps/arcgis/templates/arcgis/arcgis_detail.html create mode 100644 passerelle/apps/arcgis/templates/arcgis/query_confirm_delete.html create mode 100644 passerelle/apps/arcgis/templates/arcgis/query_form.html create mode 100644 passerelle/apps/arcgis/urls.py create mode 100644 passerelle/apps/arcgis/views.py create mode 100644 passerelle/utils/templates.py diff --git a/passerelle/apps/arcgis/migrations/0005_auto_20200207_0524.py b/passerelle/apps/arcgis/migrations/0005_auto_20200207_0524.py new file mode 100644 index 00000000..eecdf313 --- /dev/null +++ b/passerelle/apps/arcgis/migrations/0005_auto_20200207_0524.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2020-02-07 11:24 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import passerelle.apps.arcgis.models +import passerelle.utils.templates + + +class Migration(migrations.Migration): + + dependencies = [ + ('arcgis', '0004_remove_arcgis_log_level'), + ] + + operations = [ + migrations.CreateModel( + name='Query', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('slug', models.SlugField(max_length=128, verbose_name='Name')), + ('description', models.TextField(blank=True, verbose_name='Description')), + ('service', models.CharField(max_length=64, verbose_name='Service')), + ('id_template', models.TextField(help_text="Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}", validators=[passerelle.utils.templates.validate_template], verbose_name='Id template')), + ('text_template', models.TextField(help_text="Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}", validators=[passerelle.utils.templates.validate_template], verbose_name='Text template')), + ('folder', models.CharField(blank=True, max_length=64, verbose_name='Folder')), + ('layer', models.CharField(blank=True, max_length=8, verbose_name='Layer')), + ('where', models.TextField(blank=True, help_text='Use syntax {name} to introduce a string parameter and {name:d} for a decimal parameter. ex.:
adress LIKE {adress}
population < {population:d}', validators=[passerelle.apps.arcgis.models.validate_where], verbose_name='Where')), + ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='arcgis.ArcGIS', verbose_name='Resource')), + ], + options={ + 'ordering': ['name', 'slug'], + }, + ), + migrations.AlterUniqueTogether( + name='query', + unique_together=set([('resource', 'name'), ('resource', 'slug')]), + ), + ] diff --git a/passerelle/apps/arcgis/models.py b/passerelle/apps/arcgis/models.py index 6c1e72da..587e7085 100644 --- a/passerelle/apps/arcgis/models.py +++ b/passerelle/apps/arcgis/models.py @@ -14,13 +14,21 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import string + +from django.core.exceptions import ValidationError +from django.core.urlresolvers import reverse from django.db import models -from django.template import Template, Context +from django.shortcuts import get_object_or_404 from django.utils.six.moves.urllib import parse as urlparse +from django.utils import six +from django.utils.text import slugify from django.utils.translation import ugettext_lazy as _ +from django.utils.html import mark_safe, format_html from passerelle.utils.jsonresponse import APIError from passerelle.utils.api import endpoint +from passerelle.utils.templates import render_to_string, validate_template from passerelle.base.models import BaseResource, HTTPResource @@ -141,10 +149,6 @@ class ArcGIS(BaseResource, HTTPResource): return feature['attributes'][attribute] return feature['attributes'].get(aliases.get(attribute)) - if template: - template = Template(template) - if id_template: - id_template = Template(id_template) for n, feature in enumerate(features): if 'attributes' in feature: feature['id'] = '%s' % get_feature_attribute(feature, id_fieldname) @@ -152,9 +156,9 @@ class ArcGIS(BaseResource, HTTPResource): else: feature['id'] = feature['text'] = '%d' % (n+1) if template: - feature['text'] = template.render(Context(feature)) + feature['text'] = render_to_string(template, feature) if id_template: - feature['id'] = id_template.render(Context(feature)) + feature['id'] = render_to_string(id_template, feature) if not full and 'geometry' in feature: del feature['geometry'] data.append(feature) @@ -185,3 +189,244 @@ class ArcGIS(BaseResource, HTTPResource): if len(features) == 1: return {'data': features[0]} return {'data': features} + + @endpoint(name='q', + description=_('Query'), + pattern=r'^(?P[\w:_-]+)/$', + perm='can_access', + parameters={ + 'q': {'description': _('Search text in display field')}, + 'full': { + 'description': _('Returns all ArcGIS informations (geometry, metadata)'), + 'type': 'bool', + }, + }, + show=False) + def q(self, request, query_slug, q=None, full=False, **kwargs): + query = get_object_or_404(Query, resource=self, slug=query_slug) + refs = [ref for ref, _ in query.where_references] + refs += ['q', 'full'] + kwargs = {} + for key in request.GET: + if key not in refs: + kwargs[key] = request.GET[key] + return query.q(request, q=None, full=full, **kwargs) + + def export_json(self): + d = super(ArcGIS, self).export_json() + d['queries'] = [query.export_json() for query in self.query_set.all()] + return d + + @classmethod + def import_json_real(cls, overwrite, instance, d, **kwargs): + queries = d.pop('queries', []) + instance = super(ArcGIS, cls).import_json_real(overwrite, instance, d, **kwargs) + new = [] + if instance and overwrite: + Query.objects.filter(resource=instance).delete() + for query in queries: + q = Query.import_json(query) + q.resource = instance + new.append(q) + Query.objects.bulk_create(new) + return instance + + def create_query_url(self): + return reverse('arcgis-query-new', kwargs={'slug': self.slug}) + + +class SqlFormatter(string.Formatter): + def format_field(self, value, format_spec): + if format_spec and format_spec[-1].isalpha() and format_spec[-1] == 'd': + value = int(value) + formatted = super(SqlFormatter, self).format_field(value, format_spec) + if not format_spec or not format_spec[-1].isalpha() or format_spec[-1] == 's': + formatted = "'%s'" % formatted.replace("'", "''") + return formatted + + +def validate_where(format_string): + formatter = SqlFormatter() + for prefix, ref, format_spec, conversion in formatter.parse(format_string): + if ref is None: + pass + elif ref == '': + raise ValidationError(_('missing reference')) + elif ref != slugify(ref): + raise ValidationError(_('invalid reference')) + + +@six.python_2_unicode_compatible +class Query(models.Model): + resource = models.ForeignKey( + to=ArcGIS, + verbose_name=_('Resource')) + + name = models.CharField( + verbose_name=_('Name'), + max_length=128) + slug = models.SlugField( + verbose_name=_('Slug'), + max_length=128) + description = models.TextField( + verbose_name=_('Description'), + blank=True) + + service = models.CharField( + verbose_name=_('ArcGis Service'), + max_length=64) + + id_template = models.TextField( + verbose_name=_('Id template'), + validators=[validate_template], + help_text=_('Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}')) + + text_template = models.TextField( + verbose_name=_('Text template'), + help_text=_('Use Django\'s template syntax. Attributes can be accessed through {{ attributes.name }}'), + validators=[validate_template]) + + folder = models.CharField( + verbose_name=_('ArcGis Folder'), + max_length=64, + blank=True) + layer = models.CharField( + verbose_name=_('ArcGis Layer'), + max_length=8, + blank=True) + + where = models.TextField( + verbose_name=_('ArcGis Where Clause'), + blank=True, + validators=[validate_where], + help_text=mark_safe( + _( + 'Use syntax {name} to introduce a string ' + 'parameter and {name:d} for a decimal parameter. ex.:
' + 'adress LIKE (\'%\' || UPPER({adress}) || \'%\')
' + 'population < {population:d}'))) + + class Meta: + unique_together = [ + ('resource', 'name'), + ('resource', 'slug'), + ] + ordering = ['name'] + + @property + def where_references(self): + if self.where: + return [(ref, int if spec and spec[-1] == 'd' else str) + for _, ref, spec, _ in SqlFormatter().parse(self.where) if ref is not None] + else: + return [] + + def q(self, request, q=None, full=False, **kwargs): + kwargs.update({ + 'service': self.service, + 'id_template': self.id_template, + 'template': self.text_template, + }) + if self.folder: + kwargs['folder'] = self.folder + if self.layer: + kwargs['layer'] = self.layer + if self.where: + format_kwargs = {key: request.GET.get(key, '') for key, klass in self.where_references} + formatter = SqlFormatter() + kwargs['where'] = formatter.format(self.where, **format_kwargs) + return self.resource.mapservice_query(request, q=q, full=full, **kwargs) + + class QueryEndpoint: + http_method = 'get' + + def __init__(self, query): + self.object = query.resource + self.query = query + self.name = 'q/%s/' % query.slug + + @property + def description(self): + return self.query.name + + @property + def long_description(self): + return self.query.description + + def example_url(self): + kwargs = { + 'connector': self.object.get_connector_slug(), + 'slug': self.object.slug, + 'endpoint': self.name, + } + query_string = '' + if self.query.where_references: + query_string = '?' + '&'.join( + ['%s=%s' % (ref, '') for ref, _ in self.query.where_references + [('q', ''), ('full', '')]]) + return reverse('generic-endpoint', kwargs=kwargs) + query_string + + def example_url_as_html(self): + kwargs = { + 'connector': self.object.get_connector_slug(), + 'slug': self.object.slug, + 'endpoint': self.name, + } + query_string = '' + if self.query.where_references: + query_string = '?' + '&'.join( + [format_html('{0}={1}', ref, ref) + for ref, klass in self.query.where_references + [('q', ''), ('full', '')]]) + return mark_safe(reverse('generic-endpoint', kwargs=kwargs) + query_string) + + def get_params(self): + params = [] + for ref, klass in self.query.where_references: + params.append({ + 'name': ref, + 'type': 'integer' if klass is int else 'string', + }) + params.extend([ + { + 'name': 'q', + 'description': _('Search text in display field'), + }, + { + 'name': 'full', + 'description': _('Returns all ArcGIS informations (geometry, metadata)'), + 'type': 'bool', + } + ]) + return params + + @property + def endpoint(self): + return self.QueryEndpoint(self) + + def __str__(self): + return self.name + + def export_json(self): + d = {} + fields = [ + f for f in self.__class__._meta.get_fields() + if f.concrete and ( + not f.is_relation + or f.one_to_one + or (f.many_to_one and f.related_model) + ) and f.name not in ['id', 'resource'] + ] + for field in fields: + d[field.name] = getattr(self, field.name) + return d + + @classmethod + def import_json(cls, d): + return cls(**d) + + def delete_url(self): + return reverse('arcgis-query-delete', + kwargs={'slug': self.resource.slug, 'pk': self.pk}) + + def edit_url(self): + return reverse('arcgis-query-edit', + kwargs={'slug': self.resource.slug, 'pk': self.pk}) diff --git a/passerelle/apps/arcgis/templates/arcgis/arcgis_detail.html b/passerelle/apps/arcgis/templates/arcgis/arcgis_detail.html new file mode 100644 index 00000000..c0757f14 --- /dev/null +++ b/passerelle/apps/arcgis/templates/arcgis/arcgis_detail.html @@ -0,0 +1,38 @@ +{% extends "passerelle/manage/service_view.html" %} +{% load i18n passerelle %} + +{% block endpoints %} +{{ block.super }} + +

{% trans "Custom queries" %}

+
    + {% for query in object.query_set.all %} + {% include "passerelle/manage/endpoint.html" with endpoint=query.endpoint %} + {% endfor %} +
+{% endblock %} + +{% block extra-sections %} + {% if object|can_edit:request.user %} +
+

{% trans 'Queries' %}

+
+ {% if object.query_set.count %} + + {% else %} +

{% trans 'No query are defined.' %}

+ {% endif %} +

+ {% trans 'New Query' %} +

+
+
+ {% endif %} +{% endblock %} diff --git a/passerelle/apps/arcgis/templates/arcgis/query_confirm_delete.html b/passerelle/apps/arcgis/templates/arcgis/query_confirm_delete.html new file mode 100644 index 00000000..a98a97ec --- /dev/null +++ b/passerelle/apps/arcgis/templates/arcgis/query_confirm_delete.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_confirm_delete.html" %} diff --git a/passerelle/apps/arcgis/templates/arcgis/query_form.html b/passerelle/apps/arcgis/templates/arcgis/query_form.html new file mode 100644 index 00000000..1c8a99b3 --- /dev/null +++ b/passerelle/apps/arcgis/templates/arcgis/query_form.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_form.html" %} diff --git a/passerelle/apps/arcgis/urls.py b/passerelle/apps/arcgis/urls.py new file mode 100644 index 00000000..a71be3e7 --- /dev/null +++ b/passerelle/apps/arcgis/urls.py @@ -0,0 +1,28 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +management_urlpatterns = [ + url(r'^(?P[\w,-]+)/query/new/$', + views.QueryNew.as_view(), name='arcgis-query-new'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/$', + views.QueryEdit.as_view(), name='arcgis-query-edit'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/delete/$', + views.QueryDelete.as_view(), name='arcgis-query-delete'), +] diff --git a/passerelle/apps/arcgis/views.py b/passerelle/apps/arcgis/views.py new file mode 100644 index 00000000..27dd6614 --- /dev/null +++ b/passerelle/apps/arcgis/views.py @@ -0,0 +1,50 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django import forms +from django.views.generic import UpdateView, CreateView, DeleteView + +from passerelle.base.mixins import ResourceChildViewMixin + +from . import models + + +class QueryForm(forms.ModelForm): + class Meta: + model = models.Query + fields = '__all__' + exclude = ['resource'] + + +class QueryNew(ResourceChildViewMixin, CreateView): + model = models.Query + form_class = QueryForm + + def form_valid(self, form): + form.instance.resource = self.resource + return super(QueryNew, self).form_valid(form) + + def get_changed_url(self): + return self.object.get_absolute_url() + + +class QueryEdit(ResourceChildViewMixin, UpdateView): + model = models.Query + form_class = QueryForm + + +class QueryDelete(ResourceChildViewMixin, DeleteView): + model = models.Query diff --git a/passerelle/utils/templates.py b/passerelle/utils/templates.py new file mode 100644 index 00000000..81c509f5 --- /dev/null +++ b/passerelle/utils/templates.py @@ -0,0 +1,50 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +'''Simplify rendering of template to produce text values. + +Disable autoescaping. +''' + +from django.core.exceptions import ValidationError + +from django.template.backends.django import DjangoTemplates +from django.template import TemplateSyntaxError +from django.utils.translation import ugettext as _ + + +def make_template(template_string): + engine = DjangoTemplates({ + 'NAME': 'django', + 'DIRS': [], + 'APP_DIRS': False, + 'OPTIONS': { + 'autoescape': False + }, + }) + return engine.from_string(template_string) + + +def render_to_string(template_string, context): + return make_template(template_string).render(context=context) + + +def validate_template(template_string): + try: + make_template(template_string) + except TemplateSyntaxError as e: + raise ValidationError(_('Invalid template: %s') % e) + diff --git a/tests/test_arcgis.py b/tests/test_arcgis.py index 3c197057..2445d735 100644 --- a/tests/test_arcgis.py +++ b/tests/test_arcgis.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + import pytest import mock import utils +from django.core.exceptions import ValidationError from django.contrib.contenttypes.models import ContentType -from passerelle.apps.arcgis.models import ArcGIS +from passerelle.apps.arcgis.models import ArcGIS, validate_where, SqlFormatter, Query from passerelle.base.models import ApiUser, AccessRight +from passerelle.utils import import_site # from http://sampleserver1.arcgisonline.com/ArcGIS/rest/services/fold/serv/MapServer/1 STATES = '''{ @@ -236,3 +240,111 @@ def test_arcgis_mapservice_query(app, arcgis): assert resp.json['err'] == 1 assert resp.json['err_class'] == 'passerelle.utils.jsonresponse.APIError' assert resp.json['err_desc'] == ' and must be floats' + + +@pytest.mark.parametrize('format_string,fail', [ + ('x {é}', True), + ('x {aa.bb}', True), + ('x {a:s} {b:d}', False), +]) +def test_validate_where(format_string, fail): + if fail: + with pytest.raises(ValidationError): + validate_where(format_string) + else: + validate_where(format_string) + + +@pytest.mark.parametrize('format_string,kwargs,expected', [ + ('adresse LIKE {adresse:s}', {'adresse': "AVENUE D'ANNAM"}, "adresse LIKE 'AVENUE D''ANNAM'"), + ('adresse LIKE {adresse:s} AND population < {pop:d}', { + 'adresse': "AVENUE D'ANNAM", + 'pop': '34', + }, "adresse LIKE 'AVENUE D''ANNAM' AND population < 34"), + ('adresse LIKE {adresse:s} AND population < {pop:d}', { + 'adresse': "AVENUE D'ANNAM", + 'pop': 'x', + }, ValueError), +]) +def test_sql_formatter(format_string, kwargs, expected): + formatter = SqlFormatter() + if not isinstance(expected, type) or not issubclass(expected, Exception): + assert formatter.format(format_string, **kwargs) == expected + else: + with pytest.raises(expected): + formatter.format(format_string, **kwargs) + + +@pytest.fixture +def query(arcgis): + return Query.objects.create( + resource=arcgis, + name='Adresses', + slug='adresses', + description='Recherche d\'une adresse', + id_template='{{ attributes.ident }}', + text_template='{{ attributes.address }} - {{ attributes.codepost }}', + folder='fold', + layer='1', + service='serv', + where='adress LIKE {adress:s}') + + +def test_query_q_method(arcgis, query, rf): + with mock.patch('passerelle.utils.Request.get') as requests_get: + requests_get.return_value = utils.FakedResponse(content=STATES, + status_code=200) + query.q(rf.get('/', data={'adress': "AVENUE D'ANNAM"}), full=True) == [] + assert requests_get.call_count == 1 + assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query' + args = requests_get.call_args[1]['params'] + assert args == { + 'f': 'json', + 'inSR': '4326', + 'outSR': '4326', + 'outFields': '*', + 'where': "adress LIKE 'AVENUE D''ANNAM'", + } + + +def test_q_endpoint(arcgis, query, app): + endpoint = utils.generic_endpoint_url('arcgis', 'q/adresses/', slug=arcgis.slug) + assert endpoint == '/arcgis/test/q/adresses/' + + with mock.patch('passerelle.utils.Request.get') as requests_get: + requests_get.return_value = utils.FakedResponse(content=STATES, + status_code=200) + + resp = app.get(endpoint, params={}, status=403) + assert requests_get.call_count == 0 + assert resp.json['err'] == 1 + assert resp.json['err_class'] == 'django.core.exceptions.PermissionDenied' + + # open access + api = ApiUser.objects.create(username='all', keytype='', key='') + obj_type = ContentType.objects.get_for_model(arcgis) + AccessRight.objects.create(codename='can_access', apiuser=api, resource_type=obj_type, + resource_pk=arcgis.pk) + resp = app.get(endpoint, params={'adress': "AVENUE D'ANNAM"}, status=200) + assert requests_get.call_count == 1 + assert requests_get.call_args[0][0] == 'https://arcgis.example.net/services/fold/serv/MapServer/1/query' + args = requests_get.call_args[1]['params'] + assert args == { + 'f': 'json', + 'inSR': '4326', + 'outSR': '4326', + 'outFields': '*', + 'where': "adress LIKE 'AVENUE D''ANNAM'", + } + + +def test_export_import(query): + assert ArcGIS.objects.count() == 1 + assert Query.objects.count() == 1 + serialization = {'resources': [query.resource.export_json()]} + ArcGIS.objects.all().delete() + assert ArcGIS.objects.count() == 0 + assert Query.objects.count() == 0 + import_site(serialization) + assert ArcGIS.objects.count() == 1 + assert Query.objects.count() == 1 -- 2.24.0