From 9601c4c5d25b41e4932da65b977c23ea927de4a2 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 24 Mar 2020 16:36:19 +0100 Subject: [PATCH 3/5] opengis: add query system for features (#20535) --- passerelle/apps/opengis/forms.py | 38 +++++++++ .../migrations/0007_auto_20200401_1032.py | 45 ++++++++++ passerelle/apps/opengis/models.py | 84 +++++++++++++++++-- .../opengis/query_confirm_delete.html | 1 + .../opengis/templates/opengis/query_form.html | 1 + passerelle/apps/opengis/urls.py | 28 +++++++ passerelle/apps/opengis/views.py | 43 ++++++++++ 7 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 passerelle/apps/opengis/forms.py create mode 100644 passerelle/apps/opengis/migrations/0007_auto_20200401_1032.py create mode 100644 passerelle/apps/opengis/templates/opengis/query_confirm_delete.html create mode 100644 passerelle/apps/opengis/templates/opengis/query_form.html create mode 100644 passerelle/apps/opengis/urls.py create mode 100644 passerelle/apps/opengis/views.py diff --git a/passerelle/apps/opengis/forms.py b/passerelle/apps/opengis/forms.py new file mode 100644 index 00000000..f867a15f --- /dev/null +++ b/passerelle/apps/opengis/forms.py @@ -0,0 +1,38 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from xml.etree import ElementTree as ET + +from django import forms +from django.utils.translation import ugettext_lazy as _ + +from . import models + + +class QueryForm(forms.ModelForm): + class Meta: + model = models.Query + fields = '__all__' + exclude = ['resource'] + + def clean_filter_expression(self): + filter_expression = self.cleaned_data['filter_expression'] + if filter_expression: + try: + ET.fromstring(filter_expression) + except ET.ParseError: + raise forms.ValidationError(_('Filter is not valid XML.')) + return filter_expression diff --git a/passerelle/apps/opengis/migrations/0007_auto_20200401_1032.py b/passerelle/apps/opengis/migrations/0007_auto_20200401_1032.py new file mode 100644 index 00000000..65f2f80e --- /dev/null +++ b/passerelle/apps/opengis/migrations/0007_auto_20200401_1032.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-04-08 09:54 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('opengis', '0006_auto_20181118_0807'), + ] + + operations = [ + migrations.CreateModel( + name='Query', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('slug', models.SlugField(max_length=128, verbose_name='Slug')), + ('description', models.TextField(blank=True, verbose_name='Description')), + ('typename', models.CharField(max_length=256, verbose_name='Feature type')), + ('filter_expression', models.TextField(blank=True, verbose_name='XML filter')), + ], + options={ + 'ordering': ['name'], + 'abstract': False, + }, + ), + migrations.AlterField( + model_name='opengis', + name='projection', + field=models.CharField(choices=[('EPSG:2154', 'EPSG:2154 (Lambert-93)'), ('EPSG:3857', 'EPSG:3857 (WGS 84 / Pseudo-Mercator)'), ('EPSG:3945', 'EPSG:3945 (CC45)'), ('EPSG:4326', 'EPSG:4326 (WGS 84)')], default='EPSG:3857', max_length=16, verbose_name='GIS projection'), + ), + migrations.AddField( + model_name='query', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queries', to='opengis.OpenGIS', verbose_name='Resource'), + ), + migrations.AlterUniqueTogether( + name='query', + unique_together=set([('resource', 'name'), ('resource', 'slug')]), + ), + ] diff --git a/passerelle/apps/opengis/models.py b/passerelle/apps/opengis/models.py index 2d12b128..b1bdf02a 100644 --- a/passerelle/apps/opengis/models.py +++ b/passerelle/apps/opengis/models.py @@ -24,11 +24,13 @@ import pyproj from django.core.cache import cache from django.db import models from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.urls import reverse from django.utils.six.moves.html_parser import HTMLParser from django.utils.text import slugify from django.utils.translation import ugettext_lazy as _ -from passerelle.base.models import BaseResource +from passerelle.base.models import BaseResource, BaseQuery from passerelle.utils.api import endpoint from passerelle.utils.conversion import num2deg from passerelle.utils.jsonresponse import APIError @@ -143,17 +145,21 @@ class OpenGIS(BaseResource): }) response.raise_for_status() - def build_get_features_params(self, typename=None, property_name=None, cql_filter=None): + def build_get_features_params(self, typename=None, property_name=None, cql_filter=None, + xml_filter=None): params = { 'version': self.get_wfs_service_version(), 'service': 'WFS', 'request': 'GetFeature', self.get_typename_label(): typename, - 'propertyName': property_name, 'outputFormat': self.get_output_format(), } + if property_name: + params['propertyName'] = property_name if cql_filter: params['cql_filter'] = cql_filter + if xml_filter: + params['filter'] = xml_filter return params @endpoint(perm='can_access', description='Get features', @@ -180,10 +186,16 @@ class OpenGIS(BaseResource): }, 'case_insensitive': { 'description': _('Enables case-insensitive search'), + }, + 'xml_filter': { + 'description': _('Filter applied to the query'), + 'example_value': 'typeparking' + '' } }) def features(self, request, type_names, property_name, cql_filter=None, - filter_property_name=None, q=None, case_insensitive=False, **kwargs): + filter_property_name=None, q=None, case_insensitive=False, + xml_filter=None, **kwargs): if cql_filter: if filter_property_name and q: if 'case-insensitive' in kwargs or case_insensitive: @@ -191,17 +203,17 @@ class OpenGIS(BaseResource): else: operator = 'LIKE' cql_filter += ' AND %s %s \'%%%s%%\'' % (filter_property_name, operator, q) - params = self.build_get_features_params(type_names, property_name, cql_filter) + params = self.build_get_features_params(type_names, property_name, cql_filter, xml_filter) response = self.requests.get(self.wfs_service_url, params=params) data = [] try: - response.json() + response = response.json() except ValueError: self.handle_opengis_error(response) # if handle_opengis_error did not raise an error, we raise a generic one raise APIError(u'OpenGIS Error: unparsable error', data={'content': repr(response.content[:1024])}) - for feature in response.json()['features']: + for feature in response['features']: feature['text'] = feature['properties'].get(property_name) data.append(feature) return {'data': data} @@ -356,3 +368,61 @@ class OpenGIS(BaseResource): break return result raise APIError('Unable to geocode') + + @endpoint(name='query', + description=_('Query'), + pattern=r'^(?P[\w:_-]+)/$', + perm='can_access', + show=False) + def query(self, request, query_slug): + query = get_object_or_404(Query, resource=self, slug=query_slug) + return query.q(request) + + def export_json(self): + d = super(OpenGIS, self).export_json() + d['queries'] = [query.export_json() for query in self.queries.all()] + return d + + @classmethod + def import_json_real(cls, overwrite, instance, d, **kwargs): + queries = d.pop('queries', []) + instance = super(OpenGIS, cls).import_json_real(overwrite, instance, d, **kwargs) + new = [] + if instance and overwrite: + Query.objects.filter(resource=instance).delete() + for query in queries: + q = Query.import_json(query) + q.resource = instance + new.append(q) + Query.objects.bulk_create(new) + return instance + + def create_query_url(self): + return reverse('opengis-query-new', kwargs={'slug': self.slug}) + + +class Query(BaseQuery): + resource = models.ForeignKey( + to=OpenGIS, + on_delete=models.CASCADE, + related_name='queries', + verbose_name=_('Resource')) + + typename = models.CharField(_('Feature type'), max_length=256) + filter_expression = models.TextField(_('XML filter'), blank=True) + + delete_view = 'opengis-query-delete' + edit_view = 'opengis-query-edit' + + def as_endpoint(self): + resource_endpoint = self.resource.query.endpoint_info + endpoint = super(Query, self).as_endpoint(path=resource_endpoint.name) + # use parameters added by resource endpoint + endpoint.func = resource_endpoint.func + endpoint.show_undocumented_params = False + endpoint.parameters = resource_endpoint.parameters + return endpoint + + def q(self, request): + return self.resource.features(request, self.typename, property_name=None, + xml_filter=self.filter_expression) diff --git a/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html b/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html new file mode 100644 index 00000000..a98a97ec --- /dev/null +++ b/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_confirm_delete.html" %} diff --git a/passerelle/apps/opengis/templates/opengis/query_form.html b/passerelle/apps/opengis/templates/opengis/query_form.html new file mode 100644 index 00000000..1c8a99b3 --- /dev/null +++ b/passerelle/apps/opengis/templates/opengis/query_form.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_form.html" %} diff --git a/passerelle/apps/opengis/urls.py b/passerelle/apps/opengis/urls.py new file mode 100644 index 00000000..3e68b735 --- /dev/null +++ b/passerelle/apps/opengis/urls.py @@ -0,0 +1,28 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +management_urlpatterns = [ + url(r'^(?P[\w,-]+)/query/new/$', + views.QueryNew.as_view(), name='opengis-query-new'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/$', + views.QueryEdit.as_view(), name='opengis-query-edit'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/delete/$', + views.QueryDelete.as_view(), name='opengis-query-delete'), +] diff --git a/passerelle/apps/opengis/views.py b/passerelle/apps/opengis/views.py new file mode 100644 index 00000000..5f22ec82 --- /dev/null +++ b/passerelle/apps/opengis/views.py @@ -0,0 +1,43 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.views.generic import UpdateView, CreateView, DeleteView + +from passerelle.base.mixins import ResourceChildViewMixin + +from . import models +from .forms import QueryForm + + +class QueryNew(ResourceChildViewMixin, CreateView): + model = models.Query + form_class = QueryForm + + def form_valid(self, form): + form.instance.resource = self.resource + return super(QueryNew, self).form_valid(form) + + def get_changed_url(self): + return self.object.get_absolute_url() + + +class QueryEdit(ResourceChildViewMixin, UpdateView): + model = models.Query + form_class = QueryForm + + +class QueryDelete(ResourceChildViewMixin, DeleteView): + model = models.Query -- 2.20.1