From 6d2ce216b4269ecc60dc6304e0a781ee25086a42 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 24 Mar 2020 16:36:19 +0100 Subject: [PATCH 4/6] opengis: add query system for features (#20535) --- passerelle/apps/opengis/forms.py | 37 ++++++++ .../migrations/0007_auto_20200324_1019.py | 44 ++++++++++ passerelle/apps/opengis/models.py | 85 +++++++++++++++++-- .../opengis/query_confirm_delete.html | 1 + .../opengis/templates/opengis/query_form.html | 1 + passerelle/apps/opengis/urls.py | 28 ++++++ passerelle/apps/opengis/views.py | 43 ++++++++++ 7 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 passerelle/apps/opengis/forms.py create mode 100644 passerelle/apps/opengis/migrations/0007_auto_20200324_1019.py create mode 100644 passerelle/apps/opengis/templates/opengis/query_confirm_delete.html create mode 100644 passerelle/apps/opengis/templates/opengis/query_form.html create mode 100644 passerelle/apps/opengis/urls.py create mode 100644 passerelle/apps/opengis/views.py diff --git a/passerelle/apps/opengis/forms.py b/passerelle/apps/opengis/forms.py new file mode 100644 index 00000000..fe67e47a --- /dev/null +++ b/passerelle/apps/opengis/forms.py @@ -0,0 +1,37 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from xml.etree import ElementTree as ET + +from django import forms +from django.utils.translation import ugettext_lazy as _ + +from . import models + + +class QueryForm(forms.ModelForm): + class Meta: + model = models.Query + fields = '__all__' + exclude = ['resource'] + + def clean_filter_expression(self): + filter_expression = self.cleaned_data['filter_expression'] + try: + ET.fromstring(filter_expression) + except ET.ParseError: + raise forms.ValidationError(_('Filter is not valid XML.')) + return filter_expression diff --git a/passerelle/apps/opengis/migrations/0007_auto_20200324_1019.py b/passerelle/apps/opengis/migrations/0007_auto_20200324_1019.py new file mode 100644 index 00000000..58126d9a --- /dev/null +++ b/passerelle/apps/opengis/migrations/0007_auto_20200324_1019.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-03-24 09:19 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('opengis', '0006_auto_20181118_0807'), + ] + + operations = [ + migrations.CreateModel( + name='Query', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=128, verbose_name='Name')), + ('slug', models.SlugField(max_length=128, verbose_name='Slug')), + ('description', models.TextField(blank=True, verbose_name='Description')), + ('typename', models.CharField(blank=True, max_length=256, verbose_name='Feature type')), + ('filter_expression', models.CharField(blank=True, max_length=4096, verbose_name='XML filter')), + ], + options={ + 'ordering': ['name'], + }, + ), + migrations.AlterField( + model_name='opengis', + name='projection', + field=models.CharField(choices=[('EPSG:2154', 'EPSG:2154 (Lambert-93)'), ('EPSG:3857', 'EPSG:3857 (WGS 84 / Pseudo-Mercator)'), ('EPSG:3945', 'EPSG:3945 (CC45)'), ('EPSG:4326', 'EPSG:4326 (WGS 84)')], default='EPSG:3857', max_length=16, verbose_name='GIS projection'), + ), + migrations.AddField( + model_name='query', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='opengis.OpenGIS', verbose_name='Resource'), + ), + migrations.AlterUniqueTogether( + name='query', + unique_together=set([('resource', 'name'), ('resource', 'slug')]), + ), + ] diff --git a/passerelle/apps/opengis/models.py b/passerelle/apps/opengis/models.py index c1df901f..84142929 100644 --- a/passerelle/apps/opengis/models.py +++ b/passerelle/apps/opengis/models.py @@ -24,11 +24,13 @@ import pyproj from django.core.cache import cache from django.db import models from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from django.urls import reverse from django.utils.six.moves.html_parser import HTMLParser from django.utils.text import slugify from django.utils.translation import ugettext_lazy as _ -from passerelle.base.models import BaseResource +from passerelle.base.models import BaseResource, BaseQuery from passerelle.utils.api import endpoint from passerelle.utils.jsonresponse import APIError @@ -144,18 +146,24 @@ class OpenGIS(BaseResource): 'case-insensitive': { 'description': _('Enables case-insensitive search'), 'example_value': 'true' + }, + 'xml_filter': { + 'description': _('Filter applied to the query'), + 'example_value': 'typeparking' + '' } }) def features(self, request, type_names, property_name, cql_filter=None, - filter_property_name=None, q=None, **kwargs): + filter_property_name=None, q=None, xml_filter=None, **kwargs): params = { 'VERSION': self.get_wfs_service_version(), 'SERVICE': 'WFS', 'REQUEST': 'GetFeature', self.get_typename_label(): type_names, - 'PROPERTYNAME': property_name, 'OUTPUTFORMAT': 'json', } + if property_name: + params['PROPERTYNAME'] = property_name if cql_filter: params.update({'CQL_FILTER': cql_filter}) if filter_property_name and q: @@ -164,16 +172,18 @@ class OpenGIS(BaseResource): else: operator = 'LIKE' params['CQL_FILTER'] += ' AND %s %s \'%%%s%%\'' % (filter_property_name, operator, q) + if xml_filter: + params['FILTER'] = xml_filter response = self.requests.get(self.wfs_service_url, params=params) data = [] try: - response.json() + response = response.json() except ValueError: self.handle_opengis_error(response) # if handle_opengis_error did not raise an error, we raise a generic one raise APIError(u'OpenGIS Error: unparsable error', data={'content': repr(response.content[:1024])}) - for feature in response.json()['features']: + for feature in response['features']: feature['text'] = feature['properties'].get(property_name) data.append(feature) return {'data': data} @@ -343,3 +353,68 @@ class OpenGIS(BaseResource): break return result raise APIError('Unable to geocode') + + @endpoint(name='q', + description=_('Query'), + pattern=r'^(?P[\w:_-]+)/$', + perm='can_access', + show=False) + def q(self, request, query_slug): + query = get_object_or_404(Query, resource=self, slug=query_slug) + return query.q(request) + + def export_json(self): + d = super(OpenGIS, self).export_json() + d['queries'] = [query.export_json() for query in self.query_set.all()] + return d + + @classmethod + def import_json_real(cls, overwrite, instance, d, **kwargs): + queries = d.pop('queries', []) + instance = super(OpenGIS, cls).import_json_real(overwrite, instance, d, **kwargs) + new = [] + if instance and overwrite: + Query.objects.filter(resource=instance).delete() + for query in queries: + query['resource'] = instance + q = Query.import_json(query) + new.append(q) + Query.objects.bulk_create(new) + return instance + + def create_query_url(self): + return reverse('opengis-query-new', kwargs={'slug': self.slug}) + + +class Query(BaseQuery): + resource = models.ForeignKey( + to=OpenGIS, + on_delete=models.CASCADE, + verbose_name=_('Resource')) + + typename = models.CharField( + verbose_name=_('Feature type'), + max_length=256, + blank=True) + filter_expression = models.CharField( + verbose_name=_('XML filter'), + max_length=4096, + blank=True) + + delete_view = 'opengis-query-delete' + edit_view = 'opengis-query-edit' + + def __init__(self, *args, **kwargs): + super(Query, self).__init__(*args, **kwargs) + self.endpoint = self.resource.features + self.called_from = self.resource.q + + def as_endpoint(self): + endpoint = super(Query, self).as_endpoint() + # include no original parameters + endpoint.exclude_params = list(self.endpoint.endpoint_info.parameters) + return endpoint + + def q(self, request): + return self.endpoint(request, self.typename, property_name=None, + xml_filter=self.filter_expression) diff --git a/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html b/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html new file mode 100644 index 00000000..a98a97ec --- /dev/null +++ b/passerelle/apps/opengis/templates/opengis/query_confirm_delete.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_confirm_delete.html" %} diff --git a/passerelle/apps/opengis/templates/opengis/query_form.html b/passerelle/apps/opengis/templates/opengis/query_form.html new file mode 100644 index 00000000..1c8a99b3 --- /dev/null +++ b/passerelle/apps/opengis/templates/opengis/query_form.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_form.html" %} diff --git a/passerelle/apps/opengis/urls.py b/passerelle/apps/opengis/urls.py new file mode 100644 index 00000000..3e68b735 --- /dev/null +++ b/passerelle/apps/opengis/urls.py @@ -0,0 +1,28 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +management_urlpatterns = [ + url(r'^(?P[\w,-]+)/query/new/$', + views.QueryNew.as_view(), name='opengis-query-new'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/$', + views.QueryEdit.as_view(), name='opengis-query-edit'), + url(r'^(?P[\w,-]+)/query/(?P\d+)/delete/$', + views.QueryDelete.as_view(), name='opengis-query-delete'), +] diff --git a/passerelle/apps/opengis/views.py b/passerelle/apps/opengis/views.py new file mode 100644 index 00000000..5f22ec82 --- /dev/null +++ b/passerelle/apps/opengis/views.py @@ -0,0 +1,43 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2020 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.views.generic import UpdateView, CreateView, DeleteView + +from passerelle.base.mixins import ResourceChildViewMixin + +from . import models +from .forms import QueryForm + + +class QueryNew(ResourceChildViewMixin, CreateView): + model = models.Query + form_class = QueryForm + + def form_valid(self, form): + form.instance.resource = self.resource + return super(QueryNew, self).form_valid(form) + + def get_changed_url(self): + return self.object.get_absolute_url() + + +class QueryEdit(ResourceChildViewMixin, UpdateView): + model = models.Query + form_class = QueryForm + + +class QueryDelete(ResourceChildViewMixin, DeleteView): + model = models.Query -- 2.20.1