From 401873e3f796b184f707fd80131bfa59b9c6f798 Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 7 Apr 2020 16:38:26 +0200 Subject: [PATCH] opengis: add text search for queries (#40743) --- .../migrations/0009_auto_20200407_1544.py | 26 ++++++++++++++++ passerelle/apps/opengis/models.py | 28 +++++++++++++---- passerelle/settings.py | 1 + tests/test_opengis.py | 30 ++++++++++++++++++- 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 passerelle/apps/opengis/migrations/0009_auto_20200407_1544.py diff --git a/passerelle/apps/opengis/migrations/0009_auto_20200407_1544.py b/passerelle/apps/opengis/migrations/0009_auto_20200407_1544.py new file mode 100644 index 00000000..a8e20add --- /dev/null +++ b/passerelle/apps/opengis/migrations/0009_auto_20200407_1544.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.18 on 2020-04-07 13:44 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('opengis', '0008_featurecache'), + ] + + operations = [ + migrations.AddField( + model_name='featurecache', + name='text', + field=models.CharField(default='', max_length=2048), + preserve_default=False, + ), + migrations.AddField( + model_name='query', + name='index_properties', + field=models.CharField(blank=True, help_text='Comma separated list such as property1,property2', max_length=1024, verbose_name='Properties for searching'), + ), + ] diff --git a/passerelle/apps/opengis/models.py b/passerelle/apps/opengis/models.py index 9ed4303a..3af58dfa 100644 --- a/passerelle/apps/opengis/models.py +++ b/passerelle/apps/opengis/models.py @@ -34,7 +34,7 @@ from django.utils.translation import ugettext_lazy as _ from passerelle.base.models import BaseResource, BaseQuery from passerelle.utils.api import endpoint -from passerelle.utils.conversion import num2deg +from passerelle.utils.conversion import num2deg, simplify from passerelle.utils.jsonresponse import APIError @@ -397,12 +397,16 @@ class OpenGIS(BaseResource): 'bbox': { 'description': _('Only include results inside bounding box'), 'example_value': '-0.489,51.28,0.236,51.686' + }, + 'q': { + 'description': _('Text search for specified properties'), + 'example_value': 'library' } }, show=False) - def query(self, request, query_slug, bbox=None): + def query(self, request, query_slug, bbox=None, q=None): query = get_object_or_404(Query, resource=self, slug=query_slug) - return query.q(request, bbox) + return query.q(request, bbox, q) def export_json(self): d = super(OpenGIS, self).export_json() @@ -452,6 +456,11 @@ class Query(BaseQuery): typename = models.CharField(_('Feature type'), max_length=256) filter_expression = models.TextField(_('XML filter'), blank=True) + index_properties = models.CharField( + verbose_name=_('Properties for searching'), + help_text=_('Comma separated list such as property1,property2'), + max_length=1024, + blank=True) delete_view = 'opengis-query-delete' edit_view = 'opengis-query-edit' @@ -465,7 +474,7 @@ class Query(BaseQuery): endpoint.parameters = resource_endpoint.parameters return endpoint - def q(self, request, bbox): + def q(self, request, bbox, q): features = self.features.all() if not features.exists(): raise APIError('Data is not synchronized yet. Retry in a few minutes.') @@ -477,6 +486,8 @@ class Query(BaseQuery): 'floating point numbers of the form lonmin,latmin,lonmax,latmax') features = features.filter(lon__gte=lonmin, lon__lte=lonmax, lat__gte=latmin, lat__lte=latmax) + if q: + features = features.filter(text__search=simplify(q)) data = { 'type': 'FeatureCollection', 'name': self.typename @@ -494,7 +505,13 @@ class Query(BaseQuery): except (KeyError, TypeError): self.resource.logger.warning('invalid coordinates in geometry: %s', geometry) continue - features.append(FeatureCache(query=self, lat=lat, lon=lon, data=feature)) + text = '' + if self.index_properties: + properties = [x.strip() for x in self.index_properties.split(',') if x.strip()] + values = [val for prop, val in feature.get('properties', {}).items() + if prop in properties] + text = simplify(' '.join(values)) + features.append(FeatureCache(query=self, lat=lat, lon=lon, text=text, data=feature)) with transaction.atomic(): self.features.all().delete() FeatureCache.objects.bulk_create(features) @@ -512,4 +529,5 @@ class FeatureCache(models.Model): verbose_name=_('Query')) lat = models.FloatField() lon = models.FloatField() + text = models.CharField(max_length=2048) data = JSONField() diff --git a/passerelle/settings.py b/passerelle/settings.py index 04173804..46aed2e5 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -115,6 +115,7 @@ INSTALLED_APPS = ( 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.admin', + 'django.contrib.postgres', # base app 'passerelle.base', # connectors diff --git a/tests/test_opengis.py b/tests/test_opengis.py index 91a952e1..c34dc152 100644 --- a/tests/test_opengis.py +++ b/tests/test_opengis.py @@ -641,7 +641,35 @@ def test_opengis_query_endpoint_documentation(mocked_get, app, connector, query) resp = app.get(connector.get_absolute_url()) assert query.name in resp.text assert query.description in resp.text - assert '/opengis/test/query/test_query/?bbox=' in resp.text + assert '/opengis/test/query/test_query/' in resp.text + assert 'bbox=-0.489,51.28,0.236,51.686' in resp.text + assert 'q=library' in resp.text + + +@mock.patch('passerelle.utils.Request.get') +def test_opengis_query_text_search(mocked_get, app, connector, query): + endpoint = utils.generic_endpoint_url('opengis', 'query/test_query/', slug=connector.slug) + mocked_get.side_effect = geoserver_geolocated_responses + query.update_cache() + + resp = app.get(endpoint + '?q=grenoble') + assert len(resp.json['features']) == 0 + + query.index_properties = 'nom_commune,nom_voie' + query.save() + query.update_cache() + + resp = app.get(endpoint + '?q=grenoble') + assert len(resp.json['features']) == 4 + + resp = app.get(endpoint + '?q=victor') + assert len(resp.json['features']) == 2 + + resp = app.get(endpoint + '?q=hugo victor') + assert len(resp.json['features']) == 2 + + resp = app.get(endpoint + '?q=nomatch') + assert len(resp.json['features']) == 0 def test_opengis_export_import(query): -- 2.20.1