0001-opendatasoft-add-opendatasoft-connector-40979.patch

Nicolas Roche, 12 avril 2020 13:47

Voir les différences: en ligne côte à côte

Subject: [PATCH] opendatasoft: add opendatasoft connector (#40979)

 passerelle/apps/opendatasoft/__init__.py      |   0
 .../opendatasoft/migrations/0001_initial.py   |  35 ++++
 .../apps/opendatasoft/migrations/__init__.py  |   0
 passerelle/apps/opendatasoft/models.py        | 103 +++++++++
 passerelle/settings.py                        |   1 +
 passerelle/static/css/style.css               |   5 +
 tests/test_opendatasoft.py                    | 198 ++++++++++++++++++
 7 files changed, 342 insertions(+)
 create mode 100644 passerelle/apps/opendatasoft/__init__.py
 create mode 100644 passerelle/apps/opendatasoft/migrations/0001_initial.py
 create mode 100644 passerelle/apps/opendatasoft/migrations/__init__.py
 create mode 100644 passerelle/apps/opendatasoft/models.py
 create mode 100644 tests/test_opendatasoft.py

     # -*- coding: utf-8 -*-
     # Generated by Django 1.11.18 on 2020-04-12 11:33
     from __future__ import unicode_literals
     from django.db import migrations, models
     class Migration(migrations.Migration):
         initial = True
         dependencies = [
             ('base', '0018_smslog'),
+        ]
         operations = [
             migrations.CreateModel(
                 name='OpenDataSoft',
                 fields=[
                     ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                     ('title', models.CharField(max_length=50, verbose_name='Title')),
                     ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
                     ('description', models.TextField(verbose_name='Description')),
                     ('service_url', models.CharField(default='https://examples.opendatasoft.com', help_text='OpenData Adresse Web Service URL', max_length=256, verbose_name='Service URL')),
                     ('api_key', models.CharField(blank=True, default='', help_text='API key used as credentials', max_length=128, verbose_name='API key')),
                     ('dataset', models.CharField(default='world-heritage-unesco-list', help_text='dataset to query', max_length=128, verbose_name='Dataset')),
                     ('text_value_template', models.CharField(default='{{name_fr}} en/au {{country_fr}}', help_text='String composed with item key variables ({{key}})', max_length=256, verbose_name='Template for "text" key value')),
                     ('max_rows', models.PositiveSmallIntegerField(default=15, help_text='Maximum number of items returned', verbose_name='Maximum items')),
                     ('users', models.ManyToManyField(blank=True, related_name='_opendatasoft_users_+', related_query_name='+', to='base.ApiUser')),
                 ],
                 options={
                     'verbose_name': 'OpenDataSoft Web Service',
                 },
             ),
+        ]

     # passerelle - uniform access to multiple data sources and services
     # Copyright (C) 2020  Entr'ouvert
+    #
     # This program is free software: you can redistribute it and/or modify it
     # under the terms of the GNU Affero General Public License as published
     # by the Free Software Foundation, either version 3 of the License, or
     # (at your option) any later version.
+    #
     # This program is distributed in the hope that it will be useful,
     # but WITHOUT ANY WARRANTY; without even the implied warranty of
     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     # GNU Affero General Public License for more details.
+    #
     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     from django.db import models
     from django.template import Context, Template
     from django.utils.encoding import force_text
     from django.utils.six.moves.urllib import parse as urlparse
     from django.utils.translation import ugettext_lazy as _
     from passerelle.base.models import BaseResource
     from passerelle.utils.api import endpoint
     class OpenDataSoft(BaseResource):
         service_url = models.CharField(
             _('Service URL'),
             max_length=256, blank=False,
             help_text=_('OpenData Adresse Web Service URL'),
             default='https://examples.opendatasoft.com',
+        )
         api_key = models.CharField(
             _('API key'),
             max_length=128, blank=True,
             help_text=_('API key used as credentials'),
             default='',
+        )
         dataset = models.CharField(
             _('Dataset'),
             max_length=128, blank=False,
             default='world-heritage-unesco-list',
             help_text=_('dataset to query'),
+        )
         text_value_template = models.CharField(
             _('Template for "text" key value'),
             max_length=256, blank=False,
             default='{{name_fr}} en/au {{country_fr}}',
             help_text=_('String composed with item key variables ({{key}})'),
+        )
         max_rows = models.PositiveSmallIntegerField(
             _('Maximum items'),
             default=15,
             help_text=_('Maximum number of items returned'))
         category = _('Data Sources')
         documentation_url = 'https://doc-publik.entrouvert.com/admin-fonctionnel/parametrage-avance/connecteur-opendadasoft/'
         class Meta:
             verbose_name = _('OpenDataSoft Web Service')
         @endpoint(
             perm='can_access',
             description=_('Search'),
             parameters={
                 'id': {'description': _('Record identifier')},
                 'q': {'description': _('Full text query'), 'example_value': "plage"},
         })
         def search(self, request, id=None, q=None, **kwargs):
             scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
             path = urlparse.urljoin(path, 'api/records/1.0/search/')
             url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
             if id is not None:
                 query = 'recordid:%s' % id
                 row = 1
             else:
                 query = q
                 row = self.max_rows
             params = {
                 'dataset': self.dataset,
                 'q': query,
                 'rows': row,
+            }
             if self.api_key:
                 params.update({'apikey': self.api_key})
             result_response = self.requests.get(url, params=params)
             result = []
             for record in result_response.json().get('records'):
                 data = {}
                 data['id'] = record.get('recordid')
                 context = {}
                 for key, value in record.get('fields').items():
                     context[key] = force_text(value)
                 template = Template(self.text_value_template)
                 data['text'] = template.render(Context(context)).strip()
                 result.append(data)
             return {'data': result}

         'passerelle.apps.opengis',
         'passerelle.apps.orange',
         'passerelle.apps.ovh',
         'passerelle.apps.oxyd',
         'passerelle.apps.pastell',
         'passerelle.apps.phonecalls',
         'passerelle.apps.solis',
         'passerelle.apps.vivaticket',
         'passerelle.apps.opendatasoft',
         # backoffice templates and static
         'gadjo',
+    )
     # disable some applications for now
     PASSERELLE_APP_BDP_ENABLED = False
     PASSERELLE_APP_GDC_ENABLED = False
     PASSERELLE_APP_PASTELL_ENABLED = False

     li.connector.dpark a::before {
     	content: "\f1b9";  /* car */
+    }
     li.connector.cryptor a::before {
     	content: "\f023";  /* lock */
+    }
     li.connector.opendatasoft a::before {
     	content: "\f1ad"; /* building */
+    }
     li.connector.status-down span.connector-name::after {
     	font-family: FontAwesome;
     	content: "\f00d"; /* times */
     	color: #CD2026;
     	padding-left: 1ex;
     	margin-left: 1ex;
     	border-left: 1px solid #003388;
+    }

     # -*- coding: utf-8 -*-
     # passerelle - uniform access to multiple data sources and services
     # Copyright (C) 2020  Entr'ouvert
+    #
     # This program is free software: you can redistribute it and/or modify it
     # under the terms of the GNU Affero General Public License as published
     # by the Free Software Foundation, either version 3 of the License, or
     # (at your option) any later version.
+    #
     # This program is distributed in the hope that it will be useful,
     # but WITHOUT ANY WARRANTY; without even the implied warranty of
     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     # GNU Affero General Public License for more details.
+    #
     # You should have received a copy of the GNU Affero General Public License
     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     import mock
     import json
     import pytest
     import utils
     from passerelle.apps.opendatasoft.models import OpenDataSoft
     FAKED_CONTENT_Q_SEARCH = json.dumps({
         "nhits": 76,
         "parameters": {
             "dataset": "referentiel-adresse-test",
             "format": "json",
             "q": "rue de l'aubepine",
             "rows": 3,
             "timezone": "UTC"
         },
         "records": [
+            {
                 "datasetid": "referentiel-adresse-test",
                 "fields": {
                     "adresse_complete": "33 RUE DE L'AUBEPINE STRASBOURG",
                     "date_exprt": "2019-10-23",
                     "geo_point": [
 .6060963542,
 .76978279836
                     ],
                     "nom_commun": "Strasbourg",
                     "nom_rue": "RUE DE L'AUBEPINE",
                     "num_com": 482,
                     "numero": "33",
                     "source": u"Ville et Eurométropole de Strasbourg"
                 },
                 "geometry": {
                     "coordinates": [
 .76978279836,
 .6060963542
                     ],
                     "type": "Point"
                 },
                 "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                 "recordid": "e00cf6161e52a4c8fe510b2b74d4952036cb3473"
             },
+            {
                 "datasetid": "referentiel-adresse-test",
                 "fields": {
                     "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
                     "date_exprt": "2019-10-23",
                     "geo_point": [
 .4920620548,
 .66177412454
                     ],
                     "nom_commun": "Lipsheim",
                     "nom_rue": "RUE DE L'AUBEPINE",
                     "num_com": 268,
                     "numero": "19",
                     "source": u"Ville et Eurométropole de Strasbourg"
                 },
                 "geometry": {
                     "coordinates": [
 .66177412454,
 .4920620548
                     ],
                     "type": "Point"
                 },
                 "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                 "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
             },
+            {
                 "datasetid": "referentiel-adresse-test",
                 "fields": {
                     "adresse_complete": "29 RUE DE L'AUBEPINE STRASBOURG",
                     "date_exprt": "2019-10-23",
                     "geo_point": [
 .6056497224,
 .76988497729
                     ],
                     "nom_commun": "Strasbourg",
                     "nom_rue": "RUE DE L'AUBEPINE",
                     "num_com": 482,
                     "numero": "29",
                     "source": u"Ville et Eurométropole de Strasbourg"
                 },
                 "geometry": {
                     "coordinates": [
 .76988497729,
 .6056497224
                     ],
                     "type": "Point"
                 },
                 "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                 "recordid": "0984a5e1745701f71c91af73ce764e1f7132e0ff"
+            }
+        ]
     })
     FAKED_CONTENT_ID_SEARCH = json.dumps({
         "nhits": 1,
         "parameters": {
             "dataset": "referentiel-adresse-test",
             "format": "json",
             "q": "recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8",
             "rows": 1,
             "timezone": "UTC"
         },
         "records": [
+            {
                 "datasetid": "referentiel-adresse-test",
                 "fields": {
                     "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
                     "date_exprt": "2019-10-23",
                     "geo_point": [
 .4920620548,
 .66177412454
                     ],
                     "nom_commun": "Lipsheim",
                     "nom_rue": "RUE DE L'AUBEPINE",
                     "num_com": 268,
                     "numero": "19",
                     u"source": "Ville et Eurométropole de Strasbourg"
                 },
                 "geometry": {
                     "coordinates": [
 .66177412454,
 .4920620548
                     ],
                     "type": "Point"
                 },
                 "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
                 "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
+            }
+        ]
     })
     @pytest.fixture
     def connector(db):
         return utils.setup_access_rights(
             OpenDataSoft.objects.create(
                 slug='my_connector',
                 dataset='referentiel-adresse-test',
                 text_value_template='{{numero}} {{nom_rue}} {{nom_commun}}',
                 max_rows=3
             ))
     @mock.patch('passerelle.utils.Request.get')
     def test_search_using_q(mocked_get, app, connector):
         endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
         assert endpoint == '/opendatasoft/my_connector/search'
         mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
         resp = app.get(endpoint, params={'q': "rue de l'aubepine"}, status=200)
         assert not resp.json['err']
         assert len(resp.json['data']) == 3
         # order is keept
         assert resp.json['data'][0] == {
             'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
             'text': "33 RUE DE L&#39;AUBEPINE Strasbourg"  # text is quoted
+        }
         assert resp.json['data'][1] == {
             'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
             'text': "19 RUE DE L&#39;AUBEPINE Lipsheim"
+        }
         assert resp.json['data'][2] == {
             'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',
             'text': "29 RUE DE L&#39;AUBEPINE Strasbourg"
+        }
     @mock.patch('passerelle.utils.Request.get')
     def test_search_using_id(mocked_get, app, connector):
         endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
         assert endpoint == '/opendatasoft/my_connector/search'
         mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
         resp = app.get(endpoint, params={'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8'}, status=200)
         assert resp.json == {
             'err': 0,
             'data': [{
                 'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
                 'text': "19 RUE DE L&#39;AUBEPINE Lipsheim"
             }]}
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0001-opendatasoft-add-opendatasoft-connector-40979.patch