From 5641f26bd8c562a315cdbbcc7733aaddde544235 Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Mon, 30 Mar 2020 18:38:04 +0200
Subject: [PATCH] opendatasoft: add opendatasoft connector (#40979)

---
 passerelle/apps/opendatasoft/__init__.py      |   0
 .../opendatasoft/migrations/0001_initial.py   |  35 ++++
 .../apps/opendatasoft/migrations/__init__.py  |   0
 passerelle/apps/opendatasoft/models.py        | 102 +++++++++
 passerelle/settings.py                        |   1 +
 passerelle/static/css/style.css               |   5 +
 tests/test_opendatasoft.py                    | 198 ++++++++++++++++++
 7 files changed, 341 insertions(+)
 create mode 100644 passerelle/apps/opendatasoft/__init__.py
 create mode 100644 passerelle/apps/opendatasoft/migrations/0001_initial.py
 create mode 100644 passerelle/apps/opendatasoft/migrations/__init__.py
 create mode 100644 passerelle/apps/opendatasoft/models.py
 create mode 100644 tests/test_opendatasoft.py

diff --git a/passerelle/apps/opendatasoft/__init__.py b/passerelle/apps/opendatasoft/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/opendatasoft/migrations/0001_initial.py b/passerelle/apps/opendatasoft/migrations/0001_initial.py
new file mode 100644
index 00000000..39955aa2
--- /dev/null
+++ b/passerelle/apps/opendatasoft/migrations/0001_initial.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2020-03-31 08:29
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('base', '0018_smslog'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='OpenDataSoft',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=50, verbose_name='Title')),
+                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),
+                ('description', models.TextField(verbose_name='Description')),
+                ('service_url', models.CharField(default='https://test-eurometrostrasbourg.opendatasoft.com/api/records/1.0', help_text='OpenData Adresse Web Service URL', max_length=256, verbose_name='Service URL')),
+                ('api_key', models.CharField(default='881fb55527190ac221186a44226f1615f5444bce22c17f4bd074857e', help_text='API key used as credentials', max_length=128, verbose_name='API key')),
+                ('dataset', models.CharField(default='referentiel-adresse-test', help_text='dataset to query', max_length=128, verbose_name='Dataset')),
+                ('text_value_template', models.CharField(default='{{numero}} {{nom_rue}} {{num_commune}}', help_text='String composed with item key variables ({{key}})', max_length=256, verbose_name='Template for "text" key value')),
+                ('max_rows', models.PositiveSmallIntegerField(default=7, help_text='Maximum number of items returned', verbose_name='Maximum items')),
+                ('users', models.ManyToManyField(blank=True, related_name='_opendatasoft_users_+', related_query_name='+', to='base.ApiUser')),
+            ],
+            options={
+                'verbose_name': 'OpenDataSoft Web Service',
+            },
+        ),
+    ]
diff --git a/passerelle/apps/opendatasoft/migrations/__init__.py b/passerelle/apps/opendatasoft/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/passerelle/apps/opendatasoft/models.py b/passerelle/apps/opendatasoft/models.py
new file mode 100644
index 00000000..03d4b3de
--- /dev/null
+++ b/passerelle/apps/opendatasoft/models.py
@@ -0,0 +1,102 @@
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2020  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.db import models
+from django.template import Context, Template
+from django.utils.encoding import force_text
+from django.utils.six.moves.urllib import parse as urlparse
+from django.utils.translation import ugettext_lazy as _
+
+from passerelle.base.models import BaseResource
+from passerelle.utils.api import endpoint
+
+
+class OpenDataSoft(BaseResource):
+    service_url = models.CharField(
+        _('Service URL'),
+        max_length=256, blank=False,
+        help_text=_('OpenData Adresse Web Service URL'),
+        default='https://test-eurometrostrasbourg.opendatasoft.com',
+    )
+    api_key = models.CharField(
+        _('API key'),
+        max_length=128, blank=False,
+        help_text=_('API key used as credentials'),
+        default='881fb55527190ac221186a44226f1615f5444bce22c17f4bd074857e',
+    )
+    dataset = models.CharField(
+        _('Dataset'),
+        max_length=128, blank=False,
+        default='referentiel-adresse-test',
+        help_text=_('dataset to query'),
+    )
+    text_value_template = models.CharField(
+        _('Template for "text" key value'),
+        max_length=256, blank=False,
+        default='{{numero}} {{nom_rue}} {{nom_commun}}',
+        help_text=_('String composed with item key variables ({{key}})'),
+    )
+    max_rows = models.PositiveSmallIntegerField(
+        _('Maximum items'),
+        default=15,
+        help_text=_('Maximum number of items returned'))
+
+    category = _('Data Sources')
+    documentation_url = 'https://doc-publik.entrouvert.com/admin-fonctionnel/parametrage-avance/connecteur-opendadasoft/'
+
+    class Meta:
+        verbose_name = _('OpenDataSoft Web Service')
+
+    @endpoint(
+        perm='can_access',
+        description=_('Search'),
+        parameters={
+            'id': {'description': _('Record identifier')},
+            'q': {'description': _('Full text query'), 'example_value': "rue de l'aubepine"},
+    })
+    def search(self, request, id=None, q=None, **kwargs):
+        scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)
+        path = urlparse.urljoin(path, 'api/records/1.0/search/')
+        url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+
+        if id is not None:
+            query = 'recordid:%s' % id
+            row = 1
+        else:
+            query = q
+            row = self.max_rows
+        params = {
+            'dataset': self.dataset,
+            'apikey': self.api_key,
+            'q': query,
+            'rows': row,
+        }
+
+        result_response = self.requests.get(url, params=params)
+
+        result = []
+        for record in result_response.json().get('records'):
+            data = {}
+            data['id'] = record.get('recordid')
+
+            context = {}
+            for key, value in record.get('fields').items():
+                context[key] = force_text(value)
+            template = Template(self.text_value_template)
+            data['text'] = template.render(Context(context)).strip()
+
+            result.append(data)
+        return {'data': result}
diff --git a/passerelle/settings.py b/passerelle/settings.py
index 04173804..ebeb8d9c 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -149,16 +149,17 @@ INSTALLED_APPS = (
     'passerelle.apps.opengis',
     'passerelle.apps.orange',
     'passerelle.apps.ovh',
     'passerelle.apps.oxyd',
     'passerelle.apps.pastell',
     'passerelle.apps.phonecalls',
     'passerelle.apps.solis',
     'passerelle.apps.vivaticket',
+    'passerelle.apps.opendatasoft',
     # backoffice templates and static
     'gadjo',
 )
 
 # disable some applications for now
 PASSERELLE_APP_BDP_ENABLED = False
 PASSERELLE_APP_GDC_ENABLED = False
 PASSERELLE_APP_PASTELL_ENABLED = False
diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css
index d6c9672d..a1a3ae31 100644
--- a/passerelle/static/css/style.css
+++ b/passerelle/static/css/style.css
@@ -176,16 +176,21 @@ li.connector.apientreprise a::before {
 li.connector.dpark a::before {
 	content: "\f1b9";  /* car */
 }
 
 li.connector.cryptor a::before {
 	content: "\f023";  /* lock */
 }
 
+li.connector.opendatasoft a::before {
+	content: "\f1ad"; /* building */
+}
+
+
 li.connector.status-down span.connector-name::after {
 	font-family: FontAwesome;
 	content: "\f00d"; /* times */
 	color: #CD2026;
 	padding-left: 1ex;
 	margin-left: 1ex;
 	border-left: 1px solid #003388;
 }
diff --git a/tests/test_opendatasoft.py b/tests/test_opendatasoft.py
new file mode 100644
index 00000000..19ffc76d
--- /dev/null
+++ b/tests/test_opendatasoft.py
@@ -0,0 +1,198 @@
+# -*- coding: utf-8 -*-
+# passerelle - uniform access to multiple data sources and services
+# Copyright (C) 2020  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import mock
+import json
+import pytest
+
+import utils
+
+from passerelle.apps.opendatasoft.models import OpenDataSoft
+
+FAKED_CONTENT_Q_SEARCH = json.dumps({
+    "nhits": 76,
+    "parameters": {
+        "dataset": "referentiel-adresse-test",
+        "format": "json",
+        "q": "rue de l'aubepine",
+        "rows": 3,
+        "timezone": "UTC"
+    },
+    "records": [
+        {
+            "datasetid": "referentiel-adresse-test",
+            "fields": {
+                "adresse_complete": "33 RUE DE L'AUBEPINE STRASBOURG",
+                "date_exprt": "2019-10-23",
+                "geo_point": [
+                    48.6060963542,
+                    7.76978279836
+                ],
+                "nom_commun": "Strasbourg",
+                "nom_rue": "RUE DE L'AUBEPINE",
+                "num_com": 482,
+                "numero": "33",
+                "source": u"Ville et Eurométropole de Strasbourg"
+            },
+            "geometry": {
+                "coordinates": [
+                    7.76978279836,
+                    48.6060963542
+                ],
+                "type": "Point"
+            },
+            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
+            "recordid": "e00cf6161e52a4c8fe510b2b74d4952036cb3473"
+        },
+        {
+            "datasetid": "referentiel-adresse-test",
+            "fields": {
+                "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
+                "date_exprt": "2019-10-23",
+                "geo_point": [
+                    48.4920620548,
+                    7.66177412454
+                ],
+                "nom_commun": "Lipsheim",
+                "nom_rue": "RUE DE L'AUBEPINE",
+                "num_com": 268,
+                "numero": "19",
+                "source": u"Ville et Eurométropole de Strasbourg"
+            },
+            "geometry": {
+                "coordinates": [
+                    7.66177412454,
+                    48.4920620548
+                ],
+                "type": "Point"
+            },
+            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
+            "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
+        },
+        {
+            "datasetid": "referentiel-adresse-test",
+            "fields": {
+                "adresse_complete": "29 RUE DE L'AUBEPINE STRASBOURG",
+                "date_exprt": "2019-10-23",
+                "geo_point": [
+                    48.6056497224,
+                    7.76988497729
+                ],
+                "nom_commun": "Strasbourg",
+                "nom_rue": "RUE DE L'AUBEPINE",
+                "num_com": 482,
+                "numero": "29",
+                "source": u"Ville et Eurométropole de Strasbourg"
+            },
+            "geometry": {
+                "coordinates": [
+                    7.76988497729,
+                    48.6056497224
+                ],
+                "type": "Point"
+            },
+            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
+            "recordid": "0984a5e1745701f71c91af73ce764e1f7132e0ff"
+        }
+    ]
+})
+
+FAKED_CONTENT_ID_SEARCH = json.dumps({
+    "nhits": 1,
+    "parameters": {
+        "dataset": "referentiel-adresse-test",
+        "format": "json",
+        "q": "recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8",
+        "rows": 1,
+        "timezone": "UTC"
+    },
+    "records": [
+        {
+            "datasetid": "referentiel-adresse-test",
+            "fields": {
+                "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",
+                "date_exprt": "2019-10-23",
+                "geo_point": [
+                    48.4920620548,
+                    7.66177412454
+                ],
+                "nom_commun": "Lipsheim",
+                "nom_rue": "RUE DE L'AUBEPINE",
+                "num_com": 268,
+                "numero": "19",
+                u"source": "Ville et Eurométropole de Strasbourg"
+            },
+            "geometry": {
+                "coordinates": [
+                    7.66177412454,
+                    48.4920620548
+                ],
+                "type": "Point"
+            },
+            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",
+            "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"
+        }
+    ]
+})
+
+
+@pytest.fixture
+def connector(db):
+    return utils.setup_access_rights(
+        OpenDataSoft.objects.create(
+            slug='my_connector',
+            dataset='referentiel-adresse-test',
+            text_value_template='{{numero}} {{nom_rue}} {{nom_commun}}',
+            max_rows=3
+        ))
+
+
+@mock.patch('passerelle.utils.Request.get')
+def test_search_using_q(mocked_get, app, connector):
+    endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
+    assert endpoint == '/opendatasoft/my_connector/search'
+    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)
+    resp = app.get(endpoint, params={'q': "rue de l'aubepine"}, status=200)
+    assert not resp.json['err']
+    assert len(resp.json['data']) == 3
+    # order is keept
+    assert resp.json['data'][0] == {
+        'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',
+        'text': "33 RUE DE L&#39;AUBEPINE Strasbourg"  # text is quoted
+    }
+    assert resp.json['data'][1] == {
+        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
+        'text': "19 RUE DE L&#39;AUBEPINE Lipsheim"
+    }
+    assert resp.json['data'][2] == {
+        'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',
+        'text': "29 RUE DE L&#39;AUBEPINE Strasbourg"
+    }
+
+
+@mock.patch('passerelle.utils.Request.get')
+def test_search_using_id(mocked_get, app, connector):
+    endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)
+    assert endpoint == '/opendatasoft/my_connector/search'
+    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)
+    resp = app.get(endpoint, params={'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8'}, status=200)
+    assert resp.json == {
+        'err': 0,
+        'data': [{
+            'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',
+            'text': "19 RUE DE L&#39;AUBEPINE Lipsheim"
+        }]}
-- 
2.25.1

