Produits Entr'ouvert » Passerelle

# -*- coding: utf-8 -*-

# Generated by Django 1.11.18 on 2020-05-15 17:28

from __future__ import unicode_literals

from django.db import migrations, models

import django.db.models.deletion

import passerelle.utils.templates

class Migration(migrations.Migration):

    initial = True

    dependencies = [

        ('base', '0020_auto_20200515_1923'),

    ]

    operations = [

        migrations.CreateModel(

            name='OpenDataSoft',

            fields=[

                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),

                ('title', models.CharField(max_length=50, verbose_name='Title')),

                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),

                ('description', models.TextField(verbose_name='Description')),

                ('service_url', models.CharField(help_text='OpenData Web Service URL', max_length=256, verbose_name='Service URL')),

                ('api_key', models.CharField(blank=True, help_text='API key used as credentials', max_length=128, verbose_name='API key')),

                ('users', models.ManyToManyField(blank=True, related_name='_opendatasoft_users_+', related_query_name='+', to='base.ApiUser')),

            ],

            options={

                'verbose_name': 'OpenDataSoft Web Service',

            },

        ),

        migrations.CreateModel(

            name='Query',

            fields=[

                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),

                ('name', models.CharField(max_length=128, verbose_name='Name')),

                ('slug', models.SlugField(max_length=128, verbose_name='Slug')),

                ('description', models.TextField(blank=True, verbose_name='Description')),

                ('dataset', models.CharField(help_text='dataset to query', max_length=128, verbose_name='Dataset')),

                ('text_template', models.TextField(blank=True, help_text="Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}", validators=[passerelle.utils.templates.validate_template], verbose_name='Text template')),

                ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='queries', to='opendatasoft.OpenDataSoft', verbose_name='Resource')),

            ],

            options={

                'verbose_name': 'Query',

                'ordering': ['name'],

                'abstract': False,

            },

        ),

        migrations.AlterUniqueTogether(

            name='query',

            unique_together=set([('resource', 'slug'), ('resource', 'name')]),

        ),

    ]

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2020  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.db import models

from django.shortcuts import get_object_or_404

from django.template import Context, Template

from django.core.urlresolvers import reverse

from django.utils.encoding import force_text

from django.utils.six.moves.urllib import parse as urlparse

from django.utils.translation import ugettext_lazy as _

from passerelle.utils.templates import validate_template

from passerelle.base.models import BaseResource, BaseQuery

from passerelle.utils.api import endpoint

class OpenDataSoft(BaseResource):

    service_url = models.CharField(

        _('Service URL'),

        max_length=256, blank=False,

        help_text=_('OpenData Web Service URL'),

    )

    api_key = models.CharField(

        _('API key'),

        max_length=128, blank=True,

        help_text=_('API key used as credentials'),

    )

    category = _('Data Sources')

    documentation_url = 'https://doc-publik.entrouvert.com/admin-fonctionnel/parametrage-avance/connecteur-opendadasoft/'

    class Meta:

        verbose_name = _('OpenDataSoft Web Service')

    @endpoint(

        perm='can_access',

        description=_('Search'),

        parameters={

            'dataset': {'description': _('Dataset')},

            'text_template': {'description': _('Text template')},

            'id': {'description': _('Record identifier')},

            'q': {'description': _('Full text query')},

            'limit': {'description': _('Maximum items')},

        })

    def search(self, request, dataset=None, text_template='', id=None, q=None, limit=None, **kwargs):

        scheme, netloc, path, params, query, fragment = urlparse.urlparse(self.service_url)

        path = urlparse.urljoin(path, 'api/records/1.0/search/')

        url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment))

        if id is not None:

            query = 'recordid:%s' % id

        else:

            query = q

        params = {

            'dataset': dataset,

            'q': query,

        }

        if self.api_key:

            params.update({'apikey': self.api_key})

        if limit:

            params.update({'rows': limit})

        result_response = self.requests.get(url, params=params)

        result = []

        for record in result_response.json().get('records'):

            data = {}

            data['id'] = record.get('recordid')

            context = {}

            for key, value in record.get('fields').items():

                context[key] = force_text(value)

            template = Template(text_template)

            data['text'] = template.render(Context(context)).strip()

            result.append(data)

        return {'data': result}

    @endpoint(name='q',

              description=_('Query'),

              pattern=r'^(?P<query_slug>[\w:_-]+)/$',

              perm='can_access',

              show=False)

    def q(self, request, query_slug, **kwargs):

        query = get_object_or_404(Query, resource=self, slug=query_slug)

        return query.q(request, **kwargs)

    def create_query_url(self):

        return reverse('opendatasoft-query-new', kwargs={'slug': self.slug})

class Query(BaseQuery):

    resource = models.ForeignKey(

        to=OpenDataSoft,

        related_name='queries',

        verbose_name=_('Resource'))

    dataset = models.CharField(

        _('Dataset'),

        max_length=128, blank=False,

        help_text=_('dataset to query'),

    )

    text_template = models.TextField(

        verbose_name=_('Text template'),

        help_text=_(

            "Use Django's template syntax. Attributes can be accessed through {{ attributes.name }}"

        ),

        validators=[validate_template],

        blank=True

    )

    delete_view = 'opendatasoft-query-delete'

    edit_view = 'opendatasoft-query-edit'

    def q(self, request, **kwargs):

        return self.resource.search(

            request, dataset=self.dataset, text_template=self.text_template, **kwargs)

    def as_endpoint(self):

        endpoint = super(Query, self).as_endpoint(path=self.resource.q.endpoint_info.name)

        search_endpoint = self.resource.search.endpoint_info

        endpoint.func = search_endpoint.func

        endpoint.show_undocumented_params = False

        # Copy generic params descriptions from original endpoint

        # if they are not overloaded by the query

        for param in search_endpoint.parameters:

            if param in ('dataset', 'text_template') and getattr(self, param):

                continue

            endpoint.parameters[param] = search_endpoint.parameters[param]

        return endpoint

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2020 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.conf.urls import url

from . import views

management_urlpatterns = [

    url(r'^(?P<slug>[\w,-]+)/query/new/$',

        views.QueryNew.as_view(), name='opendatasoft-query-new'),

    url(r'^(?P<slug>[\w,-]+)/query/(?P<pk>\d+)/$',

        views.QueryEdit.as_view(), name='opendatasoft-query-edit'),

    url(r'^(?P<slug>[\w,-]+)/query/(?P<pk>\d+)/delete/$',

        views.QueryDelete.as_view(), name='opendatasoft-query-delete'),

]

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2020 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django import forms

from django.views.generic import UpdateView, CreateView, DeleteView

from passerelle.base.mixins import ResourceChildViewMixin

from . import models

class QueryForm(forms.ModelForm):

    class Meta:

        model = models.Query

        fields = '__all__'

        exclude = ['resource']

class QueryNew(ResourceChildViewMixin, CreateView):

    model = models.Query

    form_class = QueryForm

    template_name = "passerelle/manage/resource_child_form.html"

    def form_valid(self, form):

        form.instance.resource = self.resource

        return super(QueryNew, self).form_valid(form)

class QueryEdit(ResourceChildViewMixin, UpdateView):

    model = models.Query

    form_class = QueryForm

    template_name = "passerelle/manage/resource_child_form.html"

class QueryDelete(ResourceChildViewMixin, DeleteView):

    model = models.Query

    template_name = "passerelle/manage/resource_child_confirm_delete.html"

    'passerelle.apps.opendatasoft',

li.connector.opendatasoft a::before {

	content: "\f1c0"; /* database */

}

# -*- coding: utf-8 -*-

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2020  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import mock

import json

import pytest

import utils

from passerelle.apps.opendatasoft.models import OpenDataSoft, Query

from test_manager import login, admin_user

FAKED_CONTENT_Q_SEARCH = json.dumps({

    "nhits": 76,

    "parameters": {

        "dataset": "referentiel-adresse-test",

        "format": "json",

        "q": "rue de l'aubepine",

        "rows": 3,

        "timezone": "UTC"

    },

    "records": [

        {

            "datasetid": "referentiel-adresse-test",

            "fields": {

                "adresse_complete": "33 RUE DE L'AUBEPINE STRASBOURG",

                "date_exprt": "2019-10-23",

                "geo_point": [

                    48.6060963542,

                    7.76978279836

                ],

                "nom_commun": "Strasbourg",

                "nom_rue": "RUE DE L'AUBEPINE",

                "num_com": 482,

                "numero": "33",

                "source": u"Ville et Eurométropole de Strasbourg"

            },

            "geometry": {

                "coordinates": [

                    7.76978279836,

                    48.6060963542

                ],

                "type": "Point"

            },

            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",

            "recordid": "e00cf6161e52a4c8fe510b2b74d4952036cb3473"

        },

        {

            "datasetid": "referentiel-adresse-test",

            "fields": {

                "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",

                "date_exprt": "2019-10-23",

                "geo_point": [

                    48.4920620548,

                    7.66177412454

                ],

                "nom_commun": "Lipsheim",

                "nom_rue": "RUE DE L'AUBEPINE",

                "num_com": 268,

                "numero": "19",

                "source": u"Ville et Eurométropole de Strasbourg"

            },

            "geometry": {

                "coordinates": [

                    7.66177412454,

                    48.4920620548

                ],

                "type": "Point"

            },

            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",

            "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"

        },

        {

            "datasetid": "referentiel-adresse-test",

            "fields": {

                "adresse_complete": "29 RUE DE L'AUBEPINE STRASBOURG",

                "date_exprt": "2019-10-23",

                "geo_point": [

                    48.6056497224,

                    7.76988497729

                ],

                "nom_commun": "Strasbourg",

                "nom_rue": "RUE DE L'AUBEPINE",

                "num_com": 482,

                "numero": "29",

                "source": u"Ville et Eurométropole de Strasbourg"

            },

            "geometry": {

                "coordinates": [

                    7.76988497729,

                    48.6056497224

                ],

                "type": "Point"

            },

            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",

            "recordid": "0984a5e1745701f71c91af73ce764e1f7132e0ff"

        }

    ]

})

FAKED_CONTENT_ID_SEARCH = json.dumps({

    "nhits": 1,

    "parameters": {

        "dataset": "referentiel-adresse-test",

        "format": "json",

        "q": "recordid:7cafcd5c692773e8b863587b2d38d6be82e023d8",

        "rows": 1,

        "timezone": "UTC"

    },

    "records": [

        {

            "datasetid": "referentiel-adresse-test",

            "fields": {

                "adresse_complete": "19 RUE DE L'AUBEPINE LIPSHEIM",

                "date_exprt": "2019-10-23",

                "geo_point": [

                    48.4920620548,

                    7.66177412454

                ],

                "nom_commun": "Lipsheim",

                "nom_rue": "RUE DE L'AUBEPINE",

                "num_com": 268,

                "numero": "19",

                u"source": "Ville et Eurométropole de Strasbourg"

            },

            "geometry": {

                "coordinates": [

                    7.66177412454,

                    48.4920620548

                ],

                "type": "Point"

            },

            "record_timestamp": "2019-12-02T14:15:08.376000+00:00",

            "recordid": "7cafcd5c692773e8b863587b2d38d6be82e023d8"

        }

    ]

})

@pytest.fixture

def connector(db):

    return utils.setup_access_rights(OpenDataSoft.objects.create(

        slug='my_connector',

        api_key='my_secret',

    ))

@pytest.fixture

def query(connector):

    return Query.objects.create(

        resource=connector,

        name='Référenciel adresses de test',

        slug='my_query',

        description='Rechercher une adresse',

        dataset='referentiel-adresse-test',

        text_template='{{numero}} {{nom_rue|safe}} {{nom_commun}}',

    )

def test_views(db, admin_user, app, connector):

    app = login(app)

    resp = app.get('/opendatasoft/my_connector/', status=200)

    resp = resp.click('New Query')

    resp.form['name'] = 'my query'

    resp.form['slug'] = 'my-query'

    resp.form['dataset'] = 'my-dataset'

    resp = resp.form.submit()

    resp = resp.follow()

    assert [x.text for x in resp.html.find_all('span', {'class': 'description'})]

@mock.patch('passerelle.utils.Request.get')

def test_search_using_q(mocked_get, app, connector):

    endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)

    assert endpoint == '/opendatasoft/my_connector/search'

    params = {

        'dataset': 'referentiel-adresse-test',

        'text_template': '{{numero}} {{nom_rue|safe}} {{nom_commun}}',

        'q': "rue de l'aubepine",

        'rows': 3,

    }

    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)

    resp = app.get(endpoint, params=params, status=200)

    assert not resp.json['err']

    assert len(resp.json['data']) == 3

    # order is keept

    assert resp.json['data'][0] == {

        'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',

        'text': "33 RUE DE L'AUBEPINE Strasbourg"

    }

    assert resp.json['data'][1] == {

        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

        'text': "19 RUE DE L'AUBEPINE Lipsheim"

    }

    assert resp.json['data'][2] == {

        'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',

        'text': "29 RUE DE L'AUBEPINE Strasbourg"

    }

@mock.patch('passerelle.utils.Request.get')

def test_search_using_id(mocked_get, app, connector):

    endpoint = utils.generic_endpoint_url('opendatasoft', 'search', slug=connector.slug)

    assert endpoint == '/opendatasoft/my_connector/search'

    params = {

        'dataset': 'referentiel-adresse-test',

        'text_template': '{{numero}} {{nom_rue|safe}} {{nom_commun}}',

        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

    }

    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)

    resp = app.get(endpoint, params=params, status=200)

    assert resp.json == {

        'err': 0,

        'data': [{

            'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

            'text': "19 RUE DE L'AUBEPINE Lipsheim"

        }]}

@mock.patch('passerelle.utils.Request.get')

def test_query_q_using_q(mocked_get, app, query):

    endpoint = '/opendatasoft/my_connector/q/my_query/'

    params = {

        'q': "rue de l'aubepine",

        'rows': 3,

    }

    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_Q_SEARCH, status_code=200)

    resp = app.get(endpoint, params=params, status=200)

    assert not resp.json['err']

    assert len(resp.json['data']) == 3

    # order is keept

    assert resp.json['data'][0] == {

        'id': 'e00cf6161e52a4c8fe510b2b74d4952036cb3473',

        'text': "33 RUE DE L'AUBEPINE Strasbourg"

    }

    assert resp.json['data'][1] == {

        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

        'text': "19 RUE DE L'AUBEPINE Lipsheim"

    }

    assert resp.json['data'][2] == {

        'id': '0984a5e1745701f71c91af73ce764e1f7132e0ff',

        'text': "29 RUE DE L'AUBEPINE Strasbourg"

    }

@mock.patch('passerelle.utils.Request.get')

def test_query_q_using_id(mocked_get, app, query):

    endpoint = '/opendatasoft/my_connector/q/my_query/'

    params = {

        'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

    }

    mocked_get.return_value = utils.FakedResponse(content=FAKED_CONTENT_ID_SEARCH, status_code=200)

    resp = app.get(endpoint, params=params, status=200)

    assert resp.json == {

        'err': 0,

        'data': [{

            'id': '7cafcd5c692773e8b863587b2d38d6be82e023d8',

            'text': "19 RUE DE L'AUBEPINE Lipsheim"

        }]}