From a76a67eb53a63b951d54c86ca90930d444d608ff Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 7 May 2021 01:48:25 +0200 Subject: [PATCH] add toulouse_smart connector (#53834) --- passerelle/contrib/toulouse_smart/__init__.py | 0 .../toulouse_smart/migrations/0001_initial.py | 92 +++++++++ .../toulouse_smart/migrations/__init__.py | 0 passerelle/contrib/toulouse_smart/models.py | 122 ++++++++++++ .../toulousesmartresource_detail.html | 11 ++ .../toulouse_smart/type-intervention.html | 48 +++++ .../templates/toulouse_smart/wcs_block.wcs | 22 +++ passerelle/contrib/toulouse_smart/urls.py | 32 ++++ passerelle/contrib/toulouse_smart/views.py | 70 +++++++ tests/settings.py | 1 + tests/test_toulouse_smart.py | 178 ++++++++++++++++++ 11 files changed, 576 insertions(+) create mode 100644 passerelle/contrib/toulouse_smart/__init__.py create mode 100644 passerelle/contrib/toulouse_smart/migrations/0001_initial.py create mode 100644 passerelle/contrib/toulouse_smart/migrations/__init__.py create mode 100644 passerelle/contrib/toulouse_smart/models.py create mode 100644 passerelle/contrib/toulouse_smart/templates/toulouse_smart/toulousesmartresource_detail.html create mode 100644 passerelle/contrib/toulouse_smart/templates/toulouse_smart/type-intervention.html create mode 100644 passerelle/contrib/toulouse_smart/templates/toulouse_smart/wcs_block.wcs create mode 100644 passerelle/contrib/toulouse_smart/urls.py create mode 100644 passerelle/contrib/toulouse_smart/views.py create mode 100644 tests/test_toulouse_smart.py diff --git a/passerelle/contrib/toulouse_smart/__init__.py b/passerelle/contrib/toulouse_smart/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/contrib/toulouse_smart/migrations/0001_initial.py b/passerelle/contrib/toulouse_smart/migrations/0001_initial.py new file mode 100644 index 00000000..afda7546 --- /dev/null +++ b/passerelle/contrib/toulouse_smart/migrations/0001_initial.py @@ -0,0 +1,92 @@ +# Generated by Django 2.2.19 on 2021-05-06 22:50 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('base', '0029_auto_20210202_1627'), + ] + + operations = [ + migrations.CreateModel( + name='ToulouseSmartResource', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('title', models.CharField(max_length=50, verbose_name='Title')), + ('slug', models.SlugField(unique=True, verbose_name='Identifier')), + ('description', models.TextField(verbose_name='Description')), + ( + 'basic_auth_username', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication username' + ), + ), + ( + 'basic_auth_password', + models.CharField( + blank=True, max_length=128, verbose_name='Basic authentication password' + ), + ), + ( + 'client_certificate', + models.FileField( + blank=True, null=True, upload_to='', verbose_name='TLS client certificate' + ), + ), + ( + 'trusted_certificate_authorities', + models.FileField(blank=True, null=True, upload_to='', verbose_name='TLS trusted CAs'), + ), + ( + 'verify_cert', + models.BooleanField(blank=True, default=True, verbose_name='TLS verify certificates'), + ), + ( + 'http_proxy', + models.CharField(blank=True, max_length=128, verbose_name='HTTP and HTTPS proxy'), + ), + ('webservice_base_url', models.URLField(verbose_name='Webservice Base URL')), + ( + 'users', + models.ManyToManyField( + blank=True, + related_name='_toulousesmartresource_users_+', + related_query_name='+', + to='base.ApiUser', + ), + ), + ], + options={ + 'verbose_name': 'Toulouse Smart', + }, + ), + migrations.CreateModel( + name='Cache', + fields=[ + ( + 'id', + models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ('key', models.CharField(max_length=64, verbose_name='Key')), + ('timestamp', models.DateTimeField(auto_now=True, verbose_name='Timestamp')), + ('value', django.contrib.postgres.fields.jsonb.JSONField(default=dict, verbose_name='Value')), + ( + 'resource', + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to='toulouse_smart.ToulouseSmartResource', + verbose_name='Resource', + ), + ), + ], + ), + ] diff --git a/passerelle/contrib/toulouse_smart/migrations/__init__.py b/passerelle/contrib/toulouse_smart/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/contrib/toulouse_smart/models.py b/passerelle/contrib/toulouse_smart/models.py new file mode 100644 index 00000000..3428e3bf --- /dev/null +++ b/passerelle/contrib/toulouse_smart/models.py @@ -0,0 +1,122 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime + +from django.db import models + +from django.utils.timezone import now +from django.utils.translation import ugettext_lazy as _ + +from django.contrib.postgres.fields import JSONField + +import lxml.etree as ET + +from passerelle.base.models import BaseResource, HTTPResource +from passerelle.utils import xml +from passerelle.utils.api import endpoint + + +class ToulouseSmartResource(BaseResource, HTTPResource): + category = _('Business Process Connectors') + + webservice_base_url = models.URLField(_('Webservice Base URL')) + + log_requests_errors = False + + class Meta: + verbose_name = _('Toulouse Smart') + + def get_intervention_types(self): + try: + return self.get('intervention-types', max_age=120) + except KeyError: + pass + + try: + url = self.webservice_base_url + 'v1/type-intervention' + response = self.requests.get(url) + doc = ET.fromstring(response.content) + intervention_types = [] + for xml_item in doc: + item = xml.to_json(xml_item) + for prop in item.get('properties', []): + prop['required'] = prop.get('required') == 'true' + intervention_types.append(item) + intervention_types.sort(key=lambda x: x['name']) + for i, intervention_type in enumerate(intervention_types): + intervention_type['order'] = i + 1 + except Exception: + try: + return self.get('intervention-types') + except KeyError: + raise + self.set('intervention-types', intervention_types) + return intervention_types + + def get(self, key, max_age=None): + cache_entries = self.cache_entries + if max_age: + cache_entries = cache_entries.filter(timestamp__gt=now() - datetime.timedelta(seconds=max_age)) + try: + return cache_entries.get(key='intervention-types').value + except Cache.DoesNotExist: + raise KeyError(key) + + def set(self, key, value): + self.cache_entries.update_or_create(key=key, defaults={'value': value}) + + @endpoint( + name='type-intervention', + description=_('Get intervention types'), + perm='can_access', + ) + def type_intervention(self, request): + try: + return { + 'data': [ + { + 'id': intervention_type['id'], + 'text': intervention_type['name'], + } + for intervention_type in self.get_intervention_types() + ] + } + except Exception: + return { + 'data': [ + { + 'id': '', + 'text': _('Service is unavailable'), + 'disabled': True, + } + ] + } + + +class Cache(models.Model): + resource = models.ForeignKey( + verbose_name=_('Resource'), + to=ToulouseSmartResource, + on_delete=models.CASCADE, + related_name='cache_entries', + ) + + key = models.CharField(_('Key'), max_length=64) + + timestamp = models.DateTimeField(_('Timestamp'), auto_now=True) + + value = JSONField(_('Value'), default=dict) diff --git a/passerelle/contrib/toulouse_smart/templates/toulouse_smart/toulousesmartresource_detail.html b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/toulousesmartresource_detail.html new file mode 100644 index 00000000..d4634420 --- /dev/null +++ b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/toulousesmartresource_detail.html @@ -0,0 +1,11 @@ +{% extends "passerelle/manage/service_view.html" %} +{% load i18n passerelle %} + +{% block extra-sections %} +
+

{% trans "Details" %}

+ +
+{% endblock %} diff --git a/passerelle/contrib/toulouse_smart/templates/toulouse_smart/type-intervention.html b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/type-intervention.html new file mode 100644 index 00000000..7705dbf4 --- /dev/null +++ b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/type-intervention.html @@ -0,0 +1,48 @@ +{% extends "passerelle/manage.html" %} +{% load i18n gadjo %} + +{% block breadcrumb %} +{{ block.super }} +{{ toulousesmartresource.title }} +{% trans 'Intervention types' %} +{% endblock %} + +{% block appbar %} +

{% trans 'Intervention types' %}

+ + {% trans "Export to blocks" %} + +{% endblock %} + +{% block content %} + + + + + + + + + + + + {% for intervention_type in toulousesmartresource.get_intervention_types %} + + {% for property in intervention_type.properties %} + + + + + + + + {% endfor %} + {% endfor %} + +
Nom du type d'interventionNomTypeRequisValeur par défaut
{{ intervention_type.order }} - {{ intervention_type.name }}
{{ property.name }}{{ property.type }}{{ property.required|yesno:"✔,✘" }}{{ property.defaultValue }}
+ + + +{% endblock %} diff --git a/passerelle/contrib/toulouse_smart/templates/toulouse_smart/wcs_block.wcs b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/wcs_block.wcs new file mode 100644 index 00000000..e630bfac --- /dev/null +++ b/passerelle/contrib/toulouse_smart/templates/toulouse_smart/wcs_block.wcs @@ -0,0 +1,22 @@ + + + {{ name }} + {{ name|slugify }} + + {% for property in properties %} + + {{ property.id }} + + {{ property.type }} + {{ property.required }} + {{ property.name|slugify }} + + validation + summary + {% if property.validation %} + + {{ property.validation }} + {% endif %} + {% endfor %} + + diff --git a/passerelle/contrib/toulouse_smart/urls.py b/passerelle/contrib/toulouse_smart/urls.py new file mode 100644 index 00000000..09fe5995 --- /dev/null +++ b/passerelle/contrib/toulouse_smart/urls.py @@ -0,0 +1,32 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from .views import TypeIntervention, TypeInterventionAsBlocks + +management_urlpatterns = [ + url( + r'^(?P[\w,-]+)/type-intervention/as-blocks/$', + TypeInterventionAsBlocks.as_view(), + name='toulouse-smart-type-intervention-as-blocks', + ), + url( + r'^(?P[\w,-]+)/type-intervention/$', + TypeIntervention.as_view(), + name='toulouse-smart-type-intervention', + ), +] diff --git a/passerelle/contrib/toulouse_smart/views.py b/passerelle/contrib/toulouse_smart/views.py new file mode 100644 index 00000000..53eb31dc --- /dev/null +++ b/passerelle/contrib/toulouse_smart/views.py @@ -0,0 +1,70 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import hashlib +import uuid +import zipfile + +from django.http import HttpResponse +from django.template.loader import render_to_string +from django.utils.text import slugify +from django.views.generic import DetailView + +from .models import ToulouseSmartResource + + +class TypeIntervention(DetailView): + model = ToulouseSmartResource + template_name = 'toulouse_smart/type-intervention.html' + + +class TypeInterventionAsBlocks(DetailView): + model = ToulouseSmartResource + + def get(self, request, *args, **kwargs): + self.object = self.get_object() + + def make_id(s): + return str(uuid.UUID(bytes=hashlib.md5(s.encode()).digest())) + + # generate file contents + files = {} + for intervention_type in self.object.get_intervention_types(): + slug = slugify(intervention_type['name']) + # only export intervention_type with properties + if not intervention_type.get('properties'): + continue + for prop in intervention_type['properties']: + # generate a natural id for fields + prop['id'] = make_id(slug + slugify(prop['name'])) + # adapt types + prop.setdefault('type', 'string') + if prop['type'] == 'boolean': + prop['type'] = 'bool' + if prop['type'] == 'int': + prop['type'] = 'string' + prop['validation'] = 'digits' + filename = 'block-%s.wcs' % slug + files[filename] = render_to_string('toulouse_smart/wcs_block.wcs', context=intervention_type) + + # zip it ! + response = HttpResponse(content_type='application/zip') + response['Content-Disposition'] = 'attachment; filename=blocks.zip' + with zipfile.ZipFile(response, mode='w') as zip_file: + for name in files: + zip_file.writestr(name, files[name]) + + return response diff --git a/tests/settings.py b/tests/settings.py index a491db10..07573f9c 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -39,6 +39,7 @@ INSTALLED_APPS += ( 'passerelle.contrib.teamnet_axel', 'passerelle.contrib.tcl', 'passerelle.contrib.toulouse_axel', + 'passerelle.contrib.toulouse_smart', 'passerelle.contrib.lille_kimoce', ) diff --git a/tests/test_toulouse_smart.py b/tests/test_toulouse_smart.py new file mode 100644 index 00000000..56cf00f3 --- /dev/null +++ b/tests/test_toulouse_smart.py @@ -0,0 +1,178 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import functools +import io +import zipfile + +import lxml.etree as ET +import httmock +import pytest + +import utils + +from passerelle.contrib.toulouse_smart.models import ToulouseSmartResource +from passerelle.utils.xml import to_json + +from test_manager import login + + +@pytest.fixture +def smart(db): + return utils.make_resource( + ToulouseSmartResource, + title='Test', + slug='test', + description='Test', + webservice_base_url='https://smart.example.com/', + basic_auth_username='username', + basic_auth_password='password', + ) + + +def mock_response(*path_contents): + def decorator(func): + @httmock.urlmatch() + def error(url, request): + assert False, 'request to %s' % (url,) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + handlers = [] + for row in path_contents: + path, content = row + + @httmock.urlmatch(path=path) + def handler(url, request): + return content + + handlers.append(handler) + handlers.append(error) + + with httmock.HTTMock(*handlers): + return func(*args, **kwargs) + + return wrapper + + return decorator + + +@mock_response(['/v1/type-intervention', b'']) +def test_empty_intervention_types(smart): + assert smart.get_intervention_types() == [] + + +INTERVENTION_TYPES = b''' + + 1234 + coin + + + FIELD1 + string + false + + + FIELD2 + int + true + + + +''' + + +@mock_response(['/v1/type-intervention', INTERVENTION_TYPES]) +def test_model_intervention_types(smart): + assert smart.get_intervention_types() == [ + { + 'id': '1234', + 'name': 'coin', + 'order': 1, + 'properties': [ + {'name': 'FIELD1', 'required': False, 'type': 'string'}, + {'name': 'FIELD2', 'required': True, 'type': 'int'}, + ], + }, + ] + + +URL = '/toulouse-smart/test/' + + +@mock_response(['/v1/type-intervention', INTERVENTION_TYPES]) +def test_endpoint_intervention_types(app, smart): + resp = app.get(URL + 'type-intervention') + assert resp.json == {'data': [{'id': '1234', 'text': 'coin'}], 'err': 0} + + +@mock_response() +def test_endpoint_intervention_types_unavailable(app, smart): + resp = app.get(URL + 'type-intervention') + assert resp.json == {'data': [{'id': '', 'text': 'Service is unavailable', 'disabled': True}], 'err': 0} + + +@mock_response(['/v1/type-intervention', INTERVENTION_TYPES]) +def test_manage_intervention_types(app, smart, admin_user): + login(app) + resp = app.get('/manage' + URL + 'type-intervention/') + assert [[td.text for td in tr.cssselect('td,th')] for tr in resp.pyquery('tr')] == [ + ["Nom du type d'intervention", 'Nom', 'Type', 'Requis', 'Valeur par défaut'], + ['1 - coin'], + [None, 'FIELD1', 'string', '✘', None], + [None, 'FIELD2', 'int', '✔', None], + ] + resp = resp.click('Export to blocks') + with zipfile.ZipFile(io.BytesIO(resp.body)) as zip_file: + assert zip_file.namelist() == ['block-coin.wcs'] + with zip_file.open('block-coin.wcs') as fd: + content = ET.tostring(ET.fromstring(fd.read()), pretty_print=True).decode() + assert ( + content + == ''' + coin + coin + + + + 038a8c2e-14de-4d4f-752f-496eb7fe90d7 + + string + False + field1 + + validation + summary + + + + e72f251a-5eef-5b78-c35a-94b549510029 + + string + True + field2 + + validation + summary + + + digits + + + + +''' + ) -- 2.31.1