From ee4725c683911942a5ba77b2fad0c5daf5dc503c Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 29 Mar 2019 16:42:14 +0100 Subject: [PATCH 10/10] initialize sp_fr connector (#31595) New connector for transfering forms from Service-Public.fr to w.c.s. --- passerelle/apps/sp_fr/DOC.XSD | 108 ++++ passerelle/apps/sp_fr/RCO.XSD | 137 +++++ passerelle/apps/sp_fr/__init__.py | 0 passerelle/apps/sp_fr/forms.py | 69 +++ passerelle/apps/sp_fr/models.py | 514 ++++++++++++++++++ .../variable_and_expression_widget.html | 6 + .../sp_fr/mapping_confirm_delete.html | 1 + .../sp_fr/templates/sp_fr/mapping_form.html | 44 ++ .../templates/sp_fr/resource_detail.html | 39 ++ passerelle/apps/sp_fr/urls.py | 30 + passerelle/apps/sp_fr/views.py | 67 +++ passerelle/settings.py | 1 + passerelle/static/css/style.css | 6 + tests/test_sp_fr.py | 65 +++ 14 files changed, 1087 insertions(+) create mode 100644 passerelle/apps/sp_fr/DOC.XSD create mode 100644 passerelle/apps/sp_fr/RCO.XSD create mode 100644 passerelle/apps/sp_fr/__init__.py create mode 100644 passerelle/apps/sp_fr/forms.py create mode 100644 passerelle/apps/sp_fr/models.py create mode 100644 passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html create mode 100644 passerelle/apps/sp_fr/urls.py create mode 100644 passerelle/apps/sp_fr/views.py create mode 100644 tests/test_sp_fr.py diff --git a/passerelle/apps/sp_fr/DOC.XSD b/passerelle/apps/sp_fr/DOC.XSD new file mode 100644 index 00000000..9bd73815 --- /dev/null +++ b/passerelle/apps/sp_fr/DOC.XSD @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/sp_fr/RCO.XSD b/passerelle/apps/sp_fr/RCO.XSD new file mode 100644 index 00000000..339e3cd1 --- /dev/null +++ b/passerelle/apps/sp_fr/RCO.XSD @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/sp_fr/__init__.py b/passerelle/apps/sp_fr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/sp_fr/forms.py b/passerelle/apps/sp_fr/forms.py new file mode 100644 index 00000000..639eea16 --- /dev/null +++ b/passerelle/apps/sp_fr/forms.py @@ -0,0 +1,69 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django import forms + +from . import models, fields + + +class MappingForm(forms.ModelForm): + + def __init__(self, *args, **kwargs): + super(MappingForm, self).__init__(*args, **kwargs) + if self.instance.procedure and self.instance and self.instance.formdef: + choices = [('', '--------')] + [(v, v) for v in self.instance.variables] + for i, field in enumerate(self.schema_fields()): + label = field.label + label += ' (%s)' % (field.varname or 'NO VARNAME') + base_name = str(field.varname or i) + initial = self.instance.rules.get('fields', {}).get(base_name) + self.fields['field_%s' % base_name] = fields.VariableAndExpressionField( + label=label, + choices=choices, + initial=initial, + required=False) + + def table_fields(self): + return [field for field in self if field.name.startswith('field_')] + + def schema_fields(self): + if self.instance and self.instance.formdef: + schema = self.instance.formdef.schema + for i, field in enumerate(schema.fields): + if field.type in ('page', 'comment', 'title', 'subtitle'): + continue + yield field + + def save(self, commit=True): + fields = {} + for key in self.cleaned_data: + if not key.startswith('field_'): + continue + if not self.cleaned_data[key]: + continue + real_key = key[len('field_'):] + value = self.cleaned_data[key].copy() + value['label'] = self.fields[key].label + fields[real_key] = value + self.instance.rules['fields'] = fields + return super(MappingForm, self).save(commit=commit) + + class Meta: + model = models.Mapping + fields = [ + 'procedure', + 'formdef', + ] diff --git a/passerelle/apps/sp_fr/models.py b/passerelle/apps/sp_fr/models.py new file mode 100644 index 00000000..dadb0826 --- /dev/null +++ b/passerelle/apps/sp_fr/models.py @@ -0,0 +1,514 @@ +# -*- coding: utf-8 -*- +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re +import os +import stat +import zipfile +import collections +import io +import base64 +import datetime + +from lxml import etree as ET + +from django.core.urlresolvers import reverse +from django.db import models, transaction +from django.template import engines +from django.utils.translation import ugettext_lazy as _ + +from jsonfield import JSONField + +from passerelle.base.models import BaseResource +from passerelle.utils.api import endpoint +from passerelle.utils.sftp import SFTPField +from passerelle.utils.wcs import FormDefField, get_wcs_choices +from passerelle.utils.xsd import Schema +from passerelle.utils.xml import text_content + + +PROCEDURE_DOC = 'DOC' +PROCEDURE_RCO = 'RCO' +PROCEDURE_DDPACS = 'DDPACS' +PROCEDURES = [ + (PROCEDURE_DOC, _('Request for construction site opening')), + (PROCEDURE_RCO, _('Request for mandatory citizen census')), + (PROCEDURE_DDPACS, _('Pre-request for citizen solidarity pact')), +] + +ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier') + + +def simplify(s): + ''' + Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9) + and minimize spaces. Used to compare strings on ?q=something requests. + ''' + if not s: + return '' + if not isinstance(s, six.text_type): + s = six.text_type(s, 'utf-8', 'ignore') + s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') + s = re.sub(r'[^\w\s\'-_]', '', s) + s = re.sub(r'[\s\']+', ' ', s) + return s.strip().lower() + + +class Resource(BaseResource): + category = _('Business Process Connectors') + + input_sftp = SFTPField( + verbose_name=_('Input SFTP URL'), + null=True) + + output_sftp = SFTPField( + verbose_name=_('Output SFTP URL'), + null=True) + + def check_status(self): + with self.input_sftp as sftp: + sftp.listdir() + with self.output_sftp as sftp: + sftp.listdir() + get_wcs_choices(self.requests) + + @endpoint(name='ping', description=_('Check Solis API availability')) + def ping(self, request): + self.check_status() + return {'err': 0} + + def run_loop(self, count=1): + with transaction.atomic(): + # lock resource + r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk) + if not r: + # already locked + self.logger.info('did nothing') + return + with self.input_sftp as sftp: + try: + sftp.lstat('DONE') + except IOError: + sftp.mkdir('DONE') + + def helper(): + for file_stat in sftp.listdir_attr(): + if stat.S_ISDIR(file_stat.st_mode): + continue + yield file_stat.filename + + mappings = {mapping.procedure: mapping for mapping in self.mappings.all()} + + for filename, i in zip(helper(), range(count)): + self.handle_filename(filename, sftp, mappings) + + def handle_filename(self, filename, sftp, mappings): + nsmap = { + 'dgme-metier': 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier' + } + routage_xpath = ET.XPath( + ('dgme-metier:Routage/dgme-metier:Donnee/dgme-metier:Valeur/text()'), + namespaces=nsmap) + m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename) + if not m: + self.logger.warning('found file with an unknown pattern %s moving in DONE/', filename) + return + mdel_identifier, procedure, mdel_sequence = m.groups() + if procedure not in mappings: + self.logger.warning('found file for an unsupported procedure %s', procedure) + return + mapping = mappings[procedure] + self.logger.info('found %s %s %s, handling', mdel_identifier, procedure, mdel_sequence) + with sftp.open(filename) as fd: + try: + archive = zipfile.ZipFile(fd) + except Exception: + self.logger.error('could not load zipfile %s', filename) + return + doc_files = [] + ent_files = [] + attachments = {} + for name in archive.namelist(): + if re.match(r'^.*-ent-\d+(?:-.*)?.xml$', name): + ent_files.append(name) + if re.match(r'^.*-doc-\d+-XML-\d+(?:-.*)?\.xml$', name): + doc_files.append(name) + m = re.match(r'^.*-pj-([^-]+)-\d+\.([^.]+)$', name) + if m: + attachment_type, extension = m.groups() + attachments.setdefault(attachment_type, []).append(name) + if len(ent_files) != 1: + self.logger.warning('too many/few ent files found: %s', ent_files) + return + if len(doc_files) != 1: + self.logger.warning('too many/few doc files found: %s', doc_files) + return + for key in attachments: + if len(attachments[key]) > 1: + self.logger.warning('too many attachments of kind %s: %s', key, attachments[key]) + name = attachments[key][0] + content = archive.open(attachments[key][0]).read() + attachments[key] = { + 'filename': name, + 'content': base64.b64encode(content).decode('ascii'), + 'content_type': 'application/octet-stream', + } + if procedure == 'RCO' and not attachments: + self.logger.warning('no attachments but RCO requires them') + return + ent_file = ent_files[0] + doc_file = doc_files[0] + + with archive.open(ent_file) as fd: + document = ET.parse(fd) + insee_codes = routage_xpath(document) + if len(insee_codes) != 1: + self.logger.warning('too many/few insee codes found: %s', insee_codes) + return + insee_code = insee_codes[0] + data = {'insee_code': insee_code} + data.update(attachments) + with archive.open(doc_file) as fd: + document = ET.parse(fd) + data.update(self.extract_data(document)) + if hasattr(self, 'update_data_%s' % procedure): + getattr(self, 'update_data_%s' % procedure)(mapping, data) + + formdef = mapping.formdef + formdef.session = self.requests + + with formdef.submit() as submitter: + submitter.submission_channel = 'web' + submitter.submission_context = { + 'mdel_procedure': procedure, + 'mdel_identifier': mdel_identifier, + 'mdel_sequence': mdel_sequence, + } + for field in mapping.rules.get('fields', {}): + variable = mapping.rules['fields'][field]['variable'] + expression = mapping.rules['fields'][field]['expression'] + value = data.get(variable) + if expression.strip(): + template = engines['django'].from_string(expression) + context = data.copy() + context['value'] = value + value = template.render(context) + submitter.set(field, value) + import pprint + print 'Payload' + pprint.pprint(submitter.payload()) + + for key in sorted(data): + if not isinstance(data[key], dict): + print key, data[key] + else: + print key, '', data[key]['filename'] + + def update_data_DOC(self, mapping, data): + variables = list(mapping.variables) + assert all(key in variables for key in data) + + def get(name): + # prevent error in manual mapping + if name not in variables: + print '\n'.join(sorted(variables)) + assert False, name + return data.get(name, '') + + numero_permis_construire = get('doc_declarant_designation_permis_numero_permis_construire') + numero_permis_amenager = get('doc_declarant_designation_permis_numero_permis_amenager') + data['type_permis'] = u'Un permis de construire' if numero_permis_construire else u'Un permis d\'aménager' + data['numero_permis'] = numero_permis_construire or numero_permis_amenager + particulier = get('doc_declarant_identite_type_personne').strip().lower() == 'true' + data['type_declarant'] = u'Un particulier' if particulier else u'Une personne morale' + if particulier: + data['nom'] = get('doc_declarant_identite_personne_physique_nom') + data['prenoms'] = get('doc_declarant_identite_personne_physique_prenom') + else: + data['nom'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_nom') + data['prenoms'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_prenom') + mapping = { + '1000': 'Monsieur', + '1001': 'Madame', + '1002': 'Madame et Monsieur', + } + if particulier: + data['civilite_particulier'] = mapping.get(get('doc_declarant_identite_personne_physique_civilite'), '') + else: + data['civilite_pm'] = mapping.get( + get('doc_declarant_identite_personne_morale_representant_personne_morale_civilite'), '') + data['portee'] = (u'Pour la totalité des travaux' + if get('doc_ouverture_chantier_totalite_travaux').lower().strip() == 'true' + else u'Pour une tranche des travaux') + assert all(key in variables for key in data) + + def update_data_RCO(self, mapping, data): + variables = list(mapping.variables) + assert all(key in variables for key in data) + + def get(name): + # prevent error in manual mapping + if name not in variables: + print '\n'.join(sorted(variables)) + assert False, name + return data.get(name, '') + + motif = ( + get('recensementcitoyen_formalite_formalitemotifcode_1') + or get('recensementcitoyen_formalite_formalitemotifcode_2') + ) + data['motif'] = { + 'RECENSEMENT': '1', + 'EXEMPTION': '2' + }[motif] + if data['motif'] == '2': + data['motif_exempte'] = ( + u"Titulaire d'une carte d'invalidité de 80% minimum" + if get('recensementcitoyen_formalite_formalitemotifcode_2') == 'INFIRME' + else u"Autre situation") + data['justificatif_exemption'] = get('je') + data['double_nationalite'] = ( + 'Oui' + if get('recensementcitoyen_personne_nationalite') + else 'Non') + data['residence_differente'] = ( + 'Oui' + if get('recensementcitoyen_personne_adresseresidence_localite') + else 'Non') + data['civilite'] = ( + 'Monsieur' + if get('recensementcitoyen_personne_civilite') == 'M' + else 'Madame' + ) + + def get_lieu_naissance(variable, code): + for idx in ['', '_1', '_2']: + v = variable + idx + if get(v + '_code') == code: + return get(v + '_nom') + + data['cp_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'AUTRE') + data['commune_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'COMMUNE') + data['justificatif_identite'] = get('ji') + situation_matrimoniale = get('recensementcitoyen_personne_situationfamille_situationmatrimoniale') + data['situation_familiale'] = { + u'Célibataire': u'Célibataire', + u'Marié': u'Marié(e)', + }.get(situation_matrimoniale, u'Autres') + if data['situation_familiale'] == u'Autres': + data['situation_familiale_precision'] = situation_matrimoniale + pupille = get('recensementcitoyen_personne_situationfamille_pupille') + data['pupille'] = ( + 'Oui' + if pupille + else 'Non' + ) + data['pupille_categorie'] = { + 'NATION': u"Pupille de la nation", + 'ETAT': u"Pupille de l'État", + }.get(pupille) + for idx in ['', '_1', '_2']: + code = get('recensementcitoyen_personne_methodecontact%s_canalcode' % idx) + uri = get('recensementcitoyen_personne_methodecontact%s_uri' % idx) + if code == 'EMAIL': + data['courriel'] = uri + if code == 'TEL': + data['telephone_fixe'] = uri + data['justificatif_famille'] = data['jf'] + data['filiation_inconnue_p1'] = not get('recensementcitoyen_filiationpere_nomfamille') + data['filiation_inconnue_p2'] = not get('recensementcitoyen_filiationmere_nomfamille') + data['cp_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'AUTRE') + data['cp_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'AUTRE') + data['commune_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'COMMUNE') + data['commune_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'COMMUNE') + for key in data: + if key.endswith('_datenaissance') and data[key]: + data[key] = ( + datetime.datetime.strptime(data[key], '%d/%m/%Y') + .date() + .strftime('%Y-%m-%d') + ) + + def extract_data(self, document): + root = document.getroot() + + def tag_name(node): + return simplify(ET.QName(node.tag).localname) + + def helper(path, node): + if len(node): + tags = collections.Counter(tag_name(child) for child in node) + counter = collections.Counter() + for child in node: + name = tag_name(child) + if tags[name] > 1: + counter[name] += 1 + name += '_%s' % counter[name] + for p, value in helper(path + [name], child): + yield p, value + else: + yield path, text_content(node) + return { + '_'.join(path).replace('-', '_').replace(' ', ''): value + for path, value in helper([tag_name(root)], root) + } + + class Meta: + verbose_name = _('Service-Public.fr') + + +class SPFRMessage(object): + @classmethod + def from_file(self, resource, filename, fd): + m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename) + if not m: + resource.logger.warning('found file with an unknown pattern %s moving in DONE/', filename) + return None + mdel_number, procedure, sequence = m.groups() + + +def default_rule(): + return {} + + +class Mapping(models.Model): + + resource = models.ForeignKey( + Resource, + verbose_name=_('Resource'), + related_name='mappings') + + procedure = models.CharField( + verbose_name=_('Procedure'), + choices=PROCEDURES, + unique=True, + max_length=8) + + formdef = FormDefField( + verbose_name=_('Formulaire')) + + rules = JSONField( + verbose_name=_('Rules'), + default=default_rule) + + def get_absolute_url(self): + return reverse('sp-fr-mapping-edit', kwargs=dict( + slug=self.resource.slug, + pk=self.pk)) + + @property + def xsd(self): + doc = ET.parse(os.path.join(os.path.dirname(__file__), '%s.XSD' % self.procedure)) + schema = Schema() + schema.visit(doc.getroot()) + return schema + + @property + def variables(self): + yield 'insee_code' + for path, xsd_type in self.xsd.paths(): + names = [simplify(tag.localname).replace('-', '_').replace(' ', '') for tag in path] + yield '_'.join(names) + if hasattr(self, 'variables_%s' % self.procedure): + for variable in getattr(self, 'variables_%s' % self.procedure): + yield variable + + @property + def variables_DOC(self): + yield 'type_permis' + yield 'numero_permis' + yield 'type_declarant' + yield 'nom' + yield 'prenoms' + yield 'civilite_particulier' + yield 'civilite_pm' + yield 'portee' + + @property + def variables_RCO(self): + yield 'motif' + yield 'motif_exemple' + yield 'justificatif_exemption' + yield 'double_nationalite' + yield 'residence_differente' + yield 'civilite' + yield 'cp_naissance' + yield 'commune_naissance' + yield 'je' + yield 'ji' + yield 'situation_familiale' + yield 'situation_familiale_precision' + yield 'pupille' + yield 'pupille_categorie' + yield 'courriel' + yield 'telephone_fixe' + yield 'jf' + yield 'filiation_inconnue_p1' + yield 'filiation_inconnue_p2' + yield 'cp_naissance_p1' + yield 'cp_naissance_p2' + yield 'commune_naissance_p1' + yield 'commune_naissance_p2' + +#def archive_upload_to(instance, filename): +# return 'sp_fr/{instance.procedure}/{filename}'.format( +# instance=instance, +# filename=filename) +# +# +#class Request(models.Model): +# +# # To prevent mixing errors from analysing archive from s-p.fr and errors +# # from pushing to w.c.s we separate processing with three steps: +# # - receiving, i.e. copying zipfile from SFTP and storing them locally +# # - processing, i.e. openeing the zipfile and extracting content as we need it +# # - transferring, pushing content as a new form in w.c.s. +# STATE_RECEIVED = 'received' +# STATE_PROCESSED = 'processed' +# STATE_TRANSFERED = 'transfered' +# STATE_ERROR = 'error' +# STATES = [ +# (STATE_RECEIVED, _('Received')), +# (STATE_TRANSFERED, _('Transfered')), +# (STATE_ERROR, _('Transfered')), +# ] +# +# ressource = models.ForeignKey( +# Resource, +# verbose_name=_('Resource')) +# identifier = models.CharField( +# verbose_name=_('Identifier'), +# max_length=32) +# procedure = models.CharField( +# verbose_name=_('Procedure'), +# choices=PROCEDURES, +# max_length=8) +# sequence_number = models.PositiveIntegerField( +# verbose_name=_('Sequence number')) +# +# state = models.CharField( +# verbose_name=_('State'), +# choices=STATES, +# max_length=16) +# +# content = JSONField( +# verbose_name=_('Content'), +# null=True) +# +# archive = models.FileField( +# verbose_name=_('Archive'), +# upload_to=archive_upload_to) diff --git a/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html b/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html new file mode 100644 index 00000000..feadd733 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html @@ -0,0 +1,6 @@ +
+ {% include widget.subwidgets.0.template_name with widget=widget.subwidgets.0 %} +
+
+ {% include widget.subwidgets.1.template_name with widget=widget.subwidgets.1 %} +
diff --git a/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html b/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html new file mode 100644 index 00000000..a98a97ec --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html @@ -0,0 +1 @@ +{% extends "passerelle/manage/resource_child_confirm_delete.html" %} diff --git a/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html b/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html new file mode 100644 index 00000000..c226baf4 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html @@ -0,0 +1,44 @@ +{% extends "passerelle/manage/resource_child_form.html" %} +{% load i18n %} + + +{% block form %} + {% if form.errors %} +
+

{% trans "There were errors processing your form." %}

+ {% for error in form.non_field_errors %} +

{{ error }}

+ {% endfor %} + {% for field in form %} + {% if field.is_hidden and field.errors %} +

+ {% for error in field.errors %} + {% blocktrans with name=field.name %}(Hidden field {{name}}) {{ error }}{% endblocktrans %} + {% if not forloop.last %}
{% endif %} + {% endfor %} +

+ {% endif %} + {% endfor %} +
+ {% endif %} + {% include "gadjo/widget.html" with field=form.procedure %} + + {% include "gadjo/widget.html" with field=form.formdef%} + + + + + + + + + + {% for field in form.table_fields %} + + + + + {% endfor %} + +
LabelVariable
{{ field.label_tag }}{{ field }}
+{% endblock %} diff --git a/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html b/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html new file mode 100644 index 00000000..ceb02198 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html @@ -0,0 +1,39 @@ +{% extends "passerelle/manage/service_view.html" %} +{% load i18n passerelle %} + +{% block description %} +

+ {% blocktrans %} + Connector to forms published by service-public.fr + {% endblocktrans %} + Run +

+ {{ block.super }} +{% endblock %} + + +{% block extra-sections %} +
+

{% trans "Mappings" %} {% trans "Add" %}

+ +
+{% endblock %} diff --git a/passerelle/apps/sp_fr/urls.py b/passerelle/apps/sp_fr/urls.py new file mode 100644 index 00000000..63867a0a --- /dev/null +++ b/passerelle/apps/sp_fr/urls.py @@ -0,0 +1,30 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +management_urlpatterns = [ + url(r'^(?P[\w,-]+)/mapping/new/$', + views.MappingNew.as_view(), name='sp-fr-mapping-new'), + url(r'^(?P[\w,-]+)/mapping/(?P\d+)/$', + views.MappingEdit.as_view(), name='sp-fr-mapping-edit'), + url(r'^(?P[\w,-]+)/mapping/(?P\d+)/delete/$', + views.MappingDelete.as_view(), name='sp-fr-mapping-delete'), + url(r'^(?P[\w,-]+)/run/$', + views.run, name='sp-fr-run'), +] diff --git a/passerelle/apps/sp_fr/views.py b/passerelle/apps/sp_fr/views.py new file mode 100644 index 00000000..280049d5 --- /dev/null +++ b/passerelle/apps/sp_fr/views.py @@ -0,0 +1,67 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.views.generic import UpdateView, CreateView, DeleteView +from django.shortcuts import get_object_or_404 +from django.http import HttpResponseRedirect + +from passerelle.base.mixins import ResourceChildViewMixin + +from . import models, forms + + +class StayIfChanged(object): + has_changed = False + + def form_valid(self, form): + if set(form.changed_data) & set(['procedure', 'formdef']): + self.has_changed = True + return super(StayIfChanged, self).form_valid(form) + + def get_success_url(self): + if self.has_changed: + return self.get_changed_url() + return super(StayIfChanged, self).get_success_url() + + def get_changed_url(self): + return '' + + +class MappingNew(StayIfChanged, ResourceChildViewMixin, CreateView): + model = models.Mapping + form_class = forms.MappingForm + + def form_valid(self, form): + form.instance.resource = self.resource + return super(MappingNew, self).form_valid(form) + + def get_changed_url(self): + return self.object.get_absolute_url() + + +class MappingEdit(StayIfChanged, ResourceChildViewMixin, UpdateView): + model = models.Mapping + form_class = forms.MappingForm + + +class MappingDelete(ResourceChildViewMixin, DeleteView): + model = models.Mapping + + +def run(request, connector, slug): + resource = get_object_or_404(models.Resource, slug=slug) + resource.run_loop(1000) + return HttpResponseRedirect(resource.get_absolute_url()) diff --git a/passerelle/settings.py b/passerelle/settings.py index c87db386..1be5db9e 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -135,6 +135,7 @@ INSTALLED_APPS = ( 'passerelle.apps.feeds', 'passerelle.apps.gdc', 'passerelle.apps.jsondatastore', + 'passerelle.apps.sp_fr', 'passerelle.apps.mobyt', 'passerelle.apps.okina', 'passerelle.apps.opengis', diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css index dfd2d765..5e5125a2 100644 --- a/passerelle/static/css/style.css +++ b/passerelle/static/css/style.css @@ -184,3 +184,9 @@ li.connector.status-down span.connector-name::after { .log-dialog table td { vertical-align: top; } +.expression-widget input { + width: 100%; +} +.variable-widget select { + width: 100%; +} diff --git a/tests/test_sp_fr.py b/tests/test_sp_fr.py new file mode 100644 index 00000000..340c8b48 --- /dev/null +++ b/tests/test_sp_fr.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest + +from passerelle.apps.sp_fr.models import Resource + +import utils + + +DUMMY_CONTENT = { + 'DILA': { + 'a.zip': 'a', + } +} + + +@pytest.fixture +def spfr(settings, db, sftpserver): + settings.KNOWN_SERVICES = { + 'wcs': { + 'eservices': { + 'title': u'Démarches', + 'url': 'https://demarches-hautes-alpes.test.entrouvert.org/', + 'secret': '9a1f62b680c1cabe73cefcbc5ff4cd4f95c24c3be8dd930adb80d8aaa33bfe67', + 'orig': 'passerelle-hautes-alpes.test.entrouvert.org', + } + } + } + yield utils.make_resource( + Resource, + title='Test 1', + slug='test1', + description='Connecteur de test', + input_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver), + output_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver) + ) + + +def test_resource(spfr): + from passerelle.utils.wcs import get_wcs_choices + + assert get_wcs_choices() == [] + + +def test_sftp_access(spfr, sftpserver): + with sftpserver.serve_content(DUMMY_CONTENT): + with spfr.input_sftp as input_sftp: + assert input_sftp.listdir() == ['a.zip'] + with spfr.output_sftp as output_sftp: + assert output_sftp.listdir() == ['a.zip'] -- 2.20.1