From fe87d82adf4ad68cedf19508463f10e6ec0c26e9 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 29 Mar 2019 16:42:14 +0100 Subject: [PATCH 11/11] initialize sp_fr connector (#31595) New connector for transfering forms from Service-Public.fr to w.c.s. --- passerelle/apps/sp_fr/DDPACS.XSD | 129 ++++ passerelle/apps/sp_fr/DOC.XSD | 108 +++ passerelle/apps/sp_fr/RCO.XSD | 137 ++++ passerelle/apps/sp_fr/__init__.py | 0 passerelle/apps/sp_fr/admin.py | 27 + passerelle/apps/sp_fr/fields.py | 94 +++ passerelle/apps/sp_fr/forms.py | 69 ++ passerelle/apps/sp_fr/models.py | 652 ++++++++++++++++++ .../variable_and_expression_widget.html | 6 + .../sp_fr/mapping_confirm_delete.html | 9 + .../sp_fr/templates/sp_fr/mapping_form.html | 55 ++ .../templates/sp_fr/resource_detail.html | 69 ++ passerelle/apps/sp_fr/urls.py | 30 + passerelle/apps/sp_fr/views.py | 67 ++ passerelle/settings.py | 1 + passerelle/static/css/style.css | 6 + tests/test_sp_fr.py | 65 ++ 17 files changed, 1524 insertions(+) create mode 100644 passerelle/apps/sp_fr/DDPACS.XSD create mode 100644 passerelle/apps/sp_fr/DOC.XSD create mode 100644 passerelle/apps/sp_fr/RCO.XSD create mode 100644 passerelle/apps/sp_fr/__init__.py create mode 100644 passerelle/apps/sp_fr/admin.py create mode 100644 passerelle/apps/sp_fr/fields.py create mode 100644 passerelle/apps/sp_fr/forms.py create mode 100644 passerelle/apps/sp_fr/models.py create mode 100644 passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html create mode 100644 passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html create mode 100644 passerelle/apps/sp_fr/urls.py create mode 100644 passerelle/apps/sp_fr/views.py create mode 100644 tests/test_sp_fr.py diff --git a/passerelle/apps/sp_fr/DDPACS.XSD b/passerelle/apps/sp_fr/DDPACS.XSD new file mode 100644 index 00000000..58ae5383 --- /dev/null +++ b/passerelle/apps/sp_fr/DDPACS.XSD @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/sp_fr/DOC.XSD b/passerelle/apps/sp_fr/DOC.XSD new file mode 100644 index 00000000..9bd73815 --- /dev/null +++ b/passerelle/apps/sp_fr/DOC.XSD @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/sp_fr/RCO.XSD b/passerelle/apps/sp_fr/RCO.XSD new file mode 100644 index 00000000..339e3cd1 --- /dev/null +++ b/passerelle/apps/sp_fr/RCO.XSD @@ -0,0 +1,137 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/sp_fr/__init__.py b/passerelle/apps/sp_fr/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/sp_fr/admin.py b/passerelle/apps/sp_fr/admin.py new file mode 100644 index 00000000..27855400 --- /dev/null +++ b/passerelle/apps/sp_fr/admin.py @@ -0,0 +1,27 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.contrib import admin + +from .models import Request + + +class RequestAdmin(admin.ModelAdmin): + data_hierarchy = 'created' + search_fields = ['url', 'filename'] + list_display = ['id', 'created', 'modified', 'state', 'filename', 'url'] + +admin.site.register(Request, RequestAdmin) diff --git a/passerelle/apps/sp_fr/fields.py b/passerelle/apps/sp_fr/fields.py new file mode 100644 index 00000000..563dd6a2 --- /dev/null +++ b/passerelle/apps/sp_fr/fields.py @@ -0,0 +1,94 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.exceptions import ValidationError +from django.template import engines, TemplateSyntaxError +from django import forms + + +def validate_django_template(value): + try: + engines['django'].from_string(value) + except TemplateSyntaxError as e: + raise ValidationError('invalid template %s' % e) + + +class VariableAndExpressionWidget(forms.MultiWidget): + template_name = 'passerelle/widgets/variable_and_expression_widget.html' + + def __init__(self, **kwargs): + widgets = [ + forms.Select, + forms.TextInput, + ] + super(VariableAndExpressionWidget, self).__init__(widgets=widgets, **kwargs) + + def decompress(self, value): + if not value: + return [None, None] + return value['variable'], value['expression'] + + # XXX: bug in Django https://code.djangoproject.com/ticket/29205 + # required_attribute is initialized from the parent.field required + # attribute and not from each sub-field attribute + def use_required_attribute(self, initial): + return False + + +class VariableAndExpressionField(forms.MultiValueField): + widget = VariableAndExpressionWidget + + def __init__(self, choices=(), required=True, widget=None, label=None, + initial=None, help_text='', *args, **kwargs): + fields = [ + forms.ChoiceField(choices=choices, required=required), + forms.CharField(required=False, validators=[validate_django_template]), + ] + super(VariableAndExpressionField, self).__init__( + fields=fields, + required=required, + widget=widget, + label=label, + initial=initial, + help_text=help_text, + require_all_fields=False, *args, **kwargs) + self.choices = choices + + def _get_choices(self): + return self._choices + + def _set_choices(self, value): + # Setting choices also sets the choices on the widget. + # choices can be any iterable, but we call list() on it because + # it will be consumed more than once. + if callable(value): + value = forms.CallableChoiceIterator(value) + else: + value = list(value) + self._choices = value + self.widget.widgets[0].choices = value + choices = property(_get_choices, _set_choices) + + def compress(self, data): + try: + variable, expression = data + except (ValueError, TypeError): + return None + else: + return { + 'variable': variable, + 'expression': expression, + } diff --git a/passerelle/apps/sp_fr/forms.py b/passerelle/apps/sp_fr/forms.py new file mode 100644 index 00000000..639eea16 --- /dev/null +++ b/passerelle/apps/sp_fr/forms.py @@ -0,0 +1,69 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django import forms + +from . import models, fields + + +class MappingForm(forms.ModelForm): + + def __init__(self, *args, **kwargs): + super(MappingForm, self).__init__(*args, **kwargs) + if self.instance.procedure and self.instance and self.instance.formdef: + choices = [('', '--------')] + [(v, v) for v in self.instance.variables] + for i, field in enumerate(self.schema_fields()): + label = field.label + label += ' (%s)' % (field.varname or 'NO VARNAME') + base_name = str(field.varname or i) + initial = self.instance.rules.get('fields', {}).get(base_name) + self.fields['field_%s' % base_name] = fields.VariableAndExpressionField( + label=label, + choices=choices, + initial=initial, + required=False) + + def table_fields(self): + return [field for field in self if field.name.startswith('field_')] + + def schema_fields(self): + if self.instance and self.instance.formdef: + schema = self.instance.formdef.schema + for i, field in enumerate(schema.fields): + if field.type in ('page', 'comment', 'title', 'subtitle'): + continue + yield field + + def save(self, commit=True): + fields = {} + for key in self.cleaned_data: + if not key.startswith('field_'): + continue + if not self.cleaned_data[key]: + continue + real_key = key[len('field_'):] + value = self.cleaned_data[key].copy() + value['label'] = self.fields[key].label + fields[real_key] = value + self.instance.rules['fields'] = fields + return super(MappingForm, self).save(commit=commit) + + class Meta: + model = models.Mapping + fields = [ + 'procedure', + 'formdef', + ] diff --git a/passerelle/apps/sp_fr/models.py b/passerelle/apps/sp_fr/models.py new file mode 100644 index 00000000..1a76c0a0 --- /dev/null +++ b/passerelle/apps/sp_fr/models.py @@ -0,0 +1,652 @@ +# -*- coding: utf-8 -*- +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import re +import os +import stat +import zipfile +import collections +import base64 +import datetime +import unicodedata + +from lxml import etree as ET + +from django.core.urlresolvers import reverse +from django.core.files import File +from django.db import models, transaction +from django.template import engines +from django.utils import six +from django.utils.translation import ugettext_lazy as _, ugettext + +from jsonfield import JSONField + +from passerelle.base.models import BaseResource +from passerelle.utils.api import endpoint +from passerelle.utils.sftp import SFTPField +from passerelle.utils.wcs import FormDefField, get_wcs_choices +from passerelle.utils.xsd import Schema +from passerelle.utils.xml import text_content + + +PROCEDURE_DOC = 'DOC' +PROCEDURE_RCO = 'RCO' +PROCEDURE_DDPACS = 'DDPACS' +PROCEDURES = [ + (PROCEDURE_DOC, _('Request for construction site opening')), + (PROCEDURE_RCO, _('Request for mandatory citizen census')), + (PROCEDURE_DDPACS, _('Pre-request for citizen solidarity pact')), +] + +FILE_PATTERN = re.compile(r'^(?P.*)-(?P[A-Z]+)-(?P\d+).zip$') +ENT_PATTERN = re.compile(r'^.*-ent-\d+(?:-.*)?.xml$') +NSMAP = { + 'dgme-metier': 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier' +} +ROUTAGE_XPATH = ET.XPath( + ('dgme-metier:Routage/dgme-metier:Donnee/dgme-metier:Valeur/text()'), + namespaces=NSMAP) + +DOCUMENTS_XPATH = ET.XPath('dgme-metier:Document', namespaces=NSMAP) +PIECE_JOINTE_XPATH = ET.XPath('dgme-metier:PieceJointe', namespaces=NSMAP) +CODE_XPATH = ET.XPath('dgme-metier:Code', namespaces=NSMAP) +FICHIER_XPATH = ET.XPath('dgme-metier:Fichier', namespaces=NSMAP) +FICHIER_DONNEES_XPATH = ET.XPath('.//dgme-metier:FichierDonnees', namespaces=NSMAP) + +ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier') + + +def simplify(s): + '''Simplify XML node tag names because XSD from DGME are garbage''' + if not s: + return '' + if not isinstance(s, six.text_type): + s = six.text_type(s, 'utf-8', 'ignore') + s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') + s = re.sub(r'[^\w\s\'-_]', '', s) + s = s.replace('-', '_') + s = re.sub(r'[\s\']+', '', s) + return s.strip().lower() + + +class Resource(BaseResource): + category = _('Business Process Connectors') + + input_sftp = SFTPField( + verbose_name=_('Input SFTP URL'), + null=True) + + output_sftp = SFTPField( + verbose_name=_('Output SFTP URL'), + null=True) + + def check_status(self): + with self.input_sftp.client() as sftp: + sftp.listdir() + with self.output_sftp.client() as sftp: + sftp.listdir() + get_wcs_choices(session=self.requests) + + @endpoint(name='ping', description=_('Check Solis API availability')) + def ping(self, request): + self.check_status() + return {'err': 0} + + def run_loop(self, count=1): + with transaction.atomic(): + # lock resource + r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk) + if not r: + # already locked + self.logger.info('did nothing') + return + with self.input_sftp.client() as sftp: + try: + sftp.lstat('DONE') + except IOError: + sftp.mkdir('DONE') + + try: + sftp.lstat('FAILED') + except IOError: + sftp.mkdir('FAILED') + + def helper(): + for file_stat in sftp.listdir_attr(): + if stat.S_ISDIR(file_stat.st_mode): + continue + yield file_stat.filename + + for filename, i in zip(helper(), range(count)): + m = FILE_PATTERN.match(filename) + if not m: + self.logger.info('file "%s" did not match pattern %s, moving to FAILED/', + filename, FILE_PATTERN) + sftp.rename(filename, 'FAILED/' + filename) + continue + procedure = m.group('procedure') + try: + mapping = self.mappings.get(procedure=procedure) + except Mapping.DoesNotExist: + self.logger.info('no mapping for procedure "%s" for file "%s", moving to FAILED/', + procedure, filename) + continue + + handler = self.FileHandler( + resource=self, + sftp=sftp, + filename=filename, + identifier=m.group('identifier'), + procedure=procedure, + sequence=m.group('sequence'), + mapping=mapping) + try: + move, error = handler() + except Exception: + self.logger.exception('handling of file "%s" failed', filename) + # sftp.rename(filename, 'FAILED/' + filename) + else: + if move and error: + self.logger.error('handling of file "%s" failed: %s', filename, error) + # sftp.rename(filename, 'FAILED/' + filename) + else: + if error: + self.logger.warning('handling of file "%s" failed: %s', filename, error) + elif move: + sftp.rename(filename, 'DONE/' + filename) + + class FileHandler(object): + def __init__(self, resource, sftp, filename, identifier, procedure, sequence, mapping): + self.resource = resource + self.sftp = sftp + self.filename = filename + self.identifier = identifier + self.procedure = procedure + self.sequence = sequence + self.mapping = mapping + self.variables = list(self.mapping.variables) + self.request = Request.objects.filter(resource=resource, filename=filename).first() + + def __call__(self): + if not self.request: + with self.sftp.open(self.filename) as fd: + with transaction.atomic(): + self.request = Request.objects.create( + resource=self.resource, + filename=self.filename) + self.request.state = Request.STATE_RECEIVED + self.request.archive.save(self.filename, File(fd)) + if self.request.state == Request.STATE_RECEIVED: + with self.request.archive as fd: + # error during processing are fatal, we want to log them + data, error = self.process(fd) + if not data: + return False, error + try: + backoffice_url = self.transfer(data) + except Exception as e: + raise + return False, 'error during transfer to w.c.s %s' % e + self.request.url = backoffice_url + self.request.state = Request.STATE_TRANSFERED + self.request.save() + + if self.request.state == Request.STATE_TRANSFERED: + try: + self.response() + except Exception as e: + return False, 'error during response to service-public.fr %s' % e + self.request.state = Request.STATE_RETURNED + self.request.save() + + def process(self, fd): + try: + archive = zipfile.ZipFile(fd) + except Exception: + return False, 'could not load zipfile' + # sort files + doc_files = [] + ent_files = [] + attachments = {} + for name in archive.namelist(): + if ENT_PATTERN.match(name): + ent_files.append(name) + + if len(ent_files) != 1: + return False, 'too many/few ent files found: %s' % ent_files + + ent_file = ent_files[0] + + with archive.open(ent_file) as fd: + document = ET.parse(fd) + + for pj_node in PIECE_JOINTE_XPATH(document): + code = CODE_XPATH(pj_node)[0].text + code = 'pj_' + code.lower().replace('-', '_') + fichier = FICHIER_XPATH(pj_node)[0].text + attachments.setdefault(code, []).append(fichier) + for doc_node in DOCUMENTS_XPATH(document): + code = CODE_XPATH(doc_node)[0].text + code = 'doc_' + code.lower().replace('-', '_') + fichier = FICHIER_DONNEES_XPATH(doc_node)[0].text + attachments.setdefault(code, []).append(fichier) + + doc_files = [value for l in attachments.values() for value in l if value.lower().endswith('.xml')] + if len(doc_files) != 1: + return False, 'too many/few doc files found: %s' % doc_files + + for key in attachments: + if len(attachments[key]) > 1: + return False, 'too many attachments of kind %s: %r' % (key, attachments[key]) + name = attachments[key][0] + with archive.open(attachments[key][0]) as zip_fd: + content = zip_fd.read() + attachments[key] = { + 'filename': name, + 'content': base64.b64encode(content).decode('ascii'), + 'content_type': 'application/octet-stream', + } + + if self.procedure == 'RCO' and not attachments: + return False, 'no attachments but RCO requires them' + + doc_file = doc_files[0] + + insee_codes = ROUTAGE_XPATH(document) + if len(insee_codes) != 1: + return False, 'too many/few insee codes found: %s' % insee_codes + insee_code = insee_codes[0] + + data = {'insee_code': insee_code} + data.update(attachments) + + with archive.open(doc_file) as fd: + document = ET.parse(fd) + data.update(self.extract_data(document)) + if hasattr(self, 'update_data_%s' % self.procedure): + getattr(self, 'update_data_%s' % self.procedure)(data) + return data, None + + def transfer(self, data): + formdef = self.mapping.formdef + formdef.session = self.resource.requests + + with formdef.submit() as submitter: + submitter.submission_channel = 'web' + submitter.submission_context = { + 'mdel_procedure': self.procedure, + 'mdel_identifier': self.identifier, + 'mdel_sequence': self.sequence, + } + fields = self.mapping.rules.get('fields', {}) + for name in fields: + field = fields[name] + variable = field['variable'] + expression = field['expression'] + value = data.get(variable) + if expression.strip(): + template = engines['django'].from_string(expression) + context = data.copy() + context['value'] = value + value = template.render(context) + submitter.set(name, value) + return submitter.result.backoffice_url + + def response(self): + raise NotImplementedError + + def get_data(self, data, name): + # prevent error in manual mapping + assert name in self.variables, 'variable "%s" is unknown' % name + return data.get(name, '') + + def update_data_DOC(self, data): + def get(name): + return self.get_data(data, name) + + numero_permis_construire = get('doc_declarant_designation_permis_numero_permis_construire') + numero_permis_amenager = get('doc_declarant_designation_permis_numero_permis_amenager') + data['type_permis'] = u'Un permis de construire' if numero_permis_construire else u'Un permis d\'aménager' + data['numero_permis'] = numero_permis_construire or numero_permis_amenager + particulier = get('doc_declarant_identite_type_personne').strip().lower() == 'true' + data['type_declarant'] = u'Un particulier' if particulier else u'Une personne morale' + if particulier: + data['nom'] = get('doc_declarant_identite_personne_physique_nom') + data['prenoms'] = get('doc_declarant_identite_personne_physique_prenom') + else: + data['nom'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_nom') + data['prenoms'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_prenom') + mapping = { + '1000': 'Monsieur', + '1001': 'Madame', + '1002': 'Madame et Monsieur', + } + if particulier: + data['civilite_particulier'] = mapping.get(get('doc_declarant_identite_personne_physique_civilite'), '') + else: + data['civilite_pm'] = mapping.get( + get('doc_declarant_identite_personne_morale_representant_personne_morale_civilite'), '') + data['portee'] = (u'Pour la totalité des travaux' + if get('doc_ouverture_chantier_totalite_travaux').lower().strip() == 'true' + else u'Pour une tranche des travaux') + + def update_data_RCO(self, data): + def get(name): + return self.get_data(data, name) + + motif = ( + get('recensementcitoyen_formalite_formalitemotifcode_1') + or get('recensementcitoyen_formalite_formalitemotifcode_2') + ) + data['motif'] = { + 'RECENSEMENT': '1', + 'EXEMPTION': '2' + }[motif] + if data['motif'] == '2': + data['motif_exempte'] = ( + u"Titulaire d'une carte d'invalidité de 80% minimum" + if get('recensementcitoyen_formalite_formalitemotifcode_2') == 'INFIRME' + else u"Autre situation") + data['justificatif_exemption'] = get('pj_je') + data['double_nationalite'] = ( + 'Oui' + if get('recensementcitoyen_personne_nationalite') + else 'Non') + data['residence_differente'] = ( + 'Oui' + if get('recensementcitoyen_personne_adresseresidence_localite') + else 'Non') + data['civilite'] = ( + 'Monsieur' + if get('recensementcitoyen_personne_civilite') == 'M' + else 'Madame' + ) + + def get_lieu_naissance(variable, code): + for idx in ['', '_1', '_2']: + v = variable + idx + if get(v + '_code') == code: + return get(v + '_nom') + + data['cp_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'AUTRE') + data['commune_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'COMMUNE') + data['justificatif_identite'] = get('pj_ji') + situation_matrimoniale = get('recensementcitoyen_personne_situationfamille_situationmatrimoniale') + data['situation_familiale'] = { + u'Célibataire': u'Célibataire', + u'Marié': u'Marié(e)', + }.get(situation_matrimoniale, u'Autres') + if data['situation_familiale'] == u'Autres': + data['situation_familiale_precision'] = situation_matrimoniale + pupille = get('recensementcitoyen_personne_situationfamille_pupille') + data['pupille'] = ( + 'Oui' + if pupille + else 'Non' + ) + data['pupille_categorie'] = { + 'NATION': u"Pupille de la nation", + 'ETAT': u"Pupille de l'État", + }.get(pupille) + for idx in ['', '_1', '_2']: + code = get('recensementcitoyen_personne_methodecontact%s_canalcode' % idx) + uri = get('recensementcitoyen_personne_methodecontact%s_uri' % idx) + if code == 'EMAIL': + data['courriel'] = uri + if code == 'TEL': + data['telephone_fixe'] = uri + data['justificatif_famille'] = get('pj_jf') + data['filiation_inconnue_p1'] = not get('recensementcitoyen_filiationpere_nomfamille') + data['filiation_inconnue_p2'] = not get('recensementcitoyen_filiationmere_nomfamille') + data['cp_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'AUTRE') + data['cp_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'AUTRE') + data['commune_naissance_p1'] = get_lieu_naissance( + 'recensementcitoyen_filiationpere_lieunaissance', 'COMMUNE') + data['commune_naissance_p2'] = get_lieu_naissance( + 'recensementcitoyen_filiationmere_lieunaissance', 'COMMUNE') + for key in data: + if key.endswith('_datenaissance') and data[key]: + data[key] = ( + datetime.datetime.strptime(data[key], '%d/%m/%Y') + .date() + .strftime('%Y-%m-%d') + ) + + def update_data_DDPACS(self, data): + def get(name): + return self.get_data(data, name) + + civilite_p1 = get('pacs_partenaire1_civilite') + data['civilite_p1'] = 'Monsieur' if civilite_p1 == 'M' else 'Madame' + data['acte_naissance_p1'] = get('pj_an') + data['identite_verifiee_p1'] = 'Oui' if get('pacs_partenaire1_titreidentiteverifie') == 'true' else 'None' + + civilite_p2 = get('pacs_partenaire2_civilite') + data['civilite_p2'] = 'Monsieur' if civilite_p2 == 'M' else 'Madame' + data['acte_naissance_p2'] = get('pj_anp') + data['identite_verifiee_p2'] = 'Oui' if get('pacs_partenaire2_titreidentiteverifie') == 'true' else 'None' + + data['type_convention'] = '2' if get('pacs_convention_conventionspecifique') == 'true' else '1' + data['aide_materielle'] = ( + '1' if get('pacs_convention_conventiontype_aidemateriel_typeaidemateriel') == 'aideProportionnel' + else '2') + data['regime'] = '1' if get('pacs_convention_conventiontype_regimepacs') == 'legal' else '2' + data['convention_specifique'] = get('pj_cp') + + def extract_data(self, document): + '''Convert XML into a dictionnary of values''' + root = document.getroot() + + def tag_name(node): + return simplify(ET.QName(node.tag).localname) + + def helper(path, node): + if len(node): + tags = collections.Counter(tag_name(child) for child in node) + counter = collections.Counter() + for child in node: + name = tag_name(child) + if tags[name] > 1: + counter[name] += 1 + name += '_%s' % counter[name] + for p, value in helper(path + [name], child): + yield p, value + else: + yield path, text_content(node) + return {'_'.join(path): value for path, value in helper([tag_name(root)], root)} + + class Meta: + verbose_name = _('Service-Public.fr') + + +class SPFRMessage(object): + @classmethod + def from_file(self, resource, filename, fd): + m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename) + if not m: + resource.logger.warning('found file with an unknown pattern %s moving in DONE/', filename) + return None + mdel_number, procedure, sequence = m.groups() + + +def default_rule(): + return {} + + +@six.python_2_unicode_compatible +class Mapping(models.Model): + resource = models.ForeignKey( + Resource, + verbose_name=_('Resource'), + related_name='mappings') + + procedure = models.CharField( + verbose_name=_('Procedure'), + choices=PROCEDURES, + unique=True, + max_length=8) + + formdef = FormDefField( + verbose_name=_('Formdef')) + + rules = JSONField( + verbose_name=_('Rules'), + default=default_rule) + + def get_absolute_url(self): + return reverse('sp-fr-mapping-edit', kwargs=dict( + slug=self.resource.slug, + pk=self.pk)) + + @property + def xsd(self): + doc = ET.parse(os.path.join(os.path.dirname(__file__), '%s.XSD' % self.procedure)) + schema = Schema() + schema.visit(doc.getroot()) + return schema + + @property + def variables(self): + yield 'insee_code' + for path, xsd_type in self.xsd.paths(): + names = [simplify(tag.localname) for tag in path] + yield '_'.join(names) + if hasattr(self, 'variables_%s' % self.procedure): + for variable in getattr(self, 'variables_%s' % self.procedure): + yield variable + + @property + def variables_DOC(self): + yield 'type_permis' + yield 'numero_permis' + yield 'type_declarant' + yield 'nom' + yield 'prenoms' + yield 'civilite_particulier' + yield 'civilite_pm' + yield 'portee' + + @property + def variables_RCO(self): + yield 'motif' + yield 'motif_exemple' + yield 'justificatif_exemption' + yield 'double_nationalite' + yield 'residence_differente' + yield 'civilite' + yield 'cp_naissance' + yield 'commune_naissance' + yield 'pj_je' + yield 'pj_ji' + yield 'situation_familiale' + yield 'situation_familiale_precision' + yield 'pupille' + yield 'pupille_categorie' + yield 'courriel' + yield 'telephone_fixe' + yield 'pj_jf' + yield 'filiation_inconnue_p1' + yield 'filiation_inconnue_p2' + yield 'cp_naissance_p1' + yield 'cp_naissance_p2' + yield 'commune_naissance_p1' + yield 'commune_naissance_p2' + + @property + def variables_DDPACS(self): + yield 'pj_an' + yield 'pj_anp' + yield 'pj_cp' + yield 'doc_15725_01' + yield 'doc_flux_pacs' + yield 'doc_recappdf' + yield 'civilite_p1' + yield 'acte_naissance_p1' + yield 'identite_verifiee_p1' + + yield 'civilite_p2' + yield 'acte_naissance_p2' + yield 'identite_verifiee_p2' + + yield 'type_convention' + yield 'aide_materielle' + yield 'regime' + yield 'convention_specifique' + + def __str__(self): + return ugettext('Mapping from "{procedure}" to formdef "{formdef}"').format( + procedure=self.get_procedure_display(), + formdef=self.formdef.title if self.formdef else '-') + + class Meta: + verbose_name = _('MDEL mapping') + verbose_name_plural = _('MDEL mappings') + + +class Request(models.Model): + # To prevent mixing errors from analysing archive from s-p.fr and errors + # from pushing to w.c.s we separate processing with three steps: + # - receiving, i.e. copying zipfile from SFTP and storing them locally + # - processing, i.e. openeing the zipfile and extracting content as we need it + # - transferring, pushing content as a new form in w.c.s. + STATE_RECEIVED = 'received' + STATE_TRANSFERED = 'transfered' + STATE_RETURNED = 'returned' + STATE_ERROR = 'error' + STATES = [ + (STATE_RECEIVED, _('Received')), + (STATE_TRANSFERED, _('Transfered')), + (STATE_ERROR, _('Transfered')), + (STATE_RETURNED, _('Returned')), + ] + + resource = models.ForeignKey( + Resource, + verbose_name=_('Resource')) + + created = models.DateTimeField( + verbose_name=_('Created'), + auto_now_add=True) + + modified = models.DateTimeField( + verbose_name=_('Created'), + auto_now=True) + + filename = models.CharField( + verbose_name=_('Identifier'), + max_length=128) + + archive = models.FileField( + verbose_name=_('Archive'), + max_length=256) + + state = models.CharField( + verbose_name=_('State'), + choices=STATES, + default=STATE_RECEIVED, + max_length=16) + + url = models.URLField( + verbose_name=_('URL'), + blank=True) + + class Meta: + verbose_name = _('MDEL request') + verbose_name_plural = _('MDEL requests') + unique_together = ( + ('resource', 'filename'), + ) diff --git a/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html b/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html new file mode 100644 index 00000000..feadd733 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html @@ -0,0 +1,6 @@ +
+ {% include widget.subwidgets.0.template_name with widget=widget.subwidgets.0 %} +
+
+ {% include widget.subwidgets.1.template_name with widget=widget.subwidgets.1 %} +
diff --git a/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html b/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html new file mode 100644 index 00000000..fc792da3 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html @@ -0,0 +1,9 @@ +{% extends "passerelle/manage/resource_child_confirm_delete.html" %} + +{% block resource-child-breadcrumb %} + {% if object.id %} + {{ object.get_procedure_display }} + {% else %} + {% trans "Add mapping" %} + {% endif %} +{% endblock %} diff --git a/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html b/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html new file mode 100644 index 00000000..3fcc1247 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html @@ -0,0 +1,55 @@ +{% extends "passerelle/manage/resource_child_form.html" %} +{% load i18n %} + +{% comment %} +{% block resource-child-breadcrumb %} + {% if object.id %} + {{ object }} + {% else %} + {% trans "Add mapping" %} + {% endif %} +{% endblock %} +{% endcomment %} + +{% block form %} + {% if form.errors %} +
+

{% trans "There were errors processing your form." %}

+ {% for error in form.non_field_errors %} +

{{ error }}

+ {% endfor %} + {% for field in form %} + {% if field.is_hidden and field.errors %} +

+ {% for error in field.errors %} + {% blocktrans with name=field.name %}(Hidden field {{name}}) {{ error }}{% endblocktrans %} + {% if not forloop.last %}
{% endif %} + {% endfor %} +

+ {% endif %} + {% endfor %} +
+ {% endif %} + {% include "gadjo/widget.html" with field=form.procedure %} + + {% include "gadjo/widget.html" with field=form.formdef%} + + {% if form.table_fields %} + + + + + + + + + {% for field in form.table_fields %} + + + + + {% endfor %} + +
LabelVariable
{{ field.label_tag }}{{ field }}
+ {% endif %} +{% endblock %} diff --git a/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html b/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html new file mode 100644 index 00000000..7d4ed180 --- /dev/null +++ b/passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html @@ -0,0 +1,69 @@ +{% extends "passerelle/manage/service_view.html" %} +{% load i18n passerelle %} + +{% block description %} +

+ {% blocktrans %} + Connector to forms published by service-public.fr + {% endblocktrans %} + Run +

+ {{ block.super }} +{% endblock %} + + +{% block extra-sections %} +
+

{% trans "Mappings" %} {% trans "Add" %}

+ +
+
+

{% trans "Mappings" %} {% trans "Add" %}

+ + + + + + + + + + + + + + {% for req in object.request_set.all %} + + + + + + + + + + {% endfor %} + + +
IdCreatedModifiedStateFilenameSlugForm Id
{{ req.id }}{{ req.created }}{{ req.modified }}{{ req.get_state_display }}{{ req.filename }}{{ req.content.formdef_slug }}{{ req.content.formdata_id }}
+
+{% endblock %} diff --git a/passerelle/apps/sp_fr/urls.py b/passerelle/apps/sp_fr/urls.py new file mode 100644 index 00000000..63867a0a --- /dev/null +++ b/passerelle/apps/sp_fr/urls.py @@ -0,0 +1,30 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.conf.urls import url + +from . import views + +management_urlpatterns = [ + url(r'^(?P[\w,-]+)/mapping/new/$', + views.MappingNew.as_view(), name='sp-fr-mapping-new'), + url(r'^(?P[\w,-]+)/mapping/(?P\d+)/$', + views.MappingEdit.as_view(), name='sp-fr-mapping-edit'), + url(r'^(?P[\w,-]+)/mapping/(?P\d+)/delete/$', + views.MappingDelete.as_view(), name='sp-fr-mapping-delete'), + url(r'^(?P[\w,-]+)/run/$', + views.run, name='sp-fr-run'), +] diff --git a/passerelle/apps/sp_fr/views.py b/passerelle/apps/sp_fr/views.py new file mode 100644 index 00000000..280049d5 --- /dev/null +++ b/passerelle/apps/sp_fr/views.py @@ -0,0 +1,67 @@ +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.views.generic import UpdateView, CreateView, DeleteView +from django.shortcuts import get_object_or_404 +from django.http import HttpResponseRedirect + +from passerelle.base.mixins import ResourceChildViewMixin + +from . import models, forms + + +class StayIfChanged(object): + has_changed = False + + def form_valid(self, form): + if set(form.changed_data) & set(['procedure', 'formdef']): + self.has_changed = True + return super(StayIfChanged, self).form_valid(form) + + def get_success_url(self): + if self.has_changed: + return self.get_changed_url() + return super(StayIfChanged, self).get_success_url() + + def get_changed_url(self): + return '' + + +class MappingNew(StayIfChanged, ResourceChildViewMixin, CreateView): + model = models.Mapping + form_class = forms.MappingForm + + def form_valid(self, form): + form.instance.resource = self.resource + return super(MappingNew, self).form_valid(form) + + def get_changed_url(self): + return self.object.get_absolute_url() + + +class MappingEdit(StayIfChanged, ResourceChildViewMixin, UpdateView): + model = models.Mapping + form_class = forms.MappingForm + + +class MappingDelete(ResourceChildViewMixin, DeleteView): + model = models.Mapping + + +def run(request, connector, slug): + resource = get_object_or_404(models.Resource, slug=slug) + resource.run_loop(1000) + return HttpResponseRedirect(resource.get_absolute_url()) diff --git a/passerelle/settings.py b/passerelle/settings.py index 2d7d4e20..630d1b4a 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -135,6 +135,7 @@ INSTALLED_APPS = ( 'passerelle.apps.feeds', 'passerelle.apps.gdc', 'passerelle.apps.jsondatastore', + 'passerelle.apps.sp_fr', 'passerelle.apps.mobyt', 'passerelle.apps.okina', 'passerelle.apps.opengis', diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css index 7daf5762..67260599 100644 --- a/passerelle/static/css/style.css +++ b/passerelle/static/css/style.css @@ -188,3 +188,9 @@ li.connector.status-down span.connector-name::after { .log-dialog table td { vertical-align: top; } +.expression-widget input { + width: 100%; +} +.variable-widget select { + width: 100%; +} diff --git a/tests/test_sp_fr.py b/tests/test_sp_fr.py new file mode 100644 index 00000000..340c8b48 --- /dev/null +++ b/tests/test_sp_fr.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# passerelle - uniform access to multiple data sources and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest + +from passerelle.apps.sp_fr.models import Resource + +import utils + + +DUMMY_CONTENT = { + 'DILA': { + 'a.zip': 'a', + } +} + + +@pytest.fixture +def spfr(settings, db, sftpserver): + settings.KNOWN_SERVICES = { + 'wcs': { + 'eservices': { + 'title': u'Démarches', + 'url': 'https://demarches-hautes-alpes.test.entrouvert.org/', + 'secret': '9a1f62b680c1cabe73cefcbc5ff4cd4f95c24c3be8dd930adb80d8aaa33bfe67', + 'orig': 'passerelle-hautes-alpes.test.entrouvert.org', + } + } + } + yield utils.make_resource( + Resource, + title='Test 1', + slug='test1', + description='Connecteur de test', + input_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver), + output_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver) + ) + + +def test_resource(spfr): + from passerelle.utils.wcs import get_wcs_choices + + assert get_wcs_choices() == [] + + +def test_sftp_access(spfr, sftpserver): + with sftpserver.serve_content(DUMMY_CONTENT): + with spfr.input_sftp as input_sftp: + assert input_sftp.listdir() == ['a.zip'] + with spfr.output_sftp as output_sftp: + assert output_sftp.listdir() == ['a.zip'] -- 2.20.1