From 690cadc6a9886b24ed6e4642aa61b88933587f21 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 18 Oct 2019 10:54:20 +0200 Subject: [PATCH 11/11] add MDEL DDPACS connector (#35818) --- passerelle/apps/mdel_ddpacs/__init__.py | 0 passerelle/apps/mdel_ddpacs/abstract.py | 402 ++++++++++++++++++ .../mdel_ddpacs/migrations/0001_initial.py | 59 +++ .../apps/mdel_ddpacs/migrations/__init__.py | 0 passerelle/apps/mdel_ddpacs/models.py | 57 +++ passerelle/apps/mdel_ddpacs/schema.xsd | 129 ++++++ .../mdel_ddpacs/templates/mdel/zip/doc.xml | 106 +++++ .../mdel_ddpacs/templates/mdel/zip/entete.xml | 25 ++ .../templates/mdel/zip/manifest.json | 17 + .../templates/mdel/zip/message.xml | 66 +++ passerelle/apps/mdel_ddpacs/utils.py | 83 ++++ passerelle/settings.py | 1 + passerelle/utils/zip.py | 39 ++ tests/data/mdel_ddpacs/response_manifest.json | 9 + tests/data/mdel_ddpacs/response_message.xml | 31 ++ tests/data/mdel_ddpacs_expected.zip | Bin 0 -> 7465 bytes tests/test_mdel_ddpacs.py | 148 +++++++ 17 files changed, 1172 insertions(+) create mode 100644 passerelle/apps/mdel_ddpacs/__init__.py create mode 100644 passerelle/apps/mdel_ddpacs/abstract.py create mode 100644 passerelle/apps/mdel_ddpacs/migrations/0001_initial.py create mode 100644 passerelle/apps/mdel_ddpacs/migrations/__init__.py create mode 100644 passerelle/apps/mdel_ddpacs/models.py create mode 100644 passerelle/apps/mdel_ddpacs/schema.xsd create mode 100644 passerelle/apps/mdel_ddpacs/templates/mdel/zip/doc.xml create mode 100644 passerelle/apps/mdel_ddpacs/templates/mdel/zip/entete.xml create mode 100644 passerelle/apps/mdel_ddpacs/templates/mdel/zip/manifest.json create mode 100644 passerelle/apps/mdel_ddpacs/templates/mdel/zip/message.xml create mode 100644 passerelle/apps/mdel_ddpacs/utils.py create mode 100644 tests/data/mdel_ddpacs/response_manifest.json create mode 100644 tests/data/mdel_ddpacs/response_message.xml create mode 100644 tests/data/mdel_ddpacs_expected.zip create mode 100644 tests/test_mdel_ddpacs.py diff --git a/passerelle/apps/mdel_ddpacs/__init__.py b/passerelle/apps/mdel_ddpacs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/mdel_ddpacs/abstract.py b/passerelle/apps/mdel_ddpacs/abstract.py new file mode 100644 index 00000000..3aa3d4dd --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/abstract.py @@ -0,0 +1,402 @@ +# coding: utf-8 +# Passerelle - uniform access to data and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import unicode_literals + +from collections import namedtuple +import inspect +import os +import re +import xml.etree.ElementTree as ET +import zipfile + +from django.db import models, IntegrityError +from django.core.urlresolvers import reverse +from django.http import HttpResponse +from django.utils.translation import ugettext_lazy as _ +from django.utils import six, functional, html + +import xmlschema + +import jsonfield + +from passerelle.base.models import BaseResource, SkipJob +from passerelle.utils.api import endpoint +from passerelle.utils.jsonresponse import APIError +from passerelle.utils.zip import ZipTemplate +from passerelle.utils.conversion import exception_to_text + +from passerelle.utils import json, xml, sftp + +'''Base abstract models for implementing MDEL compatible requests. +''' + +MDELStatus = namedtuple('MDELStatus', ['code', 'slug', 'label']) + +MDEL_STATUSES = map(lambda t: MDELStatus(*t), [ + ('100', 'closed', _('closed')), + ('20', 'rejected', _('rejected')), + ('19', 'accepted', _('accepted')), + ('17', 'information needed', _('information needed')), + ('16', 'in progress', _('in progress')), + ('15', 'invalid', _('invalid')), + ('14', 'imported', _('imported')), +]) + +MDEL_STATUSES_BY_CODE = {mdel_status.code: mdel_status for mdel_status in MDEL_STATUSES} + + +class Resource(BaseResource): + outcoming_sftp = sftp.SFTPField( + verbose_name=_('Outcoming SFTP'), + blank=True, + ) + incoming_sftp = sftp.SFTPField( + verbose_name=_('Incoming SFTP'), + blank=True, + ) + recipient_siret = models.CharField( + verbose_name=_('SIRET'), + max_length=128) + recipient_service = models.CharField( + verbose_name=_('Service'), + max_length=128) + recipient_guichet = models.CharField( + verbose_name=_('Guichet'), + max_length=128) + code_insee = models.CharField( + verbose_name=_('INSEE Code'), + max_length=6) + + xsd_path = 'schema.xsd' + xsd_root_element = None + flow_type = 'flow_type CHANGEME' + doc_type = 'doc_type CHANGEME' + zip_manifest = 'mdel/zip/manifest.json' + code_insee_id = 'CODE_INSEE' + + class Meta: + abstract = True + + def check_status(self): + if self.outcoming_sftp: + with self.outcoming_sftp.client() as out_sftp: + out_sftp.listdir() + if self.incoming_sftp: + with self.incoming_sftp.client() as in_sftp: + in_sftp.listdir() + + @classmethod + def get_doc_xml_schema(cls): + base_dir = os.path.dirname(inspect.getfile(cls)) + path = os.path.join(base_dir, cls.xsd_path) + assert os.path.exists(path) + return xmlschema.XMLSchema(path, converter=xmlschema.UnorderedConverter) + + @classmethod + def get_doc_json_schema(cls): + return xml.JSONSchemaFromXMLSchema(cls.get_doc_xml_schema(), cls.xsd_root_element).json_schema + + @classmethod + def get_flattened_schema(cls): + return json.flatten_json_schema(cls.get_doc_json_schema()) + + @classmethod + def render_schema(cls): + schema = cls.get_flattened_schema() + table = html.format_html('''\ + + + +''', _('Label'), _('Type')) + for key in sorted(schema['properties']): + key_schema = schema['properties'][key] + if 'type' in key_schema: + _type = key_schema['type'] + elif 'oneOf' in key_schema: + _type = ' | '.join(x['type'] for x in key_schema['oneOf']) + else: + raise NotImplementedError + table += html.format_html('''\ + +''', key, _type) + table += html.format_html('
{0}{1}
{0}{1}
') + return table + + @classmethod + def get_create_schema(cls): + base_schema = cls.get_doc_json_schema() + base_schema['unflatten'] = True + base_schema['merge_extra'] = True + base_schema['properties'].update({ + 'display_id': {'type': 'string'}, + 'email': {'type': 'string'}, + 'code_insee': {'type': 'string'}, + }) + base_schema.setdefault('required', []).append('display_id') + return base_schema + + def _handle_create(self, request, payload): + reference = 'A-' + payload['display_id'] + try: + demand = self.demand_set.create( + reference=reference, + step=1, + data=payload) + except IntegrityError as e: + return APIError('reference-non-unique', http_status=400, + data={'original_exc': exception_to_text(e)}) + self.add_job('push_demand', demand_id=demand.id) + return self.status(request, demand) + + def push_demand(self, demand_id): + demand = self.demand_set.get(id=demand_id) + if not demand.push(): + raise SkipJob(after_timestamp=3600 * 6) + + @endpoint(perm='can_access', + methods=['get'], + description=_('Demand status'), + pattern=r'(?P\d+)/$') + def demand(self, request, demand_id): + try: + demand = self.demand_set.get(id=demand_id) + except self.demand_set.model.DoesNotExist: + raise APIError('demand not found', http_status=404) + return self.status(request, demand) + + def status(self, request, demand): + return { + 'id': demand.id, + 'status': demand.status, + 'url': request.build_absolute_uri(demand.status_url), + 'zip_url': request.build_absolute_uri(demand.zip_url), + } + + @endpoint(perm='can_access', + methods=['get'], + description=_('Demand document'), + pattern=r'(?P\d+)/.*$') + def document(self, request, demand_id): + try: + demand = self.demand_set.get(id=demand_id) + except self.demand_set.model.DoesNotExist: + raise APIError('demand not found', http_status=404) + response = HttpResponse(demand.zip_content, content_type='application/octet-stream') + response['Content-Disposition'] = 'inline; filename=%s' % demand.zip_name + return response + + @property + def response_re(self): + return re.compile( + r'(?P[^-]+-[^-]+-[^-]+)-%s-' + r'(?P\d+).zip' % self.flow_type) + + def hourly(self): + '''Get responses''' + if not self.incoming_sftp: + return + try: + with self.incoming_sftp.client() as client: + for name in client.listdir(): + m = self.response_re.match(name) + if not m: + self.logger.warning( + 'pull responses: unexpected file "%s" in sftp, file name does not match pattern %s', + name, self.response_re) + continue + reference = m.groupdict()['reference'] + step = int(m.groupdict()['step']) + demand = self.demand_set.filter(reference=reference).first() + if not demand: + self.logger.error( + 'pull responses: unexpected file "%s" in sftp, no demand for reference "%s"', + name, + reference) + continue + if step < demand.step: + demand.logger.error( + 'pull responses: unexpected file "%s" in sftp: step %s is inferior to demand step %s', + name, + step, + demand.step) + continue + demand.handle_response(sftp_client=client, filename=name, step=step) + except sftp.paramiko.SSHException as e: + self.logger.error('pull responses: sftp error %s', e) + return + + +@six.python_2_unicode_compatible +class Demand(models.Model): + STATUS_PENDING = 'pending' + STATUS_PUSHED = 'pushed' + STATUS_ERROR = 'error' + + STATUSES = [ + (STATUS_PENDING, _('pending')), + (STATUS_PUSHED, _('pushed')), + (STATUS_ERROR, _('error')), + ] + for mdel_status in MDEL_STATUSES: + STATUSES.append((mdel_status.slug, mdel_status.label)) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + reference = models.CharField(max_length=32, null=False, unique=True) + status = models.CharField(max_length=32, null=True, choices=STATUSES, default=STATUS_PENDING) + step = models.IntegerField(default=0) + data = jsonfield.JSONField() + + @functional.cached_property + def logger(self): + return self.resource.logger.context( + demand_id=self.id, + demand_status=self.status, + demand_reference=self.reference) + + def push(self): + if not self.resource.outcoming_sftp: + return False + try: + with self.resource.outcoming_sftp.client() as client: + with client.open(self.zip_name, mode='w') as fd: + fd.write(self.zip_content) + except sftp.paramiko.SSHException as e: + self.logger.error('push demand: %s failed, "%s"', + self, + exception_to_text(e)) + self.status = self.STATUS_ERROR + except Exception as e: + self.logger.exception('push demand: %s failed, "%s"', + self, + exception_to_text(e)) + self.status = self.STATUS_ERROR + else: + self.resource.logger.info('push demand: %s success', self) + self.status = self.STATUS_PUSHED + self.save() + return True + + @functional.cached_property + def zip_template(self): + return ZipTemplate(self.resource.zip_manifest, ctx={ + 'reference': self.reference, + 'flow_type': self.resource.flow_type, + 'doc_type': self.resource.doc_type, + 'step': '1', # We never create more than one document for a reference + 'siret': self.resource.recipient_siret, + 'service': self.resource.recipient_service, + 'guichet': self.resource.recipient_guichet, + 'code_insee': self.data.get('code_insee', self.resource.code_insee), + 'document': self.document, + 'code_insee_id': self.resource.code_insee_id, + 'date': self.created_at.isoformat(), + 'email': self.data.get('email', ''), + }) + + @property + def zip_name(self): + return self.zip_template.name + + @property + def zip_content(self): + return self.zip_template.render_to_bytes() + + @property + def document(self): + xml_schema = self.resource.get_doc_xml_schema() + return ET.tostring( + xml_schema.elements[self.resource.xsd_root_element].encode( + self.data[self.resource.xsd_root_element])) + + @property + def status_url(self): + return reverse( + 'generic-endpoint', + kwargs={ + 'connector': self.resource.get_connector_slug(), + 'slug': self.resource.slug, + 'endpoint': 'demand', + 'rest': '%s/' % self.id, + }) + + @property + def zip_url(self): + return reverse( + 'generic-endpoint', + kwargs={ + 'connector': self.resource.get_connector_slug(), + 'slug': self.resource.slug, + 'endpoint': 'document', + 'rest': '%s/%s' % (self.id, self.zip_name) + }) + + def handle_response(self, sftp_client, filename, step): + try: + with sftp_client.open(filename) as fd: + with zipfile.ZipFile(fd) as zip_file: + with zip_file.open('message.xml') as fd: + tree = ET.parse(fd) + ns = 'http://finances.gouv.fr/dgme/pec/message/v1' + etat_node = tree.find('.//{%s}Etat' % ns) + if etat_node is None: + self.logger.error( + 'pull responses: missing Etat node in "%s"', + filename) + return + etat = etat_node.text + if etat in MDEL_STATUSES_BY_CODE: + self.status = MDEL_STATUSES_BY_CODE[etat].slug + else: + self.logger.error( + 'pull responses: unknown etat in "%s", etat="%s"', + filename, + etat) + return + commentaire_node = tree.find('.//{%s}Etat' % ns) + if commentaire_node is not None: + commentaire = commentaire_node.text + self.data = self.data or {} + self.data.setdefault('commentaires', []).append(commentaire) + self.data['commentaire'] = commentaire + self.step = step + 1 + self.save() + self.logger.info('pull responses: status of demand %s changed to %s', + self, self.status) + except sftp.paramiko.SSHException as e: + self.logger.error( + 'pull responses: failed to read response "%s", %s', + filename, + exception_to_text(e)) + else: + try: + sftp_client.remove(filename) + except sftp.paramiko.SSHException as e: + self.logger.error( + 'pull responses: failed to remove response "%s", %s', + filename, + exception_to_text(e)) + + def __str__(self): + return '' % ( + self.id, + self.reference, + self.resource.flow_type) + + class Meta: + abstract = True diff --git a/passerelle/apps/mdel_ddpacs/migrations/0001_initial.py b/passerelle/apps/mdel_ddpacs/migrations/0001_initial.py new file mode 100644 index 00000000..1d1028f4 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/migrations/0001_initial.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-10-24 08:59 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion +import jsonfield.fields +import passerelle.utils.sftp + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('base', '0015_auto_20190921_0347'), + ] + + operations = [ + migrations.CreateModel( + name='Demand', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('reference', models.CharField(max_length=32, unique=True)), + ('status', models.CharField(choices=[('pending', 'pending'), ('pushed', 'pushed'), ('error', 'error'), ('closed', 'closed'), ('rejected', 'rejected'), ('accepted', 'accepted'), ('information needed', 'information needed'), ('in progress', 'in progress'), ('invalid', 'invalid'), ('imported', 'imported')], default='pending', max_length=32, null=True)), + ('step', models.IntegerField(default=0)), + ('data', jsonfield.fields.JSONField(default=dict)), + ], + options={ + 'verbose_name': 'MDEL compatible DDPACS request', + }, + ), + migrations.CreateModel( + name='Resource', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=50, verbose_name='Title')), + ('description', models.TextField(verbose_name='Description')), + ('slug', models.SlugField(unique=True, verbose_name='Identifier')), + ('outcoming_sftp', passerelle.utils.sftp.SFTPField(blank=True, default=None, verbose_name='Outcoming SFTP')), + ('incoming_sftp', passerelle.utils.sftp.SFTPField(blank=True, default=None, verbose_name='Incoming SFTP')), + ('recipient_siret', models.CharField(max_length=128, verbose_name='SIRET')), + ('recipient_service', models.CharField(max_length=128, verbose_name='Service')), + ('recipient_guichet', models.CharField(max_length=128, verbose_name='Guichet')), + ('code_insee', models.CharField(max_length=6, verbose_name='INSEE Code')), + ('users', models.ManyToManyField(blank=True, related_name='_resource_users_+', related_query_name='+', to='base.ApiUser')), + ], + options={ + 'verbose_name': 'MDEL compatible DDPACS request builder', + }, + ), + migrations.AddField( + model_name='demand', + name='resource', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mdel_ddpacs.Resource'), + ), + ] diff --git a/passerelle/apps/mdel_ddpacs/migrations/__init__.py b/passerelle/apps/mdel_ddpacs/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/passerelle/apps/mdel_ddpacs/models.py b/passerelle/apps/mdel_ddpacs/models.py new file mode 100644 index 00000000..4995dbd1 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/models.py @@ -0,0 +1,57 @@ +# coding: utf-8 +# Passerelle - uniform access to data and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import unicode_literals + +from django.db import models +from django.utils.translation import ugettext_lazy as _ + +from passerelle.utils.api import endpoint + +from . import abstract + + +class Resource(abstract.Resource): + category = _('Civil Status Connectors') + xsd_root_element = 'PACS' + flow_type = 'depotDossierPACS' + doc_type = 'flux-pacs' + + class Meta: + verbose_name = _('MDEL compatible DDPACS request builder') + + @endpoint(perm='can_access', + methods=['post'], + description=_('Create request'), + post={ + 'request_body': { + 'schema': { + 'application/json': None + } + } + }) + def create(self, request, post_data): + return self._handle_create(request, post_data) + +Resource.create.endpoint_info.post['request_body']['schema']['application/json'] = Resource.get_create_schema() + + +class Demand(abstract.Demand): + resource = models.ForeignKey(Resource) + + class Meta: + verbose_name = _('MDEL compatible DDPACS request') diff --git a/passerelle/apps/mdel_ddpacs/schema.xsd b/passerelle/apps/mdel_ddpacs/schema.xsd new file mode 100644 index 00000000..58ae5383 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/schema.xsd @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/mdel_ddpacs/templates/mdel/zip/doc.xml b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/doc.xml new file mode 100644 index 00000000..efe7fa51 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/doc.xml @@ -0,0 +1,106 @@ + + + + {{ partenaire1.civilite }} + {{ partenaire1.nom_naissance }} + {{ partenaire1.prenoms }} + {% for code_nationalite in partenaire1.code_nationalite %} + {{ code_nationalite }} + {% endfor %} + {{ partenaire1.jour_naissance }} + {{ partenaire1.mois_naissance }} + {{ partenaire1.annee_naissance }} + + {{ partenaire1.localite_naissance }} + {{ partenaire1.codepostal_naissance }} + {{ partenaire1.codeinsee_naissance }} + {{ partenaire1.departement_naissance }} + {{ partenaire1.codepays_naissance }} + + {{ partenaire1.ofpra|yesno:"true,false" }} + {{ partenaire1.mesure_juridique }} + + {{ partenaire1.adresse_numero_voie }} + {{ partenaire1.adresse_complement1 }} + {{ partenaire1.adresse_complement2 }} + {{ partenaire1.adresse_lieuditbpcommunedeleguee }} + {{ partenaire1.adresse_codepostal }} + {{ partenaire1.adresse_localite }} + {{ partenaire1.adresse_pays }} + + {{ partenaire1.email }} + {{ partenaire1.telephone }} + {{ partenaire1.yesno:"true,false" }} + + + {{ partenaire2.civilite }} + {{ partenaire2.nom_naissance }} + {{ partenaire2.prenoms }} + {% for code_nationalite in partenaire2.code_nationalite %} + {{ code_nationalite }} + {% endfor %} + {{ partenaire2.jour_naissance }} + {{ partenaire2.mois_naissance }} + {{ partenaire2.annee_naissance }} + + {{ partenaire2.localite_naissance }} + {{ partenaire2.codepostal_naissance }} + {{ partenaire2.codeinsee_naissance }} + {{ partenaire2.departement_naissance }} + {{ partenaire2.codepays_naissance }} + + {{ partenaire2.ofpra|yesno:"true,false" }} + {{ partenaire2.mesure_juridique }} + + {{ partenaire2.adresse_numero_voie }} + {{ partenaire2.adresse_complement1 }} + {{ partenaire2.adresse_complement2 }} + {{ partenaire2.adresse_lieuditbpcommunedeleguee }} + {{ partenaire2.adresse_codepostal }} + {{ partenaire2.adresse_localite }} + {{ partenaire2.adresse_pays }} + + {{ partenaire2.email }} + {{ partenaire2.telephone }} + {{ partenaire2.yesno:"true,false" }} + + + + 100000 + legal + + aideFixe + + + + + 3 place du test + 05100 + VILLAR ST PANCRACE + + + + true + true + + + + + + + + + + + + + + + + + + + + + + diff --git a/passerelle/apps/mdel_ddpacs/templates/mdel/zip/entete.xml b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/entete.xml new file mode 100644 index 00000000..4c044c18 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/entete.xml @@ -0,0 +1,25 @@ + + + {{ flow_type }} + + {{ reference }} + + + {{ date }} + Publik + {{ email }} + + + + {{ code_insee_id }} + {{ code_insee }} + + + + {{ doc_type }} + {{ doc_type }} + + {{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml + + + diff --git a/passerelle/apps/mdel_ddpacs/templates/mdel/zip/manifest.json b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/manifest.json new file mode 100644 index 00000000..ab2149e8 --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/manifest.json @@ -0,0 +1,17 @@ +{ + "name_template": "{{ reference }}-{{ flow_type }}-{{ step }}.zip", + "part_templates": [ + { + "name_template": "message.xml", + "template_path": "message.xml" + }, + { + "name_template": "{{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml", + "content_expression": "document" + }, + { + "name_template": "{{ reference }}-{{ flow_type }}-ent-1.xml", + "template_path": "entete.xml" + } + ] +} diff --git a/passerelle/apps/mdel_ddpacs/templates/mdel/zip/message.xml b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/message.xml new file mode 100644 index 00000000..52a1e68c --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/templates/mdel/zip/message.xml @@ -0,0 +1,66 @@ + + + + + {{ reference }} {{ step }} + {{ flow_type }} + + FR + + 13000210800012 + flux_GS_PEC_AVL + + + + + + FR + + {{ siret }} + {{ service }} + {{ guichet }} + + + + true + AVL + ANL + + FR + + 13000210800012 + flux_GS_PEC_AVL + + + + + + + + + {{ flow_type }} + + {{ reference }} + + + {{ date }} + Publik + {{ email }} + + + + {{ code_insee_id }} + {{ code_insee }} + + + + {{ doc_type }} + {{ doc_type }} + + {{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml + + + + + + diff --git a/passerelle/apps/mdel_ddpacs/utils.py b/passerelle/apps/mdel_ddpacs/utils.py new file mode 100644 index 00000000..4fc5589e --- /dev/null +++ b/passerelle/apps/mdel_ddpacs/utils.py @@ -0,0 +1,83 @@ +# Passerelle - uniform access to data and services +# Copyright (C) 2016 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import zipfile +from xml.etree import ElementTree as etree + +from django.utils.dateparse import parse_date as django_parse_date + +from passerelle.utils.jsonresponse import APIError + +def parse_date(date): + try: + parsed_date = django_parse_date(date) + except ValueError as e: + raise APIError('Invalid date: %r (%r)' % ( date, e)) + if not parsed_date: + raise APIError('date %r not iso-formated' % date) + return parsed_date.isoformat() + + +class ElementFactory(etree.Element): + + def __init__(self, *args, **kwargs): + self.text = kwargs.pop('text', None) + namespace = kwargs.pop('namespace', None) + if namespace: + super(ElementFactory, self).__init__( + etree.QName(namespace, args[0]), **kwargs + ) + self.namespace = namespace + else: + super(ElementFactory, self).__init__(*args, **kwargs) + + def append(self, element, allow_new=True): + + if not allow_new: + if isinstance(element.tag, etree.QName): + found = self.find(element.tag.text) + else: + found = self.find(element.tag) + + if found is not None: + return self + + super(ElementFactory, self).append(element) + return self + + def extend(self, elements): + super(ElementFactory, self).extend(elements) + return self + + +def zipdir(path): + """Zip directory + """ + archname = path + '.zip' + with zipfile.ZipFile(archname, 'w', zipfile.ZIP_DEFLATED) as zipf: + for root, dirs, files in os.walk(path): + for f in files: + fpath = os.path.join(root, f) + zipf.write(fpath, os.path.basename(fpath)) + return archname + + +def get_file_content_from_zip(path, filename): + """Rreturn file content + """ + with zipfile.ZipFile(path, 'r') as zipf: + return zipf.read(filename) diff --git a/passerelle/settings.py b/passerelle/settings.py index 0190fa18..d7a44185 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -140,6 +140,7 @@ INSTALLED_APPS = ( 'passerelle.apps.jsondatastore', 'passerelle.apps.sp_fr', 'passerelle.apps.mdel', + 'passerelle.apps.mdel_ddpacs', 'passerelle.apps.mobyt', 'passerelle.apps.okina', 'passerelle.apps.opengis', diff --git a/passerelle/utils/zip.py b/passerelle/utils/zip.py index a4dd60b9..c6b99697 100644 --- a/passerelle/utils/zip.py +++ b/passerelle/utils/zip.py @@ -16,6 +16,7 @@ from __future__ import unicode_literals, absolute_import +import difflib import io import os.path import json @@ -239,3 +240,41 @@ class ZipTemplate(object): full_path = os.path.join(str(path), self.name) with atomic_write(full_path, dir=tmp_dir) as fd: self.render_to_file(fd) + + +def diff_zip(one, two): + differences = [] + + def compute_diff(one, two, fd_one, fd_two): + content_one = fd_one.read() + content_two = fd_two.read() + + if content_one == content_two: + return + if one.endswith(('.xml', '.json', '.txt')): + diff = list(difflib.ndiff(content_one.splitlines(), + content_two.splitlines())) + return ['File %s differs' % one] + diff + return 'File %s differs' % one + + if not hasattr(one, 'read'): + one = open(one) + with one: + if not hasattr(two, 'read'): + two = open(two) + with two: + with zipfile.ZipFile(one) as one_zip: + with zipfile.ZipFile(two) as two_zip: + one_nl = set(one_zip.namelist()) + two_nl = set(two_zip.namelist()) + for name in one_nl - two_nl: + differences.append('File %s only in %s' % (name, one)) + for name in two_nl - one_nl: + differences.append('File %s only in %s' % (name, two)) + for name in one_nl & two_nl: + with one_zip.open(name) as fd_one: + with two_zip.open(name) as fd_two: + difference = compute_diff(name, name, fd_one, fd_two) + if difference: + differences.append(difference) + return differences diff --git a/tests/data/mdel_ddpacs/response_manifest.json b/tests/data/mdel_ddpacs/response_manifest.json new file mode 100644 index 00000000..1b21a012 --- /dev/null +++ b/tests/data/mdel_ddpacs/response_manifest.json @@ -0,0 +1,9 @@ +{ + "name_template": "{{ reference }}-{{ flow_type }}-{{ step }}.zip", + "part_templates": [ + { + "name_template": "message.xml", + "template_path": "response_message.xml" + } + ] +} diff --git a/tests/data/mdel_ddpacs/response_message.xml b/tests/data/mdel_ddpacs/response_message.xml new file mode 100644 index 00000000..c1e629b8 --- /dev/null +++ b/tests/data/mdel_ddpacs/response_message.xml @@ -0,0 +1,31 @@ + + + + {{ reference }} {{ step }} + {{ reference }} {{ old_step }} + {{ flow_type }} + + + + + + + false + + + + + + + {{ reference }} + + + + {% if etat %}{{ etat }}{% endif %} + {% if commentaire %}{{ commentaire }}{% endif %} + + + + + + diff --git a/tests/data/mdel_ddpacs_expected.zip b/tests/data/mdel_ddpacs_expected.zip new file mode 100644 index 0000000000000000000000000000000000000000..69c163073e4629f5dacf892450206298005ddeb4 GIT binary patch literal 7465 zcmeHMPj}ly5O)c1DBS3QLl3HXk4tOIveS^H8x_Zv8@IOWXS;cv$Wx`YyHwJu`jbDm zegnAh1-S89_!J!Z0Nk0~)qhKlL!mr)s7YMS{&seDW_D(FJbkwF;hmaz?oG~~{ruxk zpVVsX7k+9s_I-VYo0qnQ9>X8$9D6>Y&QV=yw(1gg3~CZ*byR<`7&P|lk{{@fsaw>+ zNA)Z0*B`65k>j@y$AXUpEFAw3C`a{ILEs+B@{%~ZV_?6zqTzXS>B;8G#Bqvo1G4O+7{MJ zrp7jV@?zLK(H7Sjl279Xixsr4b-|*-EQP^>QMtvt#_Jh=6M{>dYT$)fOvzGn{uR&R z2S0ILfOEnI=p^#;%}`phL3W=}woov1kq-QN_vD3Vf>6VSD+Y*J!$_Mk~l&=~R^p6b4jAr@y|q(-A+Wj<{ll5+wx#Ah|QuLoFdx0HiMOM#z7 zt98hK?(@G$=Hun!4NXuxSrRA}rWUYSQqRWfG<<21*GMiOE$5`!I)d!iG#JwTPH_9;u4SxO3YJq#nh`c!uS6u+h8aKpe8UcBmj!NIHx3l@mg@Z%FG zv~K{{AOTmyzoVg-ZmYeIMqpwuw;g^4jno^#(-=quw{l6ggg+=67v-v z0!9=Z2FY1c9?X~1uY9<`ME0Drw*-Xbimf8LbUj^N>Xr`+h?jI0TpB|UKMg%%k~bla zYPBLY#p|XASJOxph3F9_+Yxz*;aHwhf>lL=+D|e=DZqXZr^75D;mj@4gZrq(6{4tJ zH_^@m>B}$Loz5c`_#a5vXJ*45i-;+_icwpgiOD{J9A?1$m+;=+dlV3+c!`D*v26^Q z478InY^R##k}>7>NpS1}#10+YhpJ!|f{l--;H#CRMX3@eYCTk#6ZKNJ%0pSL$$j&5 zI2v_l61eEJJL%23>|&4%=I3yS;OG_rBUcK)kCU=ou^tJS<9v0H0zH%8JnKWR zyNEaquORz2OgOE(yN~SuU~p{TG&uerJtR~bBBJD0I<^)*qF{jx>YTG%4Rz93pCOiX z9&|_{V~CC(!PXeAqHr5hA&kS*WHm_1XcF~!67lc~lFk&Wc@?sUZXr2C$wRZuQt=oU z_8``h>}J+e8Gb-6;gqZm&^j(Vg%D7fcIyXXX<;{Vp@XQTv(SSkswr;uz=!QlMx><7t4d11q zf=ZewWsQ^dW(o8h*!3xOZXQjpX^NoDjSzN2Pni}P9wkkVYZ@DfbuW^R0{B6;nSrip zV8r8pZPNl(H7c^r3214Rogda_xb?xEo$Bul;Yjo6?Jf2JSFiXR#NxzaheAxN)!yFv n1-^T{Nx!1Vxfo(n&gSQpoRfevefH7EY{m}!`xQRhciGz?UK%e= literal 0 HcmV?d00001 diff --git a/tests/test_mdel_ddpacs.py b/tests/test_mdel_ddpacs.py new file mode 100644 index 00000000..4350815b --- /dev/null +++ b/tests/test_mdel_ddpacs.py @@ -0,0 +1,148 @@ +# coding: utf-8 +# Passerelle - uniform access to data and services +# Copyright (C) 2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a.deepcopy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import unicode_literals + +import io +import logging +import os + +import pytest +import utils + +from passerelle.apps.mdel_ddpacs.models import Resource, Demand + +from passerelle.utils import json, sftp +from passerelle.utils.zip import diff_zip, ZipTemplate + + +def build_response_zip(**kwargs): + zip_template = ZipTemplate(os.path.abspath('tests/data/mdel_ddpacs/response_manifest.json'), ctx=kwargs) + return zip_template.name, zip_template.render_to_bytes() + + +@pytest.fixture(autouse=True) +def resource(db): + return utils.setup_access_rights(Resource.objects.create( + slug='test', + code_insee='66666', + recipient_siret='999999', + recipient_service='SERVICE', + recipient_guichet='GUICHET')) + + +@pytest.fixture +def ddpacs_payload(): + xmlschema = Resource.get_doc_xml_schema() + return json.flatten({'PACS': xmlschema.to_dict('tests/data/pacs-doc.xml')}) + + +def test_create_demand(app, resource, ddpacs_payload, freezer, sftpserver, caplog): + # paramiko log socket errors when connection is closed :/ + caplog.set_level(logging.CRITICAL, 'paramiko.transport') + freezer.move_to('2019-01-01') + + # Push new demand + payload = { + 'display_id': '1-1', + } + payload.update(ddpacs_payload) + assert Demand.objects.count() == 0 + assert resource.jobs_set().count() == 0 + resp = app.post_json('/mdel-ddpacs/test/create?raise=1', params=payload) + assert resp.json['err'] == 0 + assert resp.json['status'] == 'pending' + assert Demand.objects.count() == 1 + assert resource.jobs_set().count() == 1 + + url = resp.json['url'] + zip_url = resp.json['zip_url'] + + # Check demand status URL + status = app.get(url) + assert status.json['err'] == 0 + assert status.json == resp.json + + # Check demand document URL + zip_document = app.get(zip_url) + with io.BytesIO(zip_document.body) as fd: + differences = diff_zip('tests/data/mdel_ddpacs_expected.zip', fd) + assert not differences, differences + + # Check job is skipped as no SFTP is configured + assert resource.jobs_set().get().after_timestamp is None + resource.jobs() + assert resource.jobs_set().get().after_timestamp is not None + assert resource.jobs_set().exclude(status='completed').count() == 1 + + with sftpserver.serve_content({'input': {}, 'output': {}}): + content = sftpserver.content_provider.content_object + resource.outcoming_sftp = sftp.SFTP( + 'sftp://john:doe@{server.host}:{server.port}/output/'.format( + server=sftpserver)) + resource.jobs() + assert not content['output'] + # Jump over the 6 hour wait time for retry + freezer.move_to('2019-01-02') + resource.jobs() + assert 'A-1-1-depotDossierPACS-1.zip' in content['output'] + # Check it's the same document than through the zip_url + with open('/tmp/zip.zip', 'w') as fd: + fd.write(content['output']['A-1-1-depotDossierPACS-1.zip']) + with io.BytesIO(content['output']['A-1-1-depotDossierPACS-1.zip']) as fd: + differences = diff_zip('tests/data/mdel_ddpacs_expected.zip', fd) + assert not differences, differences + # Act as if zip was consumed + content['output'] = {} + # Jump over the 6 hour wait time for retry + freezer.move_to('2019-01-03') + resource.jobs() + assert not content['output'] + assert resource.jobs_set().exclude(status='completed').count() == 0 + + # Check response + resource.hourly() + + resource.incoming_sftp = sftp.SFTP( + 'sftp://john:doe@{server.host}:{server.port}/input/'.format( + server=sftpserver)) + + response_name, response_content = build_response_zip( + reference='A-1-1', + flow_type='depotDossierPACS', + step=1, + old_step=1, + etat=100, + commentaire='coucou') + content['input'][response_name] = response_content + resource.hourly() + assert resource.demand_set.get().status == 'closed' + assert response_name not in content['input'] + + response_name, response_content = build_response_zip( + reference='A-1-1', + flow_type='depotDossierPACS', + step=1, + old_step=1, + etat=1, + commentaire='coucou') + content['input'][response_name] = response_content + resource.hourly() + assert 'unexpected file "A-1-1-depotDossierPACS-1.zip"' in caplog.messages[-1] + assert 'step 1 is inferior' in caplog.messages[-1] + + resource.check_status() -- 2.23.0