Produits Entr'ouvert » Passerelle

# coding: utf-8

# Passerelle - uniform access to data and services

# Copyright (C) 2019  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from collections import namedtuple

import inspect

import os

import re

import xml.etree.ElementTree as ET

import zipfile

from django.db import models, IntegrityError

from django.core.urlresolvers import reverse

from django.http import HttpResponse

from django.utils.translation import ugettext_lazy as _

from django.utils import six, functional, html

import xmlschema

import jsonfield

from passerelle.base.models import BaseResource, SkipJob

from passerelle.utils.api import endpoint

from passerelle.utils.jsonresponse import APIError

from passerelle.utils.zip import ZipTemplate

from passerelle.utils.conversion import exception_to_text

from passerelle.utils import json, xml, sftp

'''Base abstract models for implementing MDEL compatible requests.

'''

MDELStatus = namedtuple('MDELStatus', ['code', 'slug', 'label'])

MDEL_STATUSES = map(lambda t: MDELStatus(*t), [

    ('100', 'closed', _('closed')),

    ('20', 'rejected', _('rejected')),

    ('19', 'accepted', _('accepted')),

    ('17', 'information needed', _('information needed')),

    ('16', 'in progress', _('in progress')),

    ('15', 'invalid', _('invalid')),

    ('14', 'imported', _('imported')),

])

MDEL_STATUSES_BY_CODE = {mdel_status.code: mdel_status for mdel_status in MDEL_STATUSES}

class Resource(BaseResource):

    outcoming_sftp = sftp.SFTPField(

        verbose_name=_('Outcoming SFTP'),

        blank=True,

    )

    incoming_sftp = sftp.SFTPField(

        verbose_name=_('Incoming SFTP'),

        blank=True,

    )

    recipient_siret = models.CharField(

        verbose_name=_('SIRET'),

        max_length=128)

    recipient_service = models.CharField(

        verbose_name=_('Service'),

        max_length=128)

    recipient_guichet = models.CharField(

        verbose_name=_('Guichet'),

        max_length=128)

    code_insee = models.CharField(

        verbose_name=_('INSEE Code'),

        max_length=6)

    xsd_path = 'schema.xsd'

    xsd_root_element = None

    flow_type = 'flow_type CHANGEME'

    doc_type = 'doc_type CHANGEME'

    zip_manifest = 'mdel/zip/manifest.json'

    code_insee_id = 'CODE_INSEE'

    class Meta:

        abstract = True

    def check_status(self):

        if self.outcoming_sftp:

            with self.outcoming_sftp.client() as out_sftp:

                out_sftp.listdir()

            if self.incoming_sftp:

                with self.incoming_sftp.client() as in_sftp:

                    in_sftp.listdir()

    @classmethod

    def get_doc_xml_schema(cls):

        base_dir = os.path.dirname(inspect.getfile(cls))

        path = os.path.join(base_dir, cls.xsd_path)

        assert os.path.exists(path)

        return xmlschema.XMLSchema(path, converter=xmlschema.UnorderedConverter)

    @classmethod

    def get_doc_json_schema(cls):

        return xml.JSONSchemaFromXMLSchema(cls.get_doc_xml_schema(), cls.xsd_root_element).json_schema

    @classmethod

    def get_flattened_schema(cls):

        return json.flatten_json_schema(cls.get_doc_json_schema())

    @classmethod

    def render_schema(cls):

        schema = cls.get_flattened_schema()

        table = html.format_html('''\

<table>

   <thead><tr><td>{0}</td><td>{1}</td></tr>

   <tbody>

''', _('Label'), _('Type'))

        for key in sorted(schema['properties']):

            key_schema = schema['properties'][key]

            if 'type' in key_schema:

                _type = key_schema['type']

            elif 'oneOf' in key_schema:

                _type = ' | '.join(x['type'] for x in key_schema['oneOf'])

            else:

                raise NotImplementedError

            table += html.format_html('''\

     <tr><td><tt>{0}</tt></td><td><tt>{1}</tt></td></tr>

''', key, _type)

        table += html.format_html('</tbody></table>')

        return table

    @classmethod

    def get_create_schema(cls):

        base_schema = cls.get_doc_json_schema()

        base_schema['unflatten'] = True

        base_schema['merge_extra'] = True

        base_schema['properties'].update({

            'display_id': {'type': 'string'},

            'email': {'type': 'string'},

            'code_insee': {'type': 'string'},

        })

        base_schema.setdefault('required', []).append('display_id')

        return base_schema

    def _handle_create(self, request, payload):

        reference = 'A-' + payload['display_id']

        try:

            demand = self.demand_set.create(

                reference=reference,

                step=1,

                data=payload)

        except IntegrityError as e:

            return APIError('reference-non-unique', http_status=400,

                            data={'original_exc': exception_to_text(e)})

        self.add_job('push_demand', demand_id=demand.id)

        return self.status(request, demand)

    def push_demand(self, demand_id):

        demand = self.demand_set.get(id=demand_id)

        if not demand.push():

            raise SkipJob(after_timestamp=3600 * 6)

    @endpoint(perm='can_access',

              methods=['get'],

              description=_('Demand status'),

              pattern=r'(?P<demand_id>\d+)/$')

    def demand(self, request, demand_id):

        try:

            demand = self.demand_set.get(id=demand_id)

        except self.demand_set.model.DoesNotExist:

            raise APIError('demand not found', http_status=404)

        return self.status(request, demand)

    def status(self, request, demand):

        return {

            'id': demand.id,

            'status': demand.status,

            'url': request.build_absolute_uri(demand.status_url),

            'zip_url': request.build_absolute_uri(demand.zip_url),

        }

    @endpoint(perm='can_access',

              methods=['get'],

              description=_('Demand document'),

              pattern=r'(?P<demand_id>\d+)/.*$')

    def document(self, request, demand_id):

        try:

            demand = self.demand_set.get(id=demand_id)

        except self.demand_set.model.DoesNotExist:

            raise APIError('demand not found', http_status=404)

        response = HttpResponse(demand.zip_content, content_type='application/octet-stream')

        response['Content-Disposition'] = 'inline; filename=%s' % demand.zip_name

        return response

    @property

    def response_re(self):

        return re.compile(

            r'(?P<reference>[^-]+-[^-]+-[^-]+)-%s-'

            r'(?P<step>\d+).zip' % self.flow_type)

    def hourly(self):

        '''Get responses'''

        if not self.incoming_sftp:

            return

        try:

            with self.incoming_sftp.client() as client:

                for name in client.listdir():

                    m = self.response_re.match(name)

                    if not m:

                        self.logger.warning(

                            'pull responses: unexpected file "%s" in sftp, file name does not match pattern %s',

                            name, self.response_re)

                        continue

                    reference = m.groupdict()['reference']

                    step = int(m.groupdict()['step'])

                    demand = self.demand_set.filter(reference=reference).first()

                    if not demand:

                        self.logger.error(

                            'pull responses: unexpected file "%s" in sftp, no demand for reference "%s"',

                            name,

                            reference)

                        continue

                    if step < demand.step:

                        demand.logger.error(

                            'pull responses: unexpected file "%s" in sftp: step %s is inferior to demand step %s',

                            name,

                            step,

                            demand.step)

                        continue

                    demand.handle_response(sftp_client=client, filename=name, step=step)

        except sftp.paramiko.SSHException as e:

            self.logger.error('pull responses: sftp error %s', e)

            return

@six.python_2_unicode_compatible

class Demand(models.Model):

    STATUS_PENDING = 'pending'

    STATUS_PUSHED = 'pushed'

    STATUS_ERROR = 'error'

    STATUSES = [

        (STATUS_PENDING, _('pending')),

        (STATUS_PUSHED, _('pushed')),

        (STATUS_ERROR, _('error')),

    ]

    for mdel_status in MDEL_STATUSES:

        STATUSES.append((mdel_status.slug, mdel_status.label))

    created_at = models.DateTimeField(auto_now_add=True)

    updated_at = models.DateTimeField(auto_now=True)

    reference = models.CharField(max_length=32, null=False, unique=True)

    status = models.CharField(max_length=32, null=True, choices=STATUSES, default=STATUS_PENDING)

    step = models.IntegerField(default=0)

    data = jsonfield.JSONField()

    @functional.cached_property

    def logger(self):

        return self.resource.logger.context(

            demand_id=self.id,

            demand_status=self.status,

            demand_reference=self.reference)

    def push(self):

        if not self.resource.outcoming_sftp:

            return False

        try:

            with self.resource.outcoming_sftp.client() as client:

                with client.open(self.zip_name, mode='w') as fd:

                    fd.write(self.zip_content)

        except sftp.paramiko.SSHException as e:

            self.logger.error('push demand: %s failed, "%s"',

                              self,

                              exception_to_text(e))

            self.status = self.STATUS_ERROR

        except Exception as e:

            self.logger.exception('push demand: %s failed, "%s"',

                                  self,

                                  exception_to_text(e))

            self.status = self.STATUS_ERROR

        else:

            self.resource.logger.info('push demand: %s success', self)

            self.status = self.STATUS_PUSHED

        self.save()

        return True

    @functional.cached_property

    def zip_template(self):

        return ZipTemplate(self.resource.zip_manifest, ctx={

            'reference': self.reference,

            'flow_type': self.resource.flow_type,

            'doc_type': self.resource.doc_type,

            'step': '1',  # We never create more than one document for a reference

            'siret': self.resource.recipient_siret,

            'service': self.resource.recipient_service,

            'guichet': self.resource.recipient_guichet,

            'code_insee': self.data.get('code_insee', self.resource.code_insee),

            'document': self.document,

            'code_insee_id': self.resource.code_insee_id,

            'date': self.created_at.isoformat(),

            'email': self.data.get('email', ''),

        })

    @property

    def zip_name(self):

        return self.zip_template.name

    @property

    def zip_content(self):

        return self.zip_template.render_to_bytes()

    @property

    def document(self):

        xml_schema = self.resource.get_doc_xml_schema()

        return ET.tostring(

            xml_schema.elements[self.resource.xsd_root_element].encode(

                self.data[self.resource.xsd_root_element]))

    @property

    def status_url(self):

        return reverse(

            'generic-endpoint',

            kwargs={

                'connector': self.resource.get_connector_slug(),

                'slug': self.resource.slug,

                'endpoint': 'demand',

                'rest': '%s/' % self.id,

            })

    @property

    def zip_url(self):

        return reverse(

            'generic-endpoint',

            kwargs={

                'connector': self.resource.get_connector_slug(),

                'slug': self.resource.slug,

                'endpoint': 'document',

                'rest': '%s/%s' % (self.id, self.zip_name)

            })

    def handle_response(self, sftp_client, filename, step):

        try:

            with sftp_client.open(filename) as fd:

                with zipfile.ZipFile(fd) as zip_file:

                    with zip_file.open('message.xml') as fd:

                        tree = ET.parse(fd)

                ns = 'http://finances.gouv.fr/dgme/pec/message/v1'

                etat_node = tree.find('.//{%s}Etat' % ns)

                if etat_node is None:

                    self.logger.error(

                        'pull responses: missing Etat node in "%s"',

                        filename)

                    return

                etat = etat_node.text

                if etat in MDEL_STATUSES_BY_CODE:

                    self.status = MDEL_STATUSES_BY_CODE[etat].slug

                else:

                    self.logger.error(

                        'pull responses: unknown etat in "%s", etat="%s"',

                        filename,

                        etat)

                    return

                commentaire_node = tree.find('.//{%s}Etat' % ns)

                if commentaire_node is not None:

                    commentaire = commentaire_node.text

                    self.data = self.data or {}

                    self.data.setdefault('commentaires', []).append(commentaire)

                    self.data['commentaire'] = commentaire

                self.step = step + 1

                self.save()

                self.logger.info('pull responses: status of demand %s changed to %s',

                                 self, self.status)

        except sftp.paramiko.SSHException as e:

            self.logger.error(

                'pull responses: failed to read response "%s", %s',

                filename,

                exception_to_text(e))

        else:

            try:

                sftp_client.remove(filename)

            except sftp.paramiko.SSHException as e:

                self.logger.error(

                    'pull responses: failed to remove response "%s", %s',

                    filename,

                    exception_to_text(e))

    def __str__(self):

        return '<Demand %s reference:%s flow_type:%s>' % (

            self.id,

            self.reference,

            self.resource.flow_type)

    class Meta:

        abstract = True

# -*- coding: utf-8 -*-

# Generated by Django 1.11.20 on 2019-10-24 08:59

from __future__ import unicode_literals

from django.db import migrations, models

import django.db.models.deletion

import jsonfield.fields

import passerelle.utils.sftp

class Migration(migrations.Migration):

    initial = True

    dependencies = [

        ('base', '0015_auto_20190921_0347'),

    ]

    operations = [

        migrations.CreateModel(

            name='Demand',

            fields=[

                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),

                ('created_at', models.DateTimeField(auto_now_add=True)),

                ('updated_at', models.DateTimeField(auto_now=True)),

                ('reference', models.CharField(max_length=32, unique=True)),

                ('status', models.CharField(choices=[('pending', 'pending'), ('pushed', 'pushed'), ('error', 'error'), ('closed', 'closed'), ('rejected', 'rejected'), ('accepted', 'accepted'), ('information needed', 'information needed'), ('in progress', 'in progress'), ('invalid', 'invalid'), ('imported', 'imported')], default='pending', max_length=32, null=True)),

                ('step', models.IntegerField(default=0)),

                ('data', jsonfield.fields.JSONField(default=dict)),

            ],

            options={

                'verbose_name': 'MDEL compatible DDPACS request',

            },

        ),

        migrations.CreateModel(

            name='Resource',

            fields=[

                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),

                ('title', models.CharField(max_length=50, verbose_name='Title')),

                ('description', models.TextField(verbose_name='Description')),

                ('slug', models.SlugField(unique=True, verbose_name='Identifier')),

                ('outcoming_sftp', passerelle.utils.sftp.SFTPField(blank=True, default=None, verbose_name='Outcoming SFTP')),

                ('incoming_sftp', passerelle.utils.sftp.SFTPField(blank=True, default=None, verbose_name='Incoming SFTP')),

                ('recipient_siret', models.CharField(max_length=128, verbose_name='SIRET')),

                ('recipient_service', models.CharField(max_length=128, verbose_name='Service')),

                ('recipient_guichet', models.CharField(max_length=128, verbose_name='Guichet')),

                ('code_insee', models.CharField(max_length=6, verbose_name='INSEE Code')),

                ('users', models.ManyToManyField(blank=True, related_name='_resource_users_+', related_query_name='+', to='base.ApiUser')),

            ],

            options={

                'verbose_name': 'MDEL compatible DDPACS request builder',

            },

        ),

        migrations.AddField(

            model_name='demand',

            name='resource',

            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='mdel_ddpacs.Resource'),

        ),

    ]

# coding: utf-8

# Passerelle - uniform access to data and services

# Copyright (C) 2019  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from django.db import models

from django.utils.translation import ugettext_lazy as _

from passerelle.utils.api import endpoint

from . import abstract

class Resource(abstract.Resource):

    category = _('Civil Status Connectors')

    xsd_root_element = 'PACS'

    flow_type = 'depotDossierPACS'

    doc_type = 'flux-pacs'

    class Meta:

        verbose_name = _('MDEL compatible DDPACS request builder')

    @endpoint(perm='can_access',

              methods=['post'],

              description=_('Create request'),

              post={

                  'request_body': {

                      'schema': {

                          'application/json': None

                      }

                  }

              })

    def create(self, request, post_data):

        return self._handle_create(request, post_data)

Resource.create.endpoint_info.post['request_body']['schema']['application/json'] = Resource.get_create_schema()

class Demand(abstract.Demand):

    resource = models.ForeignKey(Resource)

    class Meta:

        verbose_name = _('MDEL compatible DDPACS request')

<?xml version="1.0" encoding="UTF-8"?>

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

    <xs:element name="PACS" type="PacsType"/>

    <xs:complexType name="PacsType">

        <xs:sequence>

            <xs:element name="partenaire1" type="PartenaireType" />

            <xs:element name="partenaire2" type="PartenaireType" />

            <xs:element name="convention" type="ConventionType" maxOccurs="1" minOccurs="1" />

            <xs:element name="residenceCommune" type="AdresseType" />

            <xs:element name="attestationHonneur" type="AttestationHonneurType" />

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name = "AttestationHonneurType">

        <xs:sequence>

            <xs:element name="nonParente" type="xs:boolean"/>

            <xs:element name="residenceCommune" type="xs:boolean"/>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="PartenaireType">

        <xs:sequence>

            <xs:element name="civilite" type="CiviliteType"></xs:element>

            <xs:element name="nomNaissance" type="xs:string" />

            <xs:element name="prenoms" type="xs:string" />

            <xs:element name="codeNationalite" type="xs:string" maxOccurs="unbounded"/>

            <xs:element name="jourNaissance" type="xs:integer" maxOccurs="1" minOccurs="0" />

            <xs:element name="moisNaissance" type="xs:integer" maxOccurs="1" minOccurs="0" />

            <xs:element name="anneeNaissance" type="xs:integer" />

            <xs:element name="LieuNaissance" type="LieuNaissanceType" />

            <xs:element name="ofpra" type="xs:boolean" />

            <xs:element name="mesureJuridique" type="xs:boolean" />

            <xs:element name="adressePostale" type="AdresseType" />

            <xs:element name="adresseElectronique" type="xs:string" />

            <xs:element name="telephone" type="xs:string" minOccurs="0"/>

            <xs:element name="filiationParent1" type="FiliationType" minOccurs="0"/>

            <xs:element name="filiationParent2" type="FiliationType" minOccurs="0" />

            <xs:element name="titreIdentiteVerifie" type="xs:boolean"/>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="ConventionType">

        <xs:choice>

            <xs:element name="conventionType" type="ConventionTypeType" />

            <xs:element name="conventionSpecifique" type="xs:boolean" />

        </xs:choice>

    </xs:complexType>

    <xs:complexType name="ConventionTypeType">

        <xs:sequence>

            <xs:element name="aideMaterielMontant" type="xs:double" maxOccurs="1" minOccurs="0"/>

            <xs:element name="regimePacs" type="regimePacsType" />

            <xs:element name="aideMateriel" type="AideMaterielType" />

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="AdresseType">

        <xs:sequence>

            <xs:element name="NumeroLibelleVoie" type="xs:string" minOccurs="0" />

            <xs:element name="Complement1" type="xs:string" minOccurs="0" />

            <xs:element name="Complement2" type="xs:string" minOccurs="0" />

            <xs:element name="LieuDitBpCommuneDeleguee" type="xs:string" minOccurs="0" />

            <xs:element name="CodePostal" type="codePostalType" />

            <xs:element name="Localite" type="localiteType" />

            <xs:element name="Pays" type="xs:string" />

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="LieuNaissanceType">

        <xs:sequence>

            <xs:element name="localite" type="localiteType"/>

            <xs:element name="codePostal" type="xs:string"/>

            <xs:element name="codeInsee" type="xs:string" minOccurs="0"/>

            <xs:element name="departement" type="xs:string" maxOccurs="1" minOccurs="0"/>

            <xs:element name="codePays" type="xs:string"/>

        </xs:sequence>

    </xs:complexType>

    <xs:simpleType name="localiteType">

        <xs:restriction base="xs:string">

            <xs:minLength value="1" />

        </xs:restriction>

    </xs:simpleType>

    <xs:simpleType name="codePostalType">

        <xs:restriction base="xs:string">

            <xs:length value="5" />

        </xs:restriction>

    </xs:simpleType>

    <xs:simpleType name="regimePacsType">

        <xs:restriction base="xs:string">

            <xs:enumeration value="indivision"/>

            <xs:enumeration value="legal"/>

        </xs:restriction>

    </xs:simpleType>

    <xs:complexType name="FiliationType">

        <xs:sequence>

            <xs:choice>

                <xs:element name="filiationInconnu" type="xs:boolean"></xs:element>

                <xs:element name="filiationConnu" type="FiliationConnuType">

                </xs:element>

            </xs:choice>

        </xs:sequence>

    </xs:complexType>

    <xs:simpleType name="CiviliteType">

        <xs:restriction base="xs:string">

            <xs:enumeration value="M"></xs:enumeration>

            <xs:enumeration value="MME"></xs:enumeration>

        </xs:restriction>

    </xs:simpleType>

    <xs:simpleType name="TypeAideMaterielType">

        <xs:restriction base="xs:string">

            <xs:enumeration value="aideFixe"/>

            <xs:enumeration value="aideProportionnel"/>

        </xs:restriction>

    </xs:simpleType>

    <xs:complexType name="AideMaterielType">

        <xs:sequence>

            <xs:element name="typeAideMateriel" type="TypeAideMaterielType"></xs:element>

        </xs:sequence>

    </xs:complexType>

    <xs:complexType name="FiliationConnuType">

        <xs:sequence>

            <xs:element name="sexe" type="SexeType"/>

            <xs:element name="nomNaissance" type="xs:string" maxOccurs="1" minOccurs="0" />

            <xs:element name="prenoms" type="xs:string" maxOccurs="1" minOccurs="0" />

            <xs:element name="dateNaissance" type="xs:string" maxOccurs="1" minOccurs="0" />

            <xs:element name="lieuNaissance" type="LieuNaissanceType" maxOccurs="1" minOccurs="0" />

        </xs:sequence>

    </xs:complexType>

    <xs:simpleType name="SexeType">

        <xs:restriction base="xs:string">

            <xs:enumeration value="M"/>

            <xs:enumeration value="F"/>

        </xs:restriction>

    </xs:simpleType>

</xs:schema>

<?xml version="1.0" encoding="UTF-8" ?>

<PACS xmlns:xs="http://www.w3.org/2001/XMLSchema" >

    <partenaire1>

        <civilite>{{ partenaire1.civilite }}</civilite>

        <nomNaissance>{{ partenaire1.nom_naissance }}</nomNaissance>

        <prenoms>{{ partenaire1.prenoms }}</prenoms>

        {% for code_nationalite in partenaire1.code_nationalite %}

        <codeNationalite>{{ code_nationalite }}</codeNationalite>

        {% endfor %}

        <jourNaissance>{{ partenaire1.jour_naissance }}</jourNaissance>

        <moisNaissance>{{ partenaire1.mois_naissance }}</moisNaissance>

        <anneeNaissance>{{ partenaire1.annee_naissance }}</anneeNaissance>

        <LieuNaissance>

            <localite>{{ partenaire1.localite_naissance }}</localite>

            <codePostal>{{ partenaire1.codepostal_naissance }}</codePostal>

            <codeInsee>{{ partenaire1.codeinsee_naissance }}</codeInsee>

            <departement>{{ partenaire1.departement_naissance }}</departement>

            <codePays>{{ partenaire1.codepays_naissance }}</codePays>

        </LieuNaissance>

        <ofpra>{{ partenaire1.ofpra|yesno:"true,false" }}</ofpra>

        <mesureJuridique>{{ partenaire1.mesure_juridique }}</mesureJuridique>

        <adressePostale>

            <NumeroLibelleVoie>{{ partenaire1.adresse_numero_voie }}</NumeroLibelleVoie>

            <Complement1>{{ partenaire1.adresse_complement1 }}</Complement1>

            <Complement2>{{ partenaire1.adresse_complement2 }}</Complement2>

            <LieuDitBpCommuneDeleguee>{{ partenaire1.adresse_lieuditbpcommunedeleguee }}</LieuDitBpCommuneDeleguee>

            <CodePostal>{{ partenaire1.adresse_codepostal }}</CodePostal>

            <Localite>{{ partenaire1.adresse_localite }}</Localite>

            <Pays>{{ partenaire1.adresse_pays }}</Pays>

        </adressePostale>

        <adresseElectronique>{{ partenaire1.email }}</adresseElectronique>

        <telephone>{{ partenaire1.telephone }}</telephone>

        <titreIdentiteVerifie>{{ partenaire1.yesno:"true,false" }}</titreIdentiteVerifie>

    </partenaire1>

    <partenaire2>

        <civilite>{{ partenaire2.civilite }}</civilite>

        <nomNaissance>{{ partenaire2.nom_naissance }}</nomNaissance>

        <prenoms>{{ partenaire2.prenoms }}</prenoms>

        {% for code_nationalite in partenaire2.code_nationalite %}

        <codeNationalite>{{ code_nationalite }}</codeNationalite>

        {% endfor %}

        <jourNaissance>{{ partenaire2.jour_naissance }}</jourNaissance>

        <moisNaissance>{{ partenaire2.mois_naissance }}</moisNaissance>

        <anneeNaissance>{{ partenaire2.annee_naissance }}</anneeNaissance>

        <LieuNaissance>

            <localite>{{ partenaire2.localite_naissance }}</localite>

            <codePostal>{{ partenaire2.codepostal_naissance }}</codePostal>

            <codeInsee>{{ partenaire2.codeinsee_naissance }}</codeInsee>

            <departement>{{ partenaire2.departement_naissance }}</departement>

            <codePays>{{ partenaire2.codepays_naissance }}</codePays>

        </LieuNaissance>

        <ofpra>{{ partenaire2.ofpra|yesno:"true,false" }}</ofpra>

        <mesureJuridique>{{ partenaire2.mesure_juridique }}</mesureJuridique>

        <adressePostale>

            <NumeroLibelleVoie>{{ partenaire2.adresse_numero_voie }}</NumeroLibelleVoie>

            <Complement1>{{ partenaire2.adresse_complement1 }}</Complement1>

            <Complement2>{{ partenaire2.adresse_complement2 }}</Complement2>

            <LieuDitBpCommuneDeleguee>{{ partenaire2.adresse_lieuditbpcommunedeleguee }}</LieuDitBpCommuneDeleguee>

            <CodePostal>{{ partenaire2.adresse_codepostal }}</CodePostal>

            <Localite>{{ partenaire2.adresse_localite }}</Localite>

            <Pays>{{ partenaire2.adresse_pays }}</Pays>

        </adressePostale>

        <adresseElectronique>{{ partenaire2.email }}</adresseElectronique>

        <telephone>{{ partenaire2.telephone }}</telephone>

        <titreIdentiteVerifie>{{ partenaire2.yesno:"true,false" }}</titreIdentiteVerifie>

    </partenaire2>

    <convention>

        <conventionType>

            <aideMaterielMontant>100000</aideMaterielMontant>

            <regimePacs>legal</regimePacs>

            <aideMateriel>

                <typeAideMateriel>aideFixe</typeAideMateriel>

            </aideMateriel>

        </conventionType>

    </convention>

    <residenceCommune>

        <NumeroLibelleVoie>3 place du test</NumeroLibelleVoie>

        <CodePostal>05100</CodePostal>

        <Localite>VILLAR ST PANCRACE</Localite>

        <Pays></Pays>

    </residenceCommune>

    <attestationHonneur>

        <nonParente>true</nonParente>

        <residenceCommune>true</residenceCommune>

    </attestationHonneur>

</PACS>

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

<EnteteMetierEnveloppe xmlns="http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier">

    <NumeroDemarche>{{ flow_type }}</NumeroDemarche>

    <Teledemarche>

        <NumeroTeledemarche>{{ reference }}</NumeroTeledemarche>

        <!-- <MotDePasse></MotDePasse> -->

        <!-- <Suivi></Suivi> -->

        <Date>{{ date }}</Date>

        <IdentifiantPlateforme>Publik</IdentifiantPlateforme>

        <Email>{{ email }}</Email>

    </Teledemarche>

    <Routage>

        <Donnee>

            <Id>{{ code_insee_id }}</Id>

            <Valeur>{{ code_insee }}</Valeur>

        </Donnee>

    </Routage>

    <Document>

        <Code>{{ doc_type }}</Code>

        <Nom>{{ doc_type }}</Nom>

        <FichierFormulaire>

            <FichierDonnees>{{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml</FichierDonnees>

        </FichierFormulaire>

    </Document>

</EnteteMetierEnveloppe>

{

  "name_template": "{{ reference }}-{{ flow_type }}-{{ step }}.zip",

  "part_templates": [

    {

      "name_template": "message.xml",

      "template_path": "message.xml"

    },

    {

      "name_template": "{{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml",

      "content_expression": "document"

    },

    {

      "name_template": "{{ reference }}-{{ flow_type }}-ent-1.xml",

      "template_path": "entete.xml"

    }

  ]

}

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

<ns2:Message xmlns:ns2="http://finances.gouv.fr/dgme/pec/message/v1" xmlns="http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier">

    <ns2:Header>

        <ns2:Routing>

          <ns2:MessageId>{{ reference }} {{ step }}</ns2:MessageId>

            <ns2:FlowType>{{ flow_type }}</ns2:FlowType>

            <ns2:Sender>

                <ns2:Country>FR</ns2:Country>

                <ns2:Location xsi:type="ns2:Location_FR" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

                    <ns2:Siret>13000210800012</ns2:Siret>

                    <ns2:Service>flux_GS_PEC_AVL</ns2:Service>

                    <ns2:Guichet></ns2:Guichet>

                </ns2:Location>

            </ns2:Sender>

            <ns2:Recipients>

                <ns2:Recipient>

                    <ns2:Country>FR</ns2:Country>

                    <ns2:Location xsi:type="ns2:Location_FR" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

                        <ns2:Siret>{{ siret }}</ns2:Siret>

                        <ns2:Service>{{ service }}</ns2:Service>

                        <ns2:Guichet>{{ guichet }}</ns2:Guichet>

                    </ns2:Location>

                </ns2:Recipient>

            </ns2:Recipients>

            <ns2:AckRequired>true</ns2:AckRequired>

            <ns2:AckType>AVL</ns2:AckType>

            <ns2:AckType>ANL</ns2:AckType>

            <ns2:AckTo>

                <ns2:Country>FR</ns2:Country>

                <ns2:Location xsi:type="ns2:Location_FR" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">

                    <ns2:Siret>13000210800012</ns2:Siret>

                    <ns2:Service>flux_GS_PEC_AVL</ns2:Service>

                    <ns2:Guichet></ns2:Guichet>

                </ns2:Location>

            </ns2:AckTo>

        </ns2:Routing>

    </ns2:Header>

    <ns2:Body>

        <ns2:Content>

            <ns2:Aller>

                <NumeroDemarche>{{ flow_type }}</NumeroDemarche>

                <Teledemarche>

                    <NumeroTeledemarche>{{ reference }}</NumeroTeledemarche>

                    <!-- <MotDePasse></MotDePasse> -->

                    <!-- <Suivi></Suivi> -->

                    <Date>{{ date }}</Date>

                    <IdentifiantPlateforme>Publik</IdentifiantPlateforme>

                    <Email>{{ email }}</Email>

                </Teledemarche>

                <Routage>

                    <Donnee>

                        <Id>{{ code_insee_id }}</Id>

                        <Valeur>{{ code_insee }}</Valeur>

                    </Donnee>

                </Routage>

                <Document>

                    <Code>{{ doc_type }}</Code>

                    <Nom>{{ doc_type }}</Nom>

                    <FichierFormulaire>

                        <FichierDonnees>{{ reference }}-{{ flow_type }}-doc-{{ doc_type }}-1-1.xml</FichierDonnees>

                    </FichierFormulaire>

                </Document>

            </ns2:Aller>

        </ns2:Content>

    </ns2:Body>

</ns2:Message>

# Passerelle - uniform access to data and services

# Copyright (C) 2016  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os

import zipfile

from xml.etree import ElementTree as etree

from django.utils.dateparse import parse_date as django_parse_date

from passerelle.utils.jsonresponse import APIError

def parse_date(date):

    try:

        parsed_date = django_parse_date(date)

    except ValueError as e:

        raise APIError('Invalid date: %r (%r)' % ( date, e))

    if not parsed_date:

        raise APIError('date %r not iso-formated' % date)

    return parsed_date.isoformat()

class ElementFactory(etree.Element):

    def __init__(self, *args, **kwargs):

        self.text = kwargs.pop('text', None)

        namespace = kwargs.pop('namespace', None)

        if namespace:

            super(ElementFactory, self).__init__(

                etree.QName(namespace, args[0]), **kwargs

            )

            self.namespace = namespace

        else:

            super(ElementFactory, self).__init__(*args, **kwargs)

    def append(self, element, allow_new=True):

        if not allow_new:

            if isinstance(element.tag, etree.QName):

                found = self.find(element.tag.text)

            else:

                found = self.find(element.tag)

            if found is not None:

                return self

        super(ElementFactory, self).append(element)

        return self

    def extend(self, elements):

        super(ElementFactory, self).extend(elements)

        return self

def zipdir(path):

    """Zip directory

    """

    archname = path + '.zip'

    with zipfile.ZipFile(archname, 'w', zipfile.ZIP_DEFLATED) as zipf:

        for root, dirs, files in os.walk(path):

            for f in files:

                fpath = os.path.join(root, f)

                zipf.write(fpath, os.path.basename(fpath))

    return archname

def get_file_content_from_zip(path, filename):

    """Rreturn file content

    """

    with zipfile.ZipFile(path, 'r') as zipf:

        return zipf.read(filename)

    'passerelle.apps.mdel_ddpacs',

import difflib

def diff_zip(one, two):

    differences = []

    def compute_diff(one, two, fd_one, fd_two):

        content_one = fd_one.read()

        content_two = fd_two.read()

        if content_one == content_two:

            return

        if one.endswith(('.xml', '.json', '.txt')):

            diff = list(difflib.ndiff(content_one.splitlines(),

                                      content_two.splitlines()))

            return ['File %s differs' % one] + diff

        return 'File %s differs' % one

    if not hasattr(one, 'read'):

        one = open(one)

    with one:

        if not hasattr(two, 'read'):

            two = open(two)

        with two:

            with zipfile.ZipFile(one) as one_zip:

                with zipfile.ZipFile(two) as two_zip:

                    one_nl = set(one_zip.namelist())

                    two_nl = set(two_zip.namelist())

                    for name in one_nl - two_nl:

                        differences.append('File %s only in %s' % (name, one))

                    for name in two_nl - one_nl:

                        differences.append('File %s only in %s' % (name, two))

                    for name in one_nl & two_nl:

                        with one_zip.open(name) as fd_one:

                            with two_zip.open(name) as fd_two:

                                difference = compute_diff(name, name, fd_one, fd_two)

                                if difference:

                                    differences.append(difference)

    return differences

{

    "name_template": "{{ reference }}-{{ flow_type }}-{{ step }}.zip",

    "part_templates": [

        {

            "name_template": "message.xml",

            "template_path": "response_message.xml"

        }

    ]

}

<ns2:Message xmlns:ns2="http://finances.gouv.fr/dgme/pec/message/v1" xmlns="http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier">

    <ns2:Header>

        <ns2:Routing>

            <ns2:MessageId>{{ reference }} {{ step }}</ns2:MessageId>

            <ns2:RefToMessageId>{{ reference }} {{ old_step }}</ns2:RefToMessageId>

            <ns2:FlowType>{{ flow_type }}</ns2:FlowType>

            <ns2:Sender/>

            <ns2:Recipients>

                <ns2:Recipient/>

            </ns2:Recipients>

        </ns2:Routing>

        <ns2:Security>

            <ns2:Horodatage>false</ns2:Horodatage>

        </ns2:Security>

    </ns2:Header>

    <ns2:Body>

        <ns2:Content>

            <ns2:Retour>

                <ns2:Enveloppe>

                    <ns2:NumeroTeledemarche>{{ reference }}</ns2:NumeroTeledemarche>

                </ns2:Enveloppe>

                <ns2:Instruction>

                    <ns2:Maj>

                        {% if etat %}<ns2:Etat>{{ etat }}</ns2:Etat>{% endif %}

                        {% if commentaire %}<ns2:Commentaire>{{ commentaire }}</ns2:Commentaire>{% endif %}

                    </ns2:Maj>

                </ns2:Instruction>

            </ns2:Retour>

        </ns2:Content>

    </ns2:Body>

</ns2:Message>

# coding: utf-8

# Passerelle - uniform access to data and services

# Copyright (C) 2019  Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a.deepcopy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import io

import logging

import os

import pytest

import utils

from passerelle.apps.mdel_ddpacs.models import Resource, Demand

from passerelle.utils import json, sftp

from passerelle.utils.zip import diff_zip, ZipTemplate

def build_response_zip(**kwargs):

    zip_template = ZipTemplate(os.path.abspath('tests/data/mdel_ddpacs/response_manifest.json'), ctx=kwargs)

    return zip_template.name, zip_template.render_to_bytes()

@pytest.fixture(autouse=True)

def resource(db):

    return utils.setup_access_rights(Resource.objects.create(

        slug='test',

        code_insee='66666',

        recipient_siret='999999',

        recipient_service='SERVICE',

        recipient_guichet='GUICHET'))

@pytest.fixture

def ddpacs_payload():

    xmlschema = Resource.get_doc_xml_schema()

    return json.flatten({'PACS': xmlschema.to_dict('tests/data/pacs-doc.xml')})

def test_create_demand(app, resource, ddpacs_payload, freezer, sftpserver, caplog):

    # paramiko log socket errors when connection is closed :/

    caplog.set_level(logging.CRITICAL, 'paramiko.transport')

    freezer.move_to('2019-01-01')

    # Push new demand

    payload = {

        'display_id': '1-1',

    }

    payload.update(ddpacs_payload)

    assert Demand.objects.count() == 0

    assert resource.jobs_set().count() == 0

    resp = app.post_json('/mdel-ddpacs/test/create?raise=1', params=payload)

    assert resp.json['err'] == 0

    assert resp.json['status'] == 'pending'

    assert Demand.objects.count() == 1

    assert resource.jobs_set().count() == 1

    url = resp.json['url']

    zip_url = resp.json['zip_url']

    # Check demand status URL

    status = app.get(url)

    assert status.json['err'] == 0

    assert status.json == resp.json

    # Check demand document URL

    zip_document = app.get(zip_url)

    with io.BytesIO(zip_document.body) as fd:

        differences = diff_zip('tests/data/mdel_ddpacs_expected.zip', fd)

        assert not differences, differences

    # Check job is skipped as no SFTP is configured

    assert resource.jobs_set().get().after_timestamp is None

    resource.jobs()

    assert resource.jobs_set().get().after_timestamp is not None

    assert resource.jobs_set().exclude(status='completed').count() == 1

    with sftpserver.serve_content({'input': {}, 'output': {}}):

        content = sftpserver.content_provider.content_object

        resource.outcoming_sftp = sftp.SFTP(

            'sftp://john:doe@{server.host}:{server.port}/output/'.format(

                server=sftpserver))

        resource.jobs()

        assert not content['output']

        # Jump over the 6 hour wait time for retry

        freezer.move_to('2019-01-02')

        resource.jobs()

        assert 'A-1-1-depotDossierPACS-1.zip' in content['output']

        # Check it's the same document than through the zip_url

        with open('/tmp/zip.zip', 'w') as fd:

            fd.write(content['output']['A-1-1-depotDossierPACS-1.zip'])

        with io.BytesIO(content['output']['A-1-1-depotDossierPACS-1.zip']) as fd:

            differences = diff_zip('tests/data/mdel_ddpacs_expected.zip', fd)

            assert not differences, differences

        # Act as if zip was consumed

        content['output'] = {}

        # Jump over the 6 hour wait time for retry

        freezer.move_to('2019-01-03')

        resource.jobs()

        assert not content['output']

        assert resource.jobs_set().exclude(status='completed').count() == 0

        # Check response

        resource.hourly()

        resource.incoming_sftp = sftp.SFTP(

            'sftp://john:doe@{server.host}:{server.port}/input/'.format(

                server=sftpserver))

        response_name, response_content = build_response_zip(

            reference='A-1-1',

            flow_type='depotDossierPACS',

            step=1,

            old_step=1,

            etat=100,

            commentaire='coucou')

        content['input'][response_name] = response_content

        resource.hourly()

        assert resource.demand_set.get().status == 'closed'

        assert response_name not in content['input']

        response_name, response_content = build_response_zip(

            reference='A-1-1',

            flow_type='depotDossierPACS',

            step=1,

            old_step=1,

            etat=1,

            commentaire='coucou')

        content['input'][response_name] = response_content

        resource.hourly()

        assert 'unexpected file "A-1-1-depotDossierPACS-1.zip"' in caplog.messages[-1]

        assert 'step 1 is inferior' in caplog.messages[-1]

        resource.check_status()