Produits Entr'ouvert » Passerelle

<?xml version="1.0" encoding="UTF-8"?>

<!-- edited with XMLSpy v2010 (http://www.altova.com) by BULL SAS (BULL SAS) -->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">

  <xs:complexType name="DECLARANT">

    <xs:sequence>

      <xs:element name="identité" type="identité" minOccurs="0"/>

      <xs:element name="designation-permis" type="designation-permis" minOccurs="0"/>

      <xs:element name="coordonnees" type="coordonnees" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="identité">

    <xs:sequence>

      <xs:element name="type-personne" type="xs:boolean" minOccurs="0"/>

      <xs:element name="personne-physique" type="personne-physique" minOccurs="0"/>

      <xs:element name="personne-morale" type="personne-morale" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="personne-physique">

    <xs:sequence>

      <xs:element name="civilité" type="xs:string" minOccurs="0"/>

      <xs:element name="nom" type="xs:string" minOccurs="0"/>

      <xs:element name="prenom" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="personne-morale">

    <xs:sequence>

      <xs:element name="denomination"/>

      <xs:element name="raison-sociale"/>

      <xs:element name="SIRET"/>

      <xs:element name="categorie-juridique"/>

      <xs:element name="representant-personne-morale" type="representant-personne-morale"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="representant-personne-morale">

    <xs:sequence>

      <xs:element name="civilité" type="xs:string" minOccurs="0"/>

      <xs:element name="nom" type="xs:string" minOccurs="0"/>

      <xs:element name="prenom" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:element name="DOC">

    <xs:complexType>

      <xs:sequence>

        <xs:element name="DECLARANT" type="DECLARANT" minOccurs="0"/>

        <xs:element name="OUVERTURE-CHANTIER" type="OUVERTURE-CHANTIER" minOccurs="0"/>

        <xs:element name="acceptation" type="xs:boolean" minOccurs="0"/>

      </xs:sequence>

    </xs:complexType>

  </xs:element>

  <xs:complexType name="designation-permis">

    <xs:sequence>

      <xs:element name="numero-permis_construire" type="xs:string" minOccurs="0"/>

      <xs:element name="numero-permis_amenager" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="coordonnees">

    <xs:sequence>

      <xs:element name="adresse" type="adresse" minOccurs="0"/>

      <xs:element name="courriel" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="adresse">

    <xs:sequence>

      <xs:element name="numero-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="extension" type="xs:string" minOccurs="0"/>

      <xs:element name="type-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="nom-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="lieu-dit" type="xs:string" minOccurs="0"/>

      <xs:element name="boite-postale" type="xs:string" minOccurs="0"/>

      <xs:element name="code-postal" type="xs:string" minOccurs="0"/>

      <xs:element name="localite" type="xs:string" minOccurs="0"/>

      <xs:element name="bureau-cedex" type="xs:string" minOccurs="0"/>

      <xs:element name="pays" type="xs:string" minOccurs="0"/>

      <xs:element name="division-territoriale" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="OUVERTURE-CHANTIER">

    <xs:sequence>

      <xs:element name="date-ouverture" type="xs:date" minOccurs="0"/>

      <xs:element name="totalite-travaux" type="xs:boolean" minOccurs="0"/>

      <xs:element name="tranche-travaux" type="tranche-travaux" minOccurs="0"/>

      <xs:element name="autorisation-differer-travaux" type="xs:string" minOccurs="0"/>

      <xs:element name="SHON" type="xs:string" minOccurs="0"/>

      <xs:element name="nombre-logements-commences" type="nombre-logements-commences" minOccurs="0"/>

      <xs:element name="repartition-type-financement" type="repartition-type-financement" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="tranche-travaux">

    <xs:sequence>

      <xs:element name="amenagements-commences" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="nombre-logements-commences">

    <xs:sequence>

      <xs:element name="total-logements" type="xs:int" minOccurs="0"/>

      <xs:element name="individuels" type="xs:int" minOccurs="0"/>

      <xs:element name="collectifs" type="xs:int" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="repartition-type-financement">

    <xs:sequence>

      <xs:element name="logement-locatif-social" type="xs:int" minOccurs="0"/>

      <xs:element name="accession-aidee" type="xs:int" minOccurs="0"/>

      <xs:element name="pret-taux-zero" type="xs:int" minOccurs="0"/>

      <xs:element name="autres-financements" type="xs:int" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

</xs:schema>

<?xml version="1.0" encoding="UTF-8"?>

<!-- edited with XMLSpy v2010 rel. 2 (http://www.altova.com) by BULL SAS (BULL SAS) -->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

	<xs:element name="RecensementCitoyen">

		<xs:complexType>

			<xs:sequence>

				<xs:element name="Convention" type="xs:string"/>

				<xs:element name="Formalite">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="Identifiant" type="xs:string"/>

							<xs:element name="FormaliteType" type="xs:string"/>

							<xs:element name="DateSoumission" type="xs:string"/>

							<xs:element name="FormaliteMotifCode" type="xs:string" maxOccurs="2"/>

							<xs:element name="FormaliteModeCode" type="xs:string"/>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="Personne">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="Civilite" type="xs:string"/>

							<xs:element name="Sexe" type="xs:string"/>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="NomUsage" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="AdresseDomicile">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="PointDeRemise" type="xs:string"/>

										<xs:element name="Complement" type="xs:string"/>

										<xs:element name="NumeroVoie" type="xs:string"/>

										<xs:element name="Extension" type="xs:string"/>

										<xs:element name="TypeVoie" type="xs:string"/>

										<xs:element name="NomVoie" type="xs:string"/>

										<xs:element name="LieuDit" type="xs:string"/>

										<xs:element name="CodePostal" type="xs:string"/>

										<xs:element name="Localite" type="xs:string"/>

										<xs:element name="CodeINSEE" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="AdresseResidence">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="PointDeRemise" type="xs:string"/>

										<xs:element name="Complement" type="xs:string"/>

										<xs:element name="NumeroVoie" type="xs:string"/>

										<xs:element name="Extension" type="xs:string"/>

										<xs:element name="TypeVoie" type="xs:string"/>

										<xs:element name="NomVoie" type="xs:string"/>

										<xs:element name="LieuDit" type="xs:string"/>

										<xs:element name="CodePostal" type="xs:string"/>

										<xs:element name="Localite" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="SituationFamille">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="SituationMatrimoniale" type="xs:string"/>

										<xs:element name="NombreEnfants" type="xs:string"/>

										<xs:element name="Pupille" type="xs:string"/>

										<xs:element name="NombreFrereSoeur" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="MethodeContact" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="URI" type="xs:string"/>

										<xs:element name="CanalCode" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="FiliationPere">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="FiliationMere">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

			</xs:sequence>

		</xs:complexType>

	</xs:element>

</xs:schema>

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django import forms

from . import models, fields

class MappingForm(forms.ModelForm):

    def __init__(self, *args, **kwargs):

        super(MappingForm, self).__init__(*args, **kwargs)

        if self.instance.procedure and self.instance and self.instance.formdef:

            choices = [('', '--------')] + [(v, v) for v in self.instance.variables]

            for i, field in enumerate(self.schema_fields()):

                label = field.label

                label += ' (%s)' % (field.varname or 'NO VARNAME')

                base_name = str(field.varname or i)

                initial = self.instance.rules.get('fields', {}).get(base_name)

                self.fields['field_%s' % base_name] = fields.VariableAndExpressionField(

                    label=label,

                    choices=choices,

                    initial=initial,

                    required=False)

    def table_fields(self):

        return [field for field in self if field.name.startswith('field_')]

    def schema_fields(self):

        if self.instance and self.instance.formdef:

            schema = self.instance.formdef.schema

            for i, field in enumerate(schema.fields):

                if field.type in ('page', 'comment', 'title', 'subtitle'):

                    continue

                yield field

    def save(self, commit=True):

        fields = {}

        for key in self.cleaned_data:

            if not key.startswith('field_'):

                continue

            if not self.cleaned_data[key]:

                continue

            real_key = key[len('field_'):]

            value = self.cleaned_data[key].copy()

            value['label'] = self.fields[key].label

            fields[real_key] = value

        self.instance.rules['fields'] = fields

        return super(MappingForm, self).save(commit=commit)

    class Meta:

        model = models.Mapping

        fields = [

            'procedure',

            'formdef',

        ]

# -*- coding: utf-8 -*-

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re

import os

import stat

import zipfile

import collections

import base64

import datetime

import unicodedata

from lxml import etree as ET

from django.core.urlresolvers import reverse

from django.core.files import File

from django.db import models, transaction

from django.template import engines

from django.utils import six

from django.utils.translation import ugettext_lazy as _

from jsonfield import JSONField

from passerelle.base.models import BaseResource

from passerelle.utils.api import endpoint

from passerelle.utils.sftp import SFTPField

from passerelle.utils.wcs import FormDefField, get_wcs_choices

from passerelle.utils.xsd import Schema

from passerelle.utils.xml import text_content

PROCEDURE_DOC = 'DOC'

PROCEDURE_RCO = 'RCO'

PROCEDURE_DDPACS = 'DDPACS'

PROCEDURES = [

    (PROCEDURE_DOC, _('Request for construction site opening')),

    (PROCEDURE_RCO, _('Request for mandatory citizen census')),

    (PROCEDURE_DDPACS, _('Pre-request for citizen solidarity pact')),

]

FILE_PATTERN = re.compile(r'^(?P<identifier>.*)-(?P<procedure>[A-Z]+)-(?P<sequence>\d+).zip$')

ENT_PATTERN = re.compile(r'^.*-ent-\d+(?:-.*)?.xml$')

DOC_PATTERN = re.compile(r'^.*-doc-\d+-XML-\d+(?:-.*)?\.xml$')

DOC_PDF_PATTERN = re.compile(r'^.*-doc-\d+-PDF-\d+(?:-.*)?\.pdf$')

PJ_PATTERN = re.compile(r'^.*-pj-(?P<type>[^-]+)-\d+\.(?P<extension>[^.]+)$')

NSMAP = {

    'dgme-metier': 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier'

}

ROUTAGE_XPATH = ET.XPath(

    ('dgme-metier:Routage/dgme-metier:Donnee/dgme-metier:Valeur/text()'),

    namespaces=NSMAP)

ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier')

def simplify(s):

    '''Simplify XML node tag names because XSD from DGME are garbage'''

    if not s:

        return ''

    if not isinstance(s, six.text_type):

        s = six.text_type(s, 'utf-8', 'ignore')

    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')

    s = re.sub(r'[^\w\s\'-_]', '', s)

    s = s.replace('-', '_')

    s = re.sub(r'[\s\']+', '', s)

    return s.strip().lower()

class Resource(BaseResource):

    category = _('Business Process Connectors')

    input_sftp = SFTPField(

        verbose_name=_('Input SFTP URL'),

        null=True)

    output_sftp = SFTPField(

        verbose_name=_('Output SFTP URL'),

        null=True)

    def check_status(self):

        with self.input_sftp.client() as sftp:

            sftp.listdir()

        with self.output_sftp.client() as sftp:

            sftp.listdir()

        get_wcs_choices(session=self.requests)

    @endpoint(name='ping', description=_('Check Solis API availability'))

    def ping(self, request):

        self.check_status()

        return {'err': 0}

    def run_loop(self, count=1):

        with transaction.atomic():

            # lock resource

            r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk)

            if not r:

                # already locked

                self.logger.info('did nothing')

                return

            with self.input_sftp.client() as sftp:

                try:

                    sftp.lstat('DONE')

                except IOError:

                    sftp.mkdir('DONE')

                try:

                    sftp.lstat('FAILED')

                except IOError:

                    sftp.mkdir('FAILED')

                def helper():

                    for file_stat in sftp.listdir_attr():

                        if stat.S_ISDIR(file_stat.st_mode):

                            continue

                        yield file_stat.filename

                for filename, i in zip(helper(), range(count)):

                    m = FILE_PATTERN.match(filename)

                    if not m:

                        self.logger.info('file "%s" did not match pattern %s, moving to FAILED/', filename, FILE_PATTERN)

                        sftp.rename(filename, 'FAILED/' + filename)

                        continue

                    procedure = m.group('procedure')

                    try:

                        mapping = self.mappings.get(procedure=procedure)

                    except Mapping.DoesNotExist:

                        self.logger.info('no mapping for procedure "%s" for file "%s", moving to FAILED/',

                                         procedure, filename)

                        continue

                    handler = self.FileHandler(

                        resource=self,

                        sftp=sftp,

                        filename=filename,

                        identifier=m.group('identifier'),

                        procedure=procedure,

                        sequence=m.group('sequence'),

                        mapping=mapping)

                    try:

                        move, error = handler()

                    except Exception:

                        self.logger.exception('handling of file "%s" failed', filename)

                        # sftp.rename(filename, 'FAILED/' + filename)

                    else:

                        if move and error:

                            self.logger.error('handling of file "%s" failed: %s', filename, error)

                            # sftp.rename(filename, 'FAILED/' + filename)

                        else:

                            if error:

                                self.logger.warning('handling of file "%s" failed: %s', filename, error)

                            elif move:

                                sftp.rename(filename, 'DONE/' + filename)

    class FileHandler(object):

        def __init__(self, resource, sftp, filename, identifier, procedure, sequence, mapping):

            self.resource = resource

            self.sftp = sftp

            self.filename = filename

            self.identifier = identifier

            self.procedure = procedure

            self.sequence = sequence

            self.mapping = mapping

            self.variables = list(self.mapping.variables)

            self.request = Request.objects.filter(resource=resource, filename=filename).first()

        def __call__(self):

            if not self.request:

                with self.sftp.open(self.filename) as fd:

                    with transaction.atomic():

                        self.request = Request.objects.create(

                            resource=self.resource,

                            filename=self.filename)

                        self.request.state = Request.STATE_RECEIVED

                        self.request.archive.save(self.filename, File(fd))

            if self.request.state == Request.STATE_RECEIVED:

                with self.request.archive as fd:

                    # error during processing are fatal, we want to log them

                    data = self.process(fd)

                try:

                    backoffice_url = self.transfer(data)

                except Exception as e:

                    return False, 'error during transfer to w.c.s %s' % e

                self.request.url = backoffice_url

                self.request.state = Request.STATE_TRANSFERED

                self.request.save()

            if self.request.state == Request.STATE_TRANSFERED:

                try:

                    self.response()

                except Exception as e:

                    return False, 'error during response to service-public.fr %s' % e

                self.request.state = Request.STATE_RETURNED

                self.request.save()

        def process(self, fd):

            try:

                archive = zipfile.ZipFile(fd)

            except Exception:

                return False, 'could not load zipfile'

            # sort files

            doc_files = []

            ent_files = []

            attachments = {}

            for name in archive.namelist():

                if ENT_PATTERN.match(name):

                    ent_files.append(name)

                elif DOC_PATTERN.match(name):

                    doc_files.append(name)

                elif DOC_PDF_PATTERN.match(name):

                    attachments['doc_pdf'] = [name]

                else:

                    m = PJ_PATTERN.match(name)

                    if m:

                        attachments.setdefault(m.group('type'), []).append(name)

            if len(ent_files) != 1:

                return False, 'too many/few ent files found: %s' % ent_files

            if len(doc_files) != 1:

                return False, 'too many/few doc files found: %s' % doc_files

            for key in attachments:

                if len(attachments[key]) > 1:

                    return False, 'too many attachments of kind %s: %s' % (key, attachments[key])

                name = attachments[key][0]

                with archive.open(attachments[key][0]) as zip_fd:

                    content = zip_fd.read()

                attachments[key] = {

                    'filename': name,

                    'content': base64.b64encode(content).decode('ascii'),

                    'content_type': 'application/octet-stream',

                }

            if self.procedure == 'RCO' and not attachments:

                return False, 'no attachments but RCO requires them'

            ent_file = ent_files[0]

            doc_file = doc_files[0]

            with archive.open(ent_file) as fd:

                document = ET.parse(fd)

            insee_codes = ROUTAGE_XPATH(document)

            if len(insee_codes) != 1:

                return False, 'too many/few insee codes found: %s' % insee_codes

            insee_code = insee_codes[0]

            data = {'insee_code': insee_code}

            data.update(attachments)

            with archive.open(doc_file) as fd:

                document = ET.parse(fd)

                data.update(self.extract_data(document))

                if hasattr(self, 'update_data_%s' % self.procedure):

                    getattr(self, 'update_data_%s' % self.procedure)(data)

            return data

        def transfer(self, data):

            formdef = self.mapping.formdef

            formdef.session = self.resource.requests

            with formdef.submit() as submitter:

                submitter.submission_channel = 'web'

                submitter.submission_context = {

                    'mdel_procedure': self.procedure,

                    'mdel_identifier': self.identifier,

                    'mdel_sequence': self.sequence,

                }

                fields = self.mapping.rules.get('fields', {})

                for name in fields:

                    field = fields[name]

                    variable = field['variable']

                    expression = field['expression']

                    value = data.get(variable)

                    if expression.strip():

                        template = engines['django'].from_string(expression)

                        context = data.copy()

                        context['value'] = value

                        value = template.render(context)

                    submitter.set(name, value)

            return submitter.result.backoffice_url

        def response(self):

            raise NotImplementedError

        def get_data(self, data, name):

            # prevent error in manual mapping

            assert name in self.variables, 'variable "%s" is unknown' % name

            return data.get(name, '')

        def update_data_DOC(self, data):

            def get(name):

                return self.get_data(data, name)

            numero_permis_construire = get('doc_declarant_designation_permis_numero_permis_construire')

            numero_permis_amenager = get('doc_declarant_designation_permis_numero_permis_amenager')

            data['type_permis'] = u'Un permis de construire' if numero_permis_construire else u'Un permis d\'aménager'

            data['numero_permis'] = numero_permis_construire or numero_permis_amenager

            particulier = get('doc_declarant_identite_type_personne').strip().lower() == 'true'

            data['type_declarant'] = u'Un particulier' if particulier else u'Une personne morale'

            if particulier:

                data['nom'] = get('doc_declarant_identite_personne_physique_nom')

                data['prenoms'] = get('doc_declarant_identite_personne_physique_prenom')

            else:

                data['nom'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_nom')

                data['prenoms'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_prenom')

            mapping = {

                '1000': 'Monsieur',

                '1001': 'Madame',

                '1002': 'Madame et Monsieur',

            }

            if particulier:

                data['civilite_particulier'] = mapping.get(get('doc_declarant_identite_personne_physique_civilite'), '')

            else:

                data['civilite_pm'] = mapping.get(

                    get('doc_declarant_identite_personne_morale_representant_personne_morale_civilite'), '')

            data['portee'] = (u'Pour la totalité des travaux'

                              if get('doc_ouverture_chantier_totalite_travaux').lower().strip() == 'true'

                              else u'Pour une tranche des travaux')

        def update_data_RCO(self, data):

            def get(name):

                return self.get_data(data, name)

            motif = (

                get('recensementcitoyen_formalite_formalitemotifcode_1')

                or get('recensementcitoyen_formalite_formalitemotifcode_2')

            )

            data['motif'] = {

                'RECENSEMENT': '1',

                'EXEMPTION': '2'

            }[motif]

            if data['motif'] == '2':

                data['motif_exempte'] = (

                    u"Titulaire d'une carte d'invalidité de 80% minimum"

                    if get('recensementcitoyen_formalite_formalitemotifcode_2') == 'INFIRME'

                    else u"Autre situation")

            data['justificatif_exemption'] = get('je')

            data['double_nationalite'] = (

                'Oui'

                if get('recensementcitoyen_personne_nationalite')

                else 'Non')

            data['residence_differente'] = (

                'Oui'

                if get('recensementcitoyen_personne_adresseresidence_localite')

                else 'Non')

            data['civilite'] = (

                'Monsieur'

                if get('recensementcitoyen_personne_civilite') == 'M'

                else 'Madame'

            )

            def get_lieu_naissance(variable, code):

                for idx in ['', '_1', '_2']:

                    v = variable + idx

                    if get(v + '_code') == code:

                        return get(v + '_nom')

            data['cp_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'AUTRE')

            data['commune_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'COMMUNE')

            data['justificatif_identite'] = get('ji')

            situation_matrimoniale = get('recensementcitoyen_personne_situationfamille_situationmatrimoniale')

            data['situation_familiale'] = {

                u'Célibataire': u'Célibataire',

                u'Marié': u'Marié(e)',

            }.get(situation_matrimoniale, u'Autres')

            if data['situation_familiale'] == u'Autres':

                data['situation_familiale_precision'] = situation_matrimoniale

            pupille = get('recensementcitoyen_personne_situationfamille_pupille')

            data['pupille'] = (

                'Oui'

                if pupille

                else 'Non'

            )

            data['pupille_categorie'] = {

                'NATION': u"Pupille de la nation",

                'ETAT': u"Pupille de l'État",

            }.get(pupille)

            for idx in ['', '_1', '_2']:

                code = get('recensementcitoyen_personne_methodecontact%s_canalcode' % idx)

                uri = get('recensementcitoyen_personne_methodecontact%s_uri' % idx)

                if code == 'EMAIL':

                    data['courriel'] = uri

                if code == 'TEL':

                    data['telephone_fixe'] = uri

            data['justificatif_famille'] = data['jf']

            data['filiation_inconnue_p1'] = not get('recensementcitoyen_filiationpere_nomfamille')

            data['filiation_inconnue_p2'] = not get('recensementcitoyen_filiationmere_nomfamille')

            data['cp_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'AUTRE')

            data['cp_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'AUTRE')

            data['commune_naissance_p1'] = get_lieu_naissance(

                'recensementcitoyen_filiationpere_lieunaissance', 'COMMUNE')

            data['commune_naissance_p2'] = get_lieu_naissance(

                'recensementcitoyen_filiationmere_lieunaissance', 'COMMUNE')

            for key in data:

                if key.endswith('_datenaissance') and data[key]:

                    data[key] = (

                        datetime.datetime.strptime(data[key], '%d/%m/%Y')

                        .date()

                        .strftime('%Y-%m-%d')

                    )

        def extract_data(self, document):

            '''Convert XML into a dictionnary of values'''

            root = document.getroot()

            def tag_name(node):

                return simplify(ET.QName(node.tag).localname)

            def helper(path, node):

                if len(node):

                    tags = collections.Counter(tag_name(child) for child in node)

                    counter = collections.Counter()

                    for child in node:

                        name = tag_name(child)

                        if tags[name] > 1:

                            counter[name] += 1

                            name += '_%s' % counter[name]

                        for p, value in helper(path + [name], child):

                            yield p, value

                else:

                    yield path, text_content(node)

            return {'_'.join(path): value for path, value in helper([tag_name(root)], root)}

    class Meta:

        verbose_name = _('Service-Public.fr')

class SPFRMessage(object):

    @classmethod

    def from_file(self, resource, filename, fd):

        m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename)

        if not m:

            resource.logger.warning('found file with an unknown pattern %s moving in DONE/', filename)

            return None

        mdel_number, procedure, sequence = m.groups()

def default_rule():

    return {}

class Mapping(models.Model):

    resource = models.ForeignKey(

        Resource,

        verbose_name=_('Resource'),

        related_name='mappings')

    procedure = models.CharField(

        verbose_name=_('Procedure'),

        choices=PROCEDURES,

        unique=True,

        max_length=8)

    formdef = FormDefField(

        verbose_name=_('Formulaire'))

    rules = JSONField(

        verbose_name=_('Rules'),

        default=default_rule)

    def get_absolute_url(self):

        return reverse('sp-fr-mapping-edit', kwargs=dict(

            slug=self.resource.slug,

            pk=self.pk))

    @property

    def xsd(self):

        doc = ET.parse(os.path.join(os.path.dirname(__file__), '%s.XSD' % self.procedure))

        schema = Schema()

        schema.visit(doc.getroot())

        return schema

    @property

    def variables(self):

        yield 'insee_code'

        for path, xsd_type in self.xsd.paths():

            names = [simplify(tag.localname) for tag in path]

            yield '_'.join(names)

        if hasattr(self, 'variables_%s' % self.procedure):

            for variable in getattr(self, 'variables_%s' % self.procedure):

                yield variable

    @property

    def variables_DOC(self):

        yield 'type_permis'

        yield 'numero_permis'

        yield 'type_declarant'

        yield 'nom'

        yield 'prenoms'

        yield 'civilite_particulier'

        yield 'civilite_pm'

        yield 'portee'

    @property

    def variables_RCO(self):

        yield 'motif'

        yield 'motif_exemple'

        yield 'justificatif_exemption'

        yield 'double_nationalite'

        yield 'residence_differente'

        yield 'civilite'

        yield 'cp_naissance'

        yield 'commune_naissance'

        yield 'je'

        yield 'ji'

        yield 'situation_familiale'

        yield 'situation_familiale_precision'

        yield 'pupille'

        yield 'pupille_categorie'

        yield 'courriel'

        yield 'telephone_fixe'

        yield 'jf'

        yield 'filiation_inconnue_p1'

        yield 'filiation_inconnue_p2'

        yield 'cp_naissance_p1'

        yield 'cp_naissance_p2'

        yield 'commune_naissance_p1'

        yield 'commune_naissance_p2'

class Request(models.Model):

    # To prevent mixing errors from analysing archive from s-p.fr and errors

    # from pushing to w.c.s we separate processing with three steps:

    # - receiving, i.e. copying zipfile from SFTP and storing them locally

    # - processing, i.e. openeing the zipfile and extracting content as we need it

    # - transferring, pushing content as a new form in w.c.s.

    STATE_RECEIVED = 'received'

    STATE_TRANSFERED = 'transfered'

    STATE_RETURNED = 'returned'

    STATE_ERROR = 'error'

    STATES = [

        (STATE_RECEIVED, _('Received')),

        (STATE_TRANSFERED, _('Transfered')),

        (STATE_ERROR, _('Transfered')),

        (STATE_RETURNED, _('Returned')),

    ]

    resource = models.ForeignKey(

        Resource,

        verbose_name=_('Resource'))

    created = models.DateTimeField(

        verbose_name=_('Created'),

        auto_now_add=True)

    modified = models.DateTimeField(

        verbose_name=_('Created'),

        auto_now=True)

    filename = models.CharField(

        verbose_name=_('Identifier'),

        max_length=128)

    archive = models.FileField(

        verbose_name=_('Archive'),

        max_length=256)

    state = models.CharField(

        verbose_name=_('State'),

        choices=STATES,

        default=STATE_RECEIVED,

        max_length=16)

    url = models.URLField(

        verbose_name=_('URL'),

        blank=True)

    class Meta:

        unique_together = (

            ('resource', 'filename'),

        )

<div class="variable-widget">

    {% include widget.subwidgets.0.template_name with widget=widget.subwidgets.0 %}

</div>

<div class="expression-widget">

    {% include widget.subwidgets.1.template_name with widget=widget.subwidgets.1 %}

</div>

{% extends "passerelle/manage/resource_child_confirm_delete.html" %}

{% extends "passerelle/manage/resource_child_form.html" %}

{% load i18n %}

{% block form %}

    {% if form.errors %}

      <div class="errornotice">

        <p>{% trans "There were errors processing your form." %}</p>

        {% for error in form.non_field_errors %}

        <p>{{ error }}</p>

        {% endfor %}

        {% for field in form %}

          {% if field.is_hidden and field.errors %}

            <p>

              {% for error in field.errors %}

              {% blocktrans with name=field.name %}(Hidden field {{name}}) {{ error }}{% endblocktrans %}

              {% if not forloop.last %}<br>{% endif %}

              {% endfor %}

            </p>

          {% endif %}

        {% endfor %}

      </div>

    {% endif %}

    {% include "gadjo/widget.html" with field=form.procedure %}

    {% include "gadjo/widget.html" with field=form.formdef%}

    <table class="main">

        <thead>

            <tr>

                <td>Label</td>

                <td>Variable</td>

            </tr>

        </thead>

        <tbody>

            {% for field in form.table_fields %}

                <tr>

                    <td>{{ field.label_tag }}</td>

                    <td>{{ field }}</td>

                </tr>

            {% endfor %}

       </tbody>

    </table>

{% endblock %}

{% extends "passerelle/manage/service_view.html" %}

{% load i18n passerelle %}

{% block description %}

    <p>

    {% blocktrans %}

        Connector to forms published by <a href="https://www.service-public.fr/">service-public.fr</a> 

    {% endblocktrans %}

        <a href="{% url "sp-fr-run" slug=object.slug %}">Run</a>

    </p>

    {{ block.super }}

{% endblock %}

{% block extra-sections %}

<div id="mappings" class="section">

    <h3>{% trans "Mappings" %} <a class="button" href="{% url "sp-fr-mapping-new" slug=object.slug %}">{% trans "Add" %}</a></h3>

    <ul>

        {% for mapping in object.mappings.all %}

            <li>

                <fieldset class="gadjo-foldable gadjo-folded" id="sp-fr-mapping-{{ mapping.pk}}">

                    <legend class="gadjo-foldable-widget">

                        <a href="{% url "sp-fr-mapping-edit" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% blocktrans with procedure=mapping.get_procedure_display formdef=mapping.formdef.title %}From procedure {{ procedure }} to form {{ formdef }}{% endblocktrans %}</a>

                    </legend>

                    <div class="gadjo-folding">

                        {% for key, value in mapping.rules.fields.items %}

                        {% if value %}

                            <p>{{ value.label }}&nbsp;: {{ value.variable }} {% if value.expression %}({% trans "with expression" %} <tt>{{ value.expression }}</tt>){% endif %}</p>

                        {% endif %}

                        {% endfor %}

                        <a rel="popup" class="delete" href="{% url "sp-fr-mapping-delete" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% trans "Delete" %}</a>

                    </div>

                </fieldset>

            </li>

        {% endfor %}

    </ul>

</div>

<div id="requests" class="section">

    <h3>{% trans "Mappings" %} <a class="button" href="{% url "sp-fr-mapping-new" slug=object.slug %}">{% trans "Add" %}</a></h3>

    <table class="main">

        <thead>

            <tr>

                <td>Id</td>

                <td>Created</td>

                <td>Modified</td>

                <td>State</td>

                <td>Filename</td>

                <td>Slug</td>

                <td>Form Id</td>

            </tr>

        </thead>

        <tbody>

            {% for req in object.request_set.all %}

            <tr>

                <td>{{ req.id }}</td>

                <td>{{ req.created }}</td>

                <td>{{ req.modified }}</td>

                <td>{{ req.get_state_display }}</td>

                <td>{{ req.filename }}</td>

                <td>{{ req.content.formdef_slug }}</td>

                <td>{{ req.content.formdata_id }}</td>

            </tr>

            {% endfor %}

        </tbody>

    </table>

</div>

{% endblock %}

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.conf.urls import url

from . import views

management_urlpatterns = [

    url(r'^(?P<slug>[\w,-]+)/mapping/new/$',

        views.MappingNew.as_view(), name='sp-fr-mapping-new'),

    url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/$',

        views.MappingEdit.as_view(), name='sp-fr-mapping-edit'),

    url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/delete/$',

        views.MappingDelete.as_view(), name='sp-fr-mapping-delete'),

    url(r'^(?P<slug>[\w,-]+)/run/$',

        views.run, name='sp-fr-run'),

]

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.views.generic import UpdateView, CreateView, DeleteView

from django.shortcuts import get_object_or_404

from django.http import HttpResponseRedirect

from passerelle.base.mixins import ResourceChildViewMixin

from . import models, forms

class StayIfChanged(object):

    has_changed = False

    def form_valid(self, form):

        if set(form.changed_data) & set(['procedure', 'formdef']):

            self.has_changed = True

        return super(StayIfChanged, self).form_valid(form)

    def get_success_url(self):

        if self.has_changed:

            return self.get_changed_url()

        return super(StayIfChanged, self).get_success_url()

    def get_changed_url(self):

        return ''

class MappingNew(StayIfChanged, ResourceChildViewMixin, CreateView):

    model = models.Mapping

    form_class = forms.MappingForm

    def form_valid(self, form):

        form.instance.resource = self.resource

        return super(MappingNew, self).form_valid(form)

    def get_changed_url(self):

        return self.object.get_absolute_url()

class MappingEdit(StayIfChanged, ResourceChildViewMixin, UpdateView):

    model = models.Mapping

    form_class = forms.MappingForm

class MappingDelete(ResourceChildViewMixin, DeleteView):

    model = models.Mapping

def run(request, connector, slug):

    resource = get_object_or_404(models.Resource, slug=slug)

    resource.run_loop(1000)

    return HttpResponseRedirect(resource.get_absolute_url())

    'passerelle.apps.sp_fr',

.expression-widget input {

	width: 100%;

}

.variable-widget select {

	width: 100%;

}

# -*- coding: utf-8 -*-

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import pytest

from passerelle.apps.sp_fr.models import Resource

import utils

DUMMY_CONTENT = {

    'DILA': {

        'a.zip': 'a',

    }

}

@pytest.fixture

def spfr(settings, db, sftpserver):

    settings.KNOWN_SERVICES = {

        'wcs': {

            'eservices': {

                'title': u'Démarches',

                'url': 'https://demarches-hautes-alpes.test.entrouvert.org/',

                'secret': '9a1f62b680c1cabe73cefcbc5ff4cd4f95c24c3be8dd930adb80d8aaa33bfe67',

                'orig': 'passerelle-hautes-alpes.test.entrouvert.org',

            }

        }

    }

    yield utils.make_resource(

        Resource,

        title='Test 1',

        slug='test1',

        description='Connecteur de test',

        input_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver),

        output_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver)

    )

def test_resource(spfr):

    from passerelle.utils.wcs import get_wcs_choices

    assert get_wcs_choices() == []

def test_sftp_access(spfr, sftpserver):

    with sftpserver.serve_content(DUMMY_CONTENT):

        with spfr.input_sftp as input_sftp:

            assert input_sftp.listdir() == ['a.zip']

        with spfr.output_sftp as output_sftp:

            assert output_sftp.listdir() == ['a.zip']