Produits Entr'ouvert » Passerelle

<?xml version="1.0" encoding="UTF-8"?>

<!-- edited with XMLSpy v2010 (http://www.altova.com) by BULL SAS (BULL SAS) -->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">

  <xs:complexType name="DECLARANT">

    <xs:sequence>

      <xs:element name="identité" type="identité" minOccurs="0"/>

      <xs:element name="designation-permis" type="designation-permis" minOccurs="0"/>

      <xs:element name="coordonnees" type="coordonnees" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="identité">

    <xs:sequence>

      <xs:element name="type-personne" type="xs:boolean" minOccurs="0"/>

      <xs:element name="personne-physique" type="personne-physique" minOccurs="0"/>

      <xs:element name="personne-morale" type="personne-morale" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="personne-physique">

    <xs:sequence>

      <xs:element name="civilité" type="xs:string" minOccurs="0"/>

      <xs:element name="nom" type="xs:string" minOccurs="0"/>

      <xs:element name="prenom" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="personne-morale">

    <xs:sequence>

      <xs:element name="denomination"/>

      <xs:element name="raison-sociale"/>

      <xs:element name="SIRET"/>

      <xs:element name="categorie-juridique"/>

      <xs:element name="representant-personne-morale" type="representant-personne-morale"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="representant-personne-morale">

    <xs:sequence>

      <xs:element name="civilité" type="xs:string" minOccurs="0"/>

      <xs:element name="nom" type="xs:string" minOccurs="0"/>

      <xs:element name="prenom" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:element name="DOC">

    <xs:complexType>

      <xs:sequence>

        <xs:element name="DECLARANT" type="DECLARANT" minOccurs="0"/>

        <xs:element name="OUVERTURE-CHANTIER" type="OUVERTURE-CHANTIER" minOccurs="0"/>

        <xs:element name="acceptation" type="xs:boolean" minOccurs="0"/>

      </xs:sequence>

    </xs:complexType>

  </xs:element>

  <xs:complexType name="designation-permis">

    <xs:sequence>

      <xs:element name="numero-permis_construire" type="xs:string" minOccurs="0"/>

      <xs:element name="numero-permis_amenager" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="coordonnees">

    <xs:sequence>

      <xs:element name="adresse" type="adresse" minOccurs="0"/>

      <xs:element name="courriel" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="adresse">

    <xs:sequence>

      <xs:element name="numero-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="extension" type="xs:string" minOccurs="0"/>

      <xs:element name="type-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="nom-voie" type="xs:string" minOccurs="0"/>

      <xs:element name="lieu-dit" type="xs:string" minOccurs="0"/>

      <xs:element name="boite-postale" type="xs:string" minOccurs="0"/>

      <xs:element name="code-postal" type="xs:string" minOccurs="0"/>

      <xs:element name="localite" type="xs:string" minOccurs="0"/>

      <xs:element name="bureau-cedex" type="xs:string" minOccurs="0"/>

      <xs:element name="pays" type="xs:string" minOccurs="0"/>

      <xs:element name="division-territoriale" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="OUVERTURE-CHANTIER">

    <xs:sequence>

      <xs:element name="date-ouverture" type="xs:date" minOccurs="0"/>

      <xs:element name="totalite-travaux" type="xs:boolean" minOccurs="0"/>

      <xs:element name="tranche-travaux" type="tranche-travaux" minOccurs="0"/>

      <xs:element name="autorisation-differer-travaux" type="xs:string" minOccurs="0"/>

      <xs:element name="SHON" type="xs:string" minOccurs="0"/>

      <xs:element name="nombre-logements-commences" type="nombre-logements-commences" minOccurs="0"/>

      <xs:element name="repartition-type-financement" type="repartition-type-financement" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="tranche-travaux">

    <xs:sequence>

      <xs:element name="amenagements-commences" type="xs:string" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="nombre-logements-commences">

    <xs:sequence>

      <xs:element name="total-logements" type="xs:int" minOccurs="0"/>

      <xs:element name="individuels" type="xs:int" minOccurs="0"/>

      <xs:element name="collectifs" type="xs:int" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

  <xs:complexType name="repartition-type-financement">

    <xs:sequence>

      <xs:element name="logement-locatif-social" type="xs:int" minOccurs="0"/>

      <xs:element name="accession-aidee" type="xs:int" minOccurs="0"/>

      <xs:element name="pret-taux-zero" type="xs:int" minOccurs="0"/>

      <xs:element name="autres-financements" type="xs:int" minOccurs="0"/>

    </xs:sequence>

  </xs:complexType>

</xs:schema>

<?xml version="1.0" encoding="UTF-8"?>

<!-- edited with XMLSpy v2010 rel. 2 (http://www.altova.com) by BULL SAS (BULL SAS) -->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

	<xs:element name="RecensementCitoyen">

		<xs:complexType>

			<xs:sequence>

				<xs:element name="Convention" type="xs:string"/>

				<xs:element name="Formalite">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="Identifiant" type="xs:string"/>

							<xs:element name="FormaliteType" type="xs:string"/>

							<xs:element name="DateSoumission" type="xs:string"/>

							<xs:element name="FormaliteMotifCode" type="xs:string" maxOccurs="2"/>

							<xs:element name="FormaliteModeCode" type="xs:string"/>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="Personne">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="Civilite" type="xs:string"/>

							<xs:element name="Sexe" type="xs:string"/>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="NomUsage" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="AdresseDomicile">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="PointDeRemise" type="xs:string"/>

										<xs:element name="Complement" type="xs:string"/>

										<xs:element name="NumeroVoie" type="xs:string"/>

										<xs:element name="Extension" type="xs:string"/>

										<xs:element name="TypeVoie" type="xs:string"/>

										<xs:element name="NomVoie" type="xs:string"/>

										<xs:element name="LieuDit" type="xs:string"/>

										<xs:element name="CodePostal" type="xs:string"/>

										<xs:element name="Localite" type="xs:string"/>

										<xs:element name="CodeINSEE" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="AdresseResidence">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="PointDeRemise" type="xs:string"/>

										<xs:element name="Complement" type="xs:string"/>

										<xs:element name="NumeroVoie" type="xs:string"/>

										<xs:element name="Extension" type="xs:string"/>

										<xs:element name="TypeVoie" type="xs:string"/>

										<xs:element name="NomVoie" type="xs:string"/>

										<xs:element name="LieuDit" type="xs:string"/>

										<xs:element name="CodePostal" type="xs:string"/>

										<xs:element name="Localite" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="SituationFamille">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="SituationMatrimoniale" type="xs:string"/>

										<xs:element name="NombreEnfants" type="xs:string"/>

										<xs:element name="Pupille" type="xs:string"/>

										<xs:element name="NombreFrereSoeur" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

							<xs:element name="MethodeContact" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="URI" type="xs:string"/>

										<xs:element name="CanalCode" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="FiliationPere">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

				<xs:element name="FiliationMere">

					<xs:complexType>

						<xs:sequence>

							<xs:element name="NomFamille" type="xs:string"/>

							<xs:element name="PrenomUsuel" type="xs:string"/>

							<xs:element name="Prenom" type="xs:string" maxOccurs="2"/>

							<xs:element name="DateNaissance" type="xs:string"/>

							<xs:element name="PaysNaissance" type="xs:string"/>

							<xs:element name="Nationalite" type="xs:string"/>

							<xs:element name="CodeINSEENaissance" type="xs:string"/>

							<xs:element name="LieuNaissance" maxOccurs="2">

								<xs:complexType>

									<xs:sequence>

										<xs:element name="Code" type="xs:string"/>

										<xs:element name="Nom" type="xs:string"/>

									</xs:sequence>

								</xs:complexType>

							</xs:element>

						</xs:sequence>

					</xs:complexType>

				</xs:element>

			</xs:sequence>

		</xs:complexType>

	</xs:element>

</xs:schema>

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django import forms

from . import models, fields

class MappingForm(forms.ModelForm):

    def __init__(self, *args, **kwargs):

        super(MappingForm, self).__init__(*args, **kwargs)

        if self.instance.procedure and self.instance and self.instance.formdef:

            choices = [('', '--------')] + [(v, v) for v in self.instance.variables]

            for i, field in enumerate(self.schema_fields()):

                label = field.label

                label += ' (%s)' % (field.varname or 'NO VARNAME')

                base_name = str(field.varname or i)

                initial = self.instance.rules.get('fields', {}).get(base_name)

                self.fields['field_%s' % base_name] = fields.VariableAndExpressionField(

                    label=label,

                    choices=choices,

                    initial=initial,

                    required=False)

    def table_fields(self):

        return [field for field in self if field.name.startswith('field_')]

    def schema_fields(self):

        if self.instance and self.instance.formdef:

            schema = self.instance.formdef.schema

            for i, field in enumerate(schema.fields):

                if field.type in ('page', 'comment', 'title', 'subtitle'):

                    continue

                yield field

    def save(self, commit=True):

        fields = {}

        for key in self.cleaned_data:

            if not key.startswith('field_'):

                continue

            if not self.cleaned_data[key]:

                continue

            real_key = key[len('field_'):]

            value = self.cleaned_data[key].copy()

            value['label'] = self.fields[key].label

            fields[real_key] = value

        self.instance.rules['fields'] = fields

        return super(MappingForm, self).save(commit=commit)

    class Meta:

        model = models.Mapping

        fields = [

            'procedure',

            'formdef',

        ]

# -*- coding: utf-8 -*-

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import re

import os

import stat

import zipfile

import collections

import io

import base64

import datetime

from lxml import etree as ET

from django.core.urlresolvers import reverse

from django.db import models, transaction

from django.template import engines

from django.utils.translation import ugettext_lazy as _

from jsonfield import JSONField

from passerelle.base.models import BaseResource

from passerelle.utils.api import endpoint

from passerelle.utils.sftp import SFTPField

from passerelle.utils.wcs import FormDefField, get_wcs_choices

from passerelle.utils.xsd import Schema

from passerelle.utils.xml import text_content

PROCEDURE_DOC = 'DOC'

PROCEDURE_RCO = 'RCO'

PROCEDURE_DDPACS = 'DDPACS'

PROCEDURES = [

    (PROCEDURE_DOC, _('Request for construction site opening')),

    (PROCEDURE_RCO, _('Request for mandatory citizen census')),

    (PROCEDURE_DDPACS, _('Pre-request for citizen solidarity pact')),

]

ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier')

def simplify(s):

    '''

    Simplify a string, trying to transform it to lower ascii chars (a-z, 0-9)

    and minimize spaces. Used to compare strings on ?q=something requests.

    '''

    if not s:

        return ''

    if not isinstance(s, six.text_type):

        s = six.text_type(s, 'utf-8', 'ignore')

    s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore')

    s = re.sub(r'[^\w\s\'-_]', '', s)

    s = re.sub(r'[\s\']+', ' ', s)

    return s.strip().lower()

class Resource(BaseResource):

    category = _('Business Process Connectors')

    input_sftp = SFTPField(

        verbose_name=_('Input SFTP URL'),

        null=True)

    output_sftp = SFTPField(

        verbose_name=_('Output SFTP URL'),

        null=True)

    def check_status(self):

        with self.input_sftp as sftp:

            sftp.listdir()

        with self.output_sftp as sftp:

            sftp.listdir()

        get_wcs_choices(self.requests)

    @endpoint(name='ping', description=_('Check Solis API availability'))

    def ping(self, request):

        self.check_status()

        return {'err': 0}

    def run_loop(self, count=1):

        with transaction.atomic():

            # lock resource

            r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk)

            if not r:

                # already locked

                self.logger.info('did nothing')

                return

            with self.input_sftp as sftp:

                try:

                    sftp.lstat('DONE')

                except IOError:

                    sftp.mkdir('DONE')

                def helper():

                    for file_stat in sftp.listdir_attr():

                        if stat.S_ISDIR(file_stat.st_mode):

                            continue

                        yield file_stat.filename

                mappings = {mapping.procedure: mapping for mapping in self.mappings.all()}

                for filename, i in zip(helper(), range(count)):

                    self.handle_filename(filename, sftp, mappings)

    def handle_filename(self, filename, sftp, mappings):

        nsmap = {

            'dgme-metier': 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier'

        }

        routage_xpath = ET.XPath(

            ('dgme-metier:Routage/dgme-metier:Donnee/dgme-metier:Valeur/text()'),

            namespaces=nsmap)

        m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename)

        if not m:

            self.logger.warning('found file with an unknown pattern %s moving in DONE/', filename)

            return

        mdel_identifier, procedure, mdel_sequence = m.groups()

        if procedure not in mappings:

            self.logger.warning('found file for an unsupported procedure %s', procedure)

            return

        mapping = mappings[procedure]

        self.logger.info('found %s %s %s, handling', mdel_identifier, procedure, mdel_sequence)

        with sftp.open(filename) as fd:

            try:

                archive = zipfile.ZipFile(fd)

            except Exception:

                self.logger.error('could not load zipfile %s', filename)

                return

            doc_files = []

            ent_files = []

            attachments = {}

            for name in archive.namelist():

                if re.match(r'^.*-ent-\d+(?:-.*)?.xml$', name):

                    ent_files.append(name)

                if re.match(r'^.*-doc-\d+-XML-\d+(?:-.*)?\.xml$', name):

                    doc_files.append(name)

                m = re.match(r'^.*-pj-([^-]+)-\d+\.([^.]+)$', name)

                if m:

                    attachment_type, extension = m.groups()

                    attachments.setdefault(attachment_type, []).append(name)

            if len(ent_files) != 1:

                self.logger.warning('too many/few ent files found: %s', ent_files)

                return

            if len(doc_files) != 1:

                self.logger.warning('too many/few doc files found: %s', doc_files)

                return

            for key in attachments:

                if len(attachments[key]) > 1:

                    self.logger.warning('too many attachments of kind %s: %s', key, attachments[key])

                name = attachments[key][0]

                content = archive.open(attachments[key][0]).read()

                attachments[key] = {

                    'filename': name,

                    'content': base64.b64encode(content).decode('ascii'),

                    'content_type': 'application/octet-stream',

                }

            if procedure == 'RCO' and not attachments:

                self.logger.warning('no attachments but RCO requires them')

                return

            ent_file = ent_files[0]

            doc_file = doc_files[0]

            with archive.open(ent_file) as fd:

                document = ET.parse(fd)

            insee_codes = routage_xpath(document)

            if len(insee_codes) != 1:

                self.logger.warning('too many/few insee codes found: %s', insee_codes)

                return

            insee_code = insee_codes[0]

            data = {'insee_code': insee_code}

            data.update(attachments)

            with archive.open(doc_file) as fd:

                document = ET.parse(fd)

                data.update(self.extract_data(document))

                if hasattr(self, 'update_data_%s' % procedure):

                    getattr(self, 'update_data_%s' % procedure)(mapping, data)

            formdef = mapping.formdef

            formdef.session = self.requests

            with formdef.submit() as submitter:

                submitter.submission_channel = 'web'

                submitter.submission_context = {

                    'mdel_procedure': procedure,

                    'mdel_identifier': mdel_identifier,

                    'mdel_sequence': mdel_sequence,

                }

                for field in mapping.rules.get('fields', {}):

                    variable = mapping.rules['fields'][field]['variable']

                    expression = mapping.rules['fields'][field]['expression']

                    value = data.get(variable)

                    if expression.strip():

                        template = engines['django'].from_string(expression)

                        context = data.copy()

                        context['value'] = value

                        value = template.render(context)

                    submitter.set(field, value)

                import pprint

                print 'Payload'

                pprint.pprint(submitter.payload())

            for key in sorted(data):

                if not isinstance(data[key], dict):

                    print key, data[key]

                else:

                    print key, '<fichier>', data[key]['filename']

    def update_data_DOC(self, mapping, data):

        variables = list(mapping.variables)

        assert all(key in variables for key in data)

        def get(name):

            # prevent error in manual mapping

            if name not in variables:

                print '\n'.join(sorted(variables))

                assert False, name

            return data.get(name, '')

        numero_permis_construire = get('doc_declarant_designation_permis_numero_permis_construire')

        numero_permis_amenager = get('doc_declarant_designation_permis_numero_permis_amenager')

        data['type_permis'] = u'Un permis de construire' if numero_permis_construire else u'Un permis d\'aménager'

        data['numero_permis'] = numero_permis_construire or numero_permis_amenager

        particulier = get('doc_declarant_identite_type_personne').strip().lower() == 'true'

        data['type_declarant'] = u'Un particulier' if particulier else u'Une personne morale'

        if particulier:

            data['nom'] = get('doc_declarant_identite_personne_physique_nom')

            data['prenoms'] = get('doc_declarant_identite_personne_physique_prenom')

        else:

            data['nom'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_nom')

            data['prenoms'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_prenom')

        mapping = {

            '1000': 'Monsieur',

            '1001': 'Madame',

            '1002': 'Madame et Monsieur',

        }

        if particulier:

            data['civilite_particulier'] = mapping.get(get('doc_declarant_identite_personne_physique_civilite'), '')

        else:

            data['civilite_pm'] = mapping.get(

                get('doc_declarant_identite_personne_morale_representant_personne_morale_civilite'), '')

        data['portee'] = (u'Pour la totalité des travaux'

                          if get('doc_ouverture_chantier_totalite_travaux').lower().strip() == 'true'

                          else u'Pour une tranche des travaux')

        assert all(key in variables for key in data)

    def update_data_RCO(self, mapping, data):

        variables = list(mapping.variables)

        assert all(key in variables for key in data)

        def get(name):

            # prevent error in manual mapping

            if name not in variables:

                print '\n'.join(sorted(variables))

                assert False, name

            return data.get(name, '')

        motif = (

            get('recensementcitoyen_formalite_formalitemotifcode_1')

            or get('recensementcitoyen_formalite_formalitemotifcode_2')

        )

        data['motif'] = {

            'RECENSEMENT': '1',

            'EXEMPTION': '2'

        }[motif]

        if data['motif'] == '2':

            data['motif_exempte'] = (

                u"Titulaire d'une carte d'invalidité de 80% minimum"

                if get('recensementcitoyen_formalite_formalitemotifcode_2') == 'INFIRME'

                else u"Autre situation")

        data['justificatif_exemption'] = get('je')

        data['double_nationalite'] = (

            'Oui'

            if get('recensementcitoyen_personne_nationalite')

            else 'Non')

        data['residence_differente'] = (

            'Oui'

            if get('recensementcitoyen_personne_adresseresidence_localite')

            else 'Non')

        data['civilite'] = (

            'Monsieur'

            if get('recensementcitoyen_personne_civilite') == 'M'

            else 'Madame'

        )

        def get_lieu_naissance(variable, code):

            for idx in ['', '_1', '_2']:

                v = variable + idx

                if get(v + '_code') == code:

                    return get(v + '_nom')

        data['cp_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'AUTRE')

        data['commune_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'COMMUNE')

        data['justificatif_identite'] = get('ji')

        situation_matrimoniale = get('recensementcitoyen_personne_situationfamille_situationmatrimoniale')

        data['situation_familiale'] = {

            u'Célibataire': u'Célibataire',

            u'Marié': u'Marié(e)',

        }.get(situation_matrimoniale, u'Autres')

        if data['situation_familiale'] == u'Autres':

            data['situation_familiale_precision'] = situation_matrimoniale

        pupille = get('recensementcitoyen_personne_situationfamille_pupille')

        data['pupille'] = (

            'Oui'

            if pupille

            else 'Non'

        )

        data['pupille_categorie'] = {

            'NATION': u"Pupille de la nation",

            'ETAT': u"Pupille de l'État",

        }.get(pupille)

        for idx in ['', '_1', '_2']:

            code = get('recensementcitoyen_personne_methodecontact%s_canalcode' % idx)

            uri = get('recensementcitoyen_personne_methodecontact%s_uri' % idx)

            if code == 'EMAIL':

                data['courriel'] = uri

            if code == 'TEL':

                data['telephone_fixe'] = uri

        data['justificatif_famille'] = data['jf']

        data['filiation_inconnue_p1'] = not get('recensementcitoyen_filiationpere_nomfamille')

        data['filiation_inconnue_p2'] = not get('recensementcitoyen_filiationmere_nomfamille')

        data['cp_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'AUTRE')

        data['cp_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'AUTRE')

        data['commune_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'COMMUNE')

        data['commune_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'COMMUNE')

        for key in data:

            if key.endswith('_datenaissance') and data[key]:

                data[key] = (

                    datetime.datetime.strptime(data[key], '%d/%m/%Y')

                    .date()

                    .strftime('%Y-%m-%d')

                )

    def extract_data(self, document):

        root = document.getroot()

        def tag_name(node):

            return simplify(ET.QName(node.tag).localname)

        def helper(path, node):

            if len(node):

                tags = collections.Counter(tag_name(child) for child in node)

                counter = collections.Counter()

                for child in node:

                    name = tag_name(child)

                    if tags[name] > 1:

                        counter[name] += 1

                        name += '_%s' % counter[name]

                    for p, value in helper(path + [name], child):

                        yield p, value

            else:

                yield path, text_content(node)

        return {

            '_'.join(path).replace('-', '_').replace(' ', ''): value

            for path, value in helper([tag_name(root)], root)

        }

    class Meta:

        verbose_name = _('Service-Public.fr')

class SPFRMessage(object):

    @classmethod

    def from_file(self, resource, filename, fd):

        m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename)

        if not m:

            resource.logger.warning('found file with an unknown pattern %s moving in DONE/', filename)

            return None

        mdel_number, procedure, sequence = m.groups()

def default_rule():

    return {}

class Mapping(models.Model):

    resource = models.ForeignKey(

        Resource,

        verbose_name=_('Resource'),

        related_name='mappings')

    procedure = models.CharField(

        verbose_name=_('Procedure'),

        choices=PROCEDURES,

        unique=True,

        max_length=8)

    formdef = FormDefField(

        verbose_name=_('Formulaire'))

    rules = JSONField(

        verbose_name=_('Rules'),

        default=default_rule)

    def get_absolute_url(self):

        return reverse('sp-fr-mapping-edit', kwargs=dict(

            slug=self.resource.slug,

            pk=self.pk))

    @property

    def xsd(self):

        doc = ET.parse(os.path.join(os.path.dirname(__file__), '%s.XSD' % self.procedure))

        schema = Schema()

        schema.visit(doc.getroot())

        return schema

    @property

    def variables(self):

        yield 'insee_code'

        for path, xsd_type in self.xsd.paths():

            names = [simplify(tag.localname).replace('-', '_').replace(' ', '') for tag in path]

            yield '_'.join(names)

        if hasattr(self, 'variables_%s' % self.procedure):

            for variable in getattr(self, 'variables_%s' % self.procedure):

                yield variable

    @property

    def variables_DOC(self):

        yield 'type_permis'

        yield 'numero_permis'

        yield 'type_declarant'

        yield 'nom'

        yield 'prenoms'

        yield 'civilite_particulier'

        yield 'civilite_pm'

        yield 'portee'

    @property

    def variables_RCO(self):

        yield 'motif'

        yield 'motif_exemple'

        yield 'justificatif_exemption'

        yield 'double_nationalite'

        yield 'residence_differente'

        yield 'civilite'

        yield 'cp_naissance'

        yield 'commune_naissance'

        yield 'je'

        yield 'ji'

        yield 'situation_familiale'

        yield 'situation_familiale_precision'

        yield 'pupille'

        yield 'pupille_categorie'

        yield 'courriel'

        yield 'telephone_fixe'

        yield 'jf'

        yield 'filiation_inconnue_p1'

        yield 'filiation_inconnue_p2'

        yield 'cp_naissance_p1'

        yield 'cp_naissance_p2'

        yield 'commune_naissance_p1'

        yield 'commune_naissance_p2'

#def archive_upload_to(instance, filename):

#    return 'sp_fr/{instance.procedure}/{filename}'.format(

#        instance=instance,

#        filename=filename)

#

#

#class Request(models.Model):

#

#    # To prevent mixing errors from analysing archive from s-p.fr and errors

#    # from pushing to w.c.s we separate processing with three steps:

#    # - receiving, i.e. copying zipfile from SFTP and storing them locally

#    # - processing, i.e. openeing the zipfile and extracting content as we need it

#    # - transferring, pushing content as a new form in w.c.s.

#    STATE_RECEIVED = 'received'

#    STATE_PROCESSED = 'processed'

#    STATE_TRANSFERED = 'transfered'

#    STATE_ERROR = 'error'

#    STATES = [

#        (STATE_RECEIVED, _('Received')),

#        (STATE_TRANSFERED, _('Transfered')),

#        (STATE_ERROR, _('Transfered')),

#    ]

#

#    ressource = models.ForeignKey(

#        Resource,

#        verbose_name=_('Resource'))

#    identifier = models.CharField(

#        verbose_name=_('Identifier'),

#        max_length=32)

#    procedure = models.CharField(

#        verbose_name=_('Procedure'),

#        choices=PROCEDURES,

#        max_length=8)

#    sequence_number = models.PositiveIntegerField(

#        verbose_name=_('Sequence number'))

#

#    state = models.CharField(

#        verbose_name=_('State'),

#        choices=STATES,

#        max_length=16)

#

#    content = JSONField(

#        verbose_name=_('Content'),

#        null=True)

#

#    archive = models.FileField(

#        verbose_name=_('Archive'),

#        upload_to=archive_upload_to)

<div class="variable-widget">

    {% include widget.subwidgets.0.template_name with widget=widget.subwidgets.0 %}

</div>

<div class="expression-widget">

    {% include widget.subwidgets.1.template_name with widget=widget.subwidgets.1 %}

</div>

{% extends "passerelle/manage/resource_child_confirm_delete.html" %}

{% extends "passerelle/manage/resource_child_form.html" %}

{% load i18n %}

{% block form %}

    {% if form.errors %}

      <div class="errornotice">

        <p>{% trans "There were errors processing your form." %}</p>

        {% for error in form.non_field_errors %}

        <p>{{ error }}</p>

        {% endfor %}

        {% for field in form %}

          {% if field.is_hidden and field.errors %}

            <p>

              {% for error in field.errors %}

              {% blocktrans with name=field.name %}(Hidden field {{name}}) {{ error }}{% endblocktrans %}

              {% if not forloop.last %}<br>{% endif %}

              {% endfor %}

            </p>

          {% endif %}

        {% endfor %}

      </div>

    {% endif %}

    {% include "gadjo/widget.html" with field=form.procedure %}

    {% include "gadjo/widget.html" with field=form.formdef%}

    <table class="main">

        <thead>

            <tr>

                <td>Label</td>

                <td>Variable</td>

            </tr>

        </thead>

        <tbody>

            {% for field in form.table_fields %}

                <tr>

                    <td>{{ field.label_tag }}</td>

                    <td>{{ field }}</td>

                </tr>

            {% endfor %}

       </tbody>

    </table>

{% endblock %}

{% extends "passerelle/manage/service_view.html" %}

{% load i18n passerelle %}

{% block description %}

    <p>

    {% blocktrans %}

        Connector to forms published by <a href="https://www.service-public.fr/">service-public.fr</a> 

    {% endblocktrans %}

        <a href="{% url "sp-fr-run" slug=object.slug %}">Run</a>

    </p>

    {{ block.super }}

{% endblock %}

{% block extra-sections %}

<div id="mappings" class="section">

    <h3>{% trans "Mappings" %} <a class="button" href="{% url "sp-fr-mapping-new" slug=object.slug %}">{% trans "Add" %}</a></h3>

    <ul>

        {% for mapping in object.mappings.all %}

            <li>

                <fieldset class="gadjo-foldable gadjo-folded" id="sp-fr-mapping-{{ mapping.pk}}">

                    <legend class="gadjo-foldable-widget">

                        <a href="{% url "sp-fr-mapping-edit" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% blocktrans with procedure=mapping.get_procedure_display formdef=mapping.formdef.title %}From procedure {{ procedure }} to form {{ formdef }}{% endblocktrans %}</a>

                    </legend>

                    <div class="gadjo-folding">

                        {% for key, value in mapping.rules.fields.items %}

                        {% if value %}

                            <p>{{ value.label }}&nbsp;: {{ value.variable }} {% if value.expression %}({% trans "with expression" %} <tt>{{ value.expression }}</tt>){% endif %}</p>

                        {% endif %}

                        {% endfor %}

                        <a rel="popup" class="delete" href="{% url "sp-fr-mapping-delete" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% trans "Delete" %}</a>

                    </div>

                </fieldset>

            </li>

        {% endfor %}

    </ul>

</div>

{% endblock %}

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.conf.urls import url

from . import views

management_urlpatterns = [

    url(r'^(?P<slug>[\w,-]+)/mapping/new/$',

        views.MappingNew.as_view(), name='sp-fr-mapping-new'),

    url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/$',

        views.MappingEdit.as_view(), name='sp-fr-mapping-edit'),

    url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/delete/$',

        views.MappingDelete.as_view(), name='sp-fr-mapping-delete'),

    url(r'^(?P<slug>[\w,-]+)/run/$',

        views.run, name='sp-fr-run'),

]

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.views.generic import UpdateView, CreateView, DeleteView

from django.shortcuts import get_object_or_404

from django.http import HttpResponseRedirect

from passerelle.base.mixins import ResourceChildViewMixin

from . import models, forms

class StayIfChanged(object):

    has_changed = False

    def form_valid(self, form):

        if set(form.changed_data) & set(['procedure', 'formdef']):

            self.has_changed = True

        return super(StayIfChanged, self).form_valid(form)

    def get_success_url(self):

        if self.has_changed:

            return self.get_changed_url()

        return super(StayIfChanged, self).get_success_url()

    def get_changed_url(self):

        return ''

class MappingNew(StayIfChanged, ResourceChildViewMixin, CreateView):

    model = models.Mapping

    form_class = forms.MappingForm

    def form_valid(self, form):

        form.instance.resource = self.resource

        return super(MappingNew, self).form_valid(form)

    def get_changed_url(self):

        return self.object.get_absolute_url()

class MappingEdit(StayIfChanged, ResourceChildViewMixin, UpdateView):

    model = models.Mapping

    form_class = forms.MappingForm

class MappingDelete(ResourceChildViewMixin, DeleteView):

    model = models.Mapping

def run(request, connector, slug):

    resource = get_object_or_404(models.Resource, slug=slug)

    resource.run_loop(1000)

    return HttpResponseRedirect(resource.get_absolute_url())

    'passerelle.apps.sp_fr',

.expression-widget input {

	width: 100%;

}

.variable-widget select {

	width: 100%;

}

# -*- coding: utf-8 -*-

# passerelle - uniform access to multiple data sources and services

# Copyright (C) 2019 Entr'ouvert

#

# This program is free software: you can redistribute it and/or modify it

# under the terms of the GNU Affero General Public License as published

# by the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import pytest

from passerelle.apps.sp_fr.models import Resource

import utils

DUMMY_CONTENT = {

    'DILA': {

        'a.zip': 'a',

    }

}

@pytest.fixture

def spfr(settings, db, sftpserver):

    settings.KNOWN_SERVICES = {

        'wcs': {

            'eservices': {

                'title': u'Démarches',

                'url': 'https://demarches-hautes-alpes.test.entrouvert.org/',

                'secret': '9a1f62b680c1cabe73cefcbc5ff4cd4f95c24c3be8dd930adb80d8aaa33bfe67',

                'orig': 'passerelle-hautes-alpes.test.entrouvert.org',

            }

        }

    }

    yield utils.make_resource(

        Resource,

        title='Test 1',

        slug='test1',

        description='Connecteur de test',

        input_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver),

        output_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver)

    )

def test_resource(spfr):

    from passerelle.utils.wcs import get_wcs_choices

    assert get_wcs_choices() == []

def test_sftp_access(spfr, sftpserver):

    with sftpserver.serve_content(DUMMY_CONTENT):

        with spfr.input_sftp as input_sftp:

            assert input_sftp.listdir() == ['a.zip']

        with spfr.output_sftp as output_sftp:

            assert output_sftp.listdir() == ['a.zip']