0010-initialize-sp_fr-connector-31595.patch
passerelle/apps/sp_fr/DOC.XSD | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<!-- edited with XMLSpy v2010 (http://www.altova.com) by BULL SAS (BULL SAS) --> |
|
3 |
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"> |
|
4 |
<xs:complexType name="DECLARANT"> |
|
5 |
<xs:sequence> |
|
6 |
<xs:element name="identité" type="identité" minOccurs="0"/> |
|
7 |
<xs:element name="designation-permis" type="designation-permis" minOccurs="0"/> |
|
8 |
<xs:element name="coordonnees" type="coordonnees" minOccurs="0"/> |
|
9 |
</xs:sequence> |
|
10 |
</xs:complexType> |
|
11 |
<xs:complexType name="identité"> |
|
12 |
<xs:sequence> |
|
13 |
<xs:element name="type-personne" type="xs:boolean" minOccurs="0"/> |
|
14 |
<xs:element name="personne-physique" type="personne-physique" minOccurs="0"/> |
|
15 |
<xs:element name="personne-morale" type="personne-morale" minOccurs="0"/> |
|
16 |
</xs:sequence> |
|
17 |
</xs:complexType> |
|
18 |
<xs:complexType name="personne-physique"> |
|
19 |
<xs:sequence> |
|
20 |
<xs:element name="civilité" type="xs:string" minOccurs="0"/> |
|
21 |
<xs:element name="nom" type="xs:string" minOccurs="0"/> |
|
22 |
<xs:element name="prenom" type="xs:string" minOccurs="0"/> |
|
23 |
</xs:sequence> |
|
24 |
</xs:complexType> |
|
25 |
<xs:complexType name="personne-morale"> |
|
26 |
<xs:sequence> |
|
27 |
<xs:element name="denomination"/> |
|
28 |
<xs:element name="raison-sociale"/> |
|
29 |
<xs:element name="SIRET"/> |
|
30 |
<xs:element name="categorie-juridique"/> |
|
31 |
<xs:element name="representant-personne-morale" type="representant-personne-morale"/> |
|
32 |
</xs:sequence> |
|
33 |
</xs:complexType> |
|
34 |
<xs:complexType name="representant-personne-morale"> |
|
35 |
<xs:sequence> |
|
36 |
<xs:element name="civilité" type="xs:string" minOccurs="0"/> |
|
37 |
<xs:element name="nom" type="xs:string" minOccurs="0"/> |
|
38 |
<xs:element name="prenom" type="xs:string" minOccurs="0"/> |
|
39 |
</xs:sequence> |
|
40 |
</xs:complexType> |
|
41 |
<xs:element name="DOC"> |
|
42 |
<xs:complexType> |
|
43 |
<xs:sequence> |
|
44 |
<xs:element name="DECLARANT" type="DECLARANT" minOccurs="0"/> |
|
45 |
<xs:element name="OUVERTURE-CHANTIER" type="OUVERTURE-CHANTIER" minOccurs="0"/> |
|
46 |
<xs:element name="acceptation" type="xs:boolean" minOccurs="0"/> |
|
47 |
</xs:sequence> |
|
48 |
</xs:complexType> |
|
49 |
</xs:element> |
|
50 |
<xs:complexType name="designation-permis"> |
|
51 |
<xs:sequence> |
|
52 |
<xs:element name="numero-permis_construire" type="xs:string" minOccurs="0"/> |
|
53 |
<xs:element name="numero-permis_amenager" type="xs:string" minOccurs="0"/> |
|
54 |
</xs:sequence> |
|
55 |
</xs:complexType> |
|
56 |
<xs:complexType name="coordonnees"> |
|
57 |
<xs:sequence> |
|
58 |
<xs:element name="adresse" type="adresse" minOccurs="0"/> |
|
59 |
<xs:element name="courriel" type="xs:string" minOccurs="0"/> |
|
60 |
</xs:sequence> |
|
61 |
</xs:complexType> |
|
62 |
<xs:complexType name="adresse"> |
|
63 |
<xs:sequence> |
|
64 |
<xs:element name="numero-voie" type="xs:string" minOccurs="0"/> |
|
65 |
<xs:element name="extension" type="xs:string" minOccurs="0"/> |
|
66 |
<xs:element name="type-voie" type="xs:string" minOccurs="0"/> |
|
67 |
<xs:element name="nom-voie" type="xs:string" minOccurs="0"/> |
|
68 |
<xs:element name="lieu-dit" type="xs:string" minOccurs="0"/> |
|
69 |
<xs:element name="boite-postale" type="xs:string" minOccurs="0"/> |
|
70 |
<xs:element name="code-postal" type="xs:string" minOccurs="0"/> |
|
71 |
<xs:element name="localite" type="xs:string" minOccurs="0"/> |
|
72 |
<xs:element name="bureau-cedex" type="xs:string" minOccurs="0"/> |
|
73 |
<xs:element name="pays" type="xs:string" minOccurs="0"/> |
|
74 |
<xs:element name="division-territoriale" type="xs:string" minOccurs="0"/> |
|
75 |
</xs:sequence> |
|
76 |
</xs:complexType> |
|
77 |
<xs:complexType name="OUVERTURE-CHANTIER"> |
|
78 |
<xs:sequence> |
|
79 |
<xs:element name="date-ouverture" type="xs:date" minOccurs="0"/> |
|
80 |
<xs:element name="totalite-travaux" type="xs:boolean" minOccurs="0"/> |
|
81 |
<xs:element name="tranche-travaux" type="tranche-travaux" minOccurs="0"/> |
|
82 |
<xs:element name="autorisation-differer-travaux" type="xs:string" minOccurs="0"/> |
|
83 |
<xs:element name="SHON" type="xs:string" minOccurs="0"/> |
|
84 |
<xs:element name="nombre-logements-commences" type="nombre-logements-commences" minOccurs="0"/> |
|
85 |
<xs:element name="repartition-type-financement" type="repartition-type-financement" minOccurs="0"/> |
|
86 |
</xs:sequence> |
|
87 |
</xs:complexType> |
|
88 |
<xs:complexType name="tranche-travaux"> |
|
89 |
<xs:sequence> |
|
90 |
<xs:element name="amenagements-commences" type="xs:string" minOccurs="0"/> |
|
91 |
</xs:sequence> |
|
92 |
</xs:complexType> |
|
93 |
<xs:complexType name="nombre-logements-commences"> |
|
94 |
<xs:sequence> |
|
95 |
<xs:element name="total-logements" type="xs:int" minOccurs="0"/> |
|
96 |
<xs:element name="individuels" type="xs:int" minOccurs="0"/> |
|
97 |
<xs:element name="collectifs" type="xs:int" minOccurs="0"/> |
|
98 |
</xs:sequence> |
|
99 |
</xs:complexType> |
|
100 |
<xs:complexType name="repartition-type-financement"> |
|
101 |
<xs:sequence> |
|
102 |
<xs:element name="logement-locatif-social" type="xs:int" minOccurs="0"/> |
|
103 |
<xs:element name="accession-aidee" type="xs:int" minOccurs="0"/> |
|
104 |
<xs:element name="pret-taux-zero" type="xs:int" minOccurs="0"/> |
|
105 |
<xs:element name="autres-financements" type="xs:int" minOccurs="0"/> |
|
106 |
</xs:sequence> |
|
107 |
</xs:complexType> |
|
108 |
</xs:schema> |
passerelle/apps/sp_fr/RCO.XSD | ||
---|---|---|
1 |
<?xml version="1.0" encoding="UTF-8"?> |
|
2 |
<!-- edited with XMLSpy v2010 rel. 2 (http://www.altova.com) by BULL SAS (BULL SAS) --> |
|
3 |
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> |
|
4 |
<xs:element name="RecensementCitoyen"> |
|
5 |
<xs:complexType> |
|
6 |
<xs:sequence> |
|
7 |
<xs:element name="Convention" type="xs:string"/> |
|
8 |
<xs:element name="Formalite"> |
|
9 |
<xs:complexType> |
|
10 |
<xs:sequence> |
|
11 |
<xs:element name="Identifiant" type="xs:string"/> |
|
12 |
<xs:element name="FormaliteType" type="xs:string"/> |
|
13 |
<xs:element name="DateSoumission" type="xs:string"/> |
|
14 |
<xs:element name="FormaliteMotifCode" type="xs:string" maxOccurs="2"/> |
|
15 |
<xs:element name="FormaliteModeCode" type="xs:string"/> |
|
16 |
</xs:sequence> |
|
17 |
</xs:complexType> |
|
18 |
</xs:element> |
|
19 |
<xs:element name="Personne"> |
|
20 |
<xs:complexType> |
|
21 |
<xs:sequence> |
|
22 |
<xs:element name="Civilite" type="xs:string"/> |
|
23 |
<xs:element name="Sexe" type="xs:string"/> |
|
24 |
<xs:element name="NomFamille" type="xs:string"/> |
|
25 |
<xs:element name="NomUsage" type="xs:string"/> |
|
26 |
<xs:element name="PrenomUsuel" type="xs:string"/> |
|
27 |
<xs:element name="Prenom" type="xs:string" maxOccurs="2"/> |
|
28 |
<xs:element name="DateNaissance" type="xs:string"/> |
|
29 |
<xs:element name="PaysNaissance" type="xs:string"/> |
|
30 |
<xs:element name="Nationalite" type="xs:string"/> |
|
31 |
<xs:element name="CodeINSEENaissance" type="xs:string"/> |
|
32 |
<xs:element name="LieuNaissance" maxOccurs="2"> |
|
33 |
<xs:complexType> |
|
34 |
<xs:sequence> |
|
35 |
<xs:element name="Code" type="xs:string"/> |
|
36 |
<xs:element name="Nom" type="xs:string"/> |
|
37 |
</xs:sequence> |
|
38 |
</xs:complexType> |
|
39 |
</xs:element> |
|
40 |
<xs:element name="AdresseDomicile"> |
|
41 |
<xs:complexType> |
|
42 |
<xs:sequence> |
|
43 |
<xs:element name="PointDeRemise" type="xs:string"/> |
|
44 |
<xs:element name="Complement" type="xs:string"/> |
|
45 |
<xs:element name="NumeroVoie" type="xs:string"/> |
|
46 |
<xs:element name="Extension" type="xs:string"/> |
|
47 |
<xs:element name="TypeVoie" type="xs:string"/> |
|
48 |
<xs:element name="NomVoie" type="xs:string"/> |
|
49 |
<xs:element name="LieuDit" type="xs:string"/> |
|
50 |
<xs:element name="CodePostal" type="xs:string"/> |
|
51 |
<xs:element name="Localite" type="xs:string"/> |
|
52 |
<xs:element name="CodeINSEE" type="xs:string"/> |
|
53 |
</xs:sequence> |
|
54 |
</xs:complexType> |
|
55 |
</xs:element> |
|
56 |
<xs:element name="AdresseResidence"> |
|
57 |
<xs:complexType> |
|
58 |
<xs:sequence> |
|
59 |
<xs:element name="PointDeRemise" type="xs:string"/> |
|
60 |
<xs:element name="Complement" type="xs:string"/> |
|
61 |
<xs:element name="NumeroVoie" type="xs:string"/> |
|
62 |
<xs:element name="Extension" type="xs:string"/> |
|
63 |
<xs:element name="TypeVoie" type="xs:string"/> |
|
64 |
<xs:element name="NomVoie" type="xs:string"/> |
|
65 |
<xs:element name="LieuDit" type="xs:string"/> |
|
66 |
<xs:element name="CodePostal" type="xs:string"/> |
|
67 |
<xs:element name="Localite" type="xs:string"/> |
|
68 |
</xs:sequence> |
|
69 |
</xs:complexType> |
|
70 |
</xs:element> |
|
71 |
<xs:element name="SituationFamille"> |
|
72 |
<xs:complexType> |
|
73 |
<xs:sequence> |
|
74 |
<xs:element name="SituationMatrimoniale" type="xs:string"/> |
|
75 |
<xs:element name="NombreEnfants" type="xs:string"/> |
|
76 |
<xs:element name="Pupille" type="xs:string"/> |
|
77 |
<xs:element name="NombreFrereSoeur" type="xs:string"/> |
|
78 |
</xs:sequence> |
|
79 |
</xs:complexType> |
|
80 |
</xs:element> |
|
81 |
<xs:element name="MethodeContact" maxOccurs="2"> |
|
82 |
<xs:complexType> |
|
83 |
<xs:sequence> |
|
84 |
<xs:element name="URI" type="xs:string"/> |
|
85 |
<xs:element name="CanalCode" type="xs:string"/> |
|
86 |
</xs:sequence> |
|
87 |
</xs:complexType> |
|
88 |
</xs:element> |
|
89 |
</xs:sequence> |
|
90 |
</xs:complexType> |
|
91 |
</xs:element> |
|
92 |
<xs:element name="FiliationPere"> |
|
93 |
<xs:complexType> |
|
94 |
<xs:sequence> |
|
95 |
<xs:element name="NomFamille" type="xs:string"/> |
|
96 |
<xs:element name="PrenomUsuel" type="xs:string"/> |
|
97 |
<xs:element name="Prenom" type="xs:string" maxOccurs="2"/> |
|
98 |
<xs:element name="DateNaissance" type="xs:string"/> |
|
99 |
<xs:element name="PaysNaissance" type="xs:string"/> |
|
100 |
<xs:element name="Nationalite" type="xs:string"/> |
|
101 |
<xs:element name="CodeINSEENaissance" type="xs:string"/> |
|
102 |
<xs:element name="LieuNaissance" maxOccurs="2"> |
|
103 |
<xs:complexType> |
|
104 |
<xs:sequence> |
|
105 |
<xs:element name="Code" type="xs:string"/> |
|
106 |
<xs:element name="Nom" type="xs:string"/> |
|
107 |
</xs:sequence> |
|
108 |
</xs:complexType> |
|
109 |
</xs:element> |
|
110 |
</xs:sequence> |
|
111 |
</xs:complexType> |
|
112 |
</xs:element> |
|
113 |
<xs:element name="FiliationMere"> |
|
114 |
<xs:complexType> |
|
115 |
<xs:sequence> |
|
116 |
<xs:element name="NomFamille" type="xs:string"/> |
|
117 |
<xs:element name="PrenomUsuel" type="xs:string"/> |
|
118 |
<xs:element name="Prenom" type="xs:string" maxOccurs="2"/> |
|
119 |
<xs:element name="DateNaissance" type="xs:string"/> |
|
120 |
<xs:element name="PaysNaissance" type="xs:string"/> |
|
121 |
<xs:element name="Nationalite" type="xs:string"/> |
|
122 |
<xs:element name="CodeINSEENaissance" type="xs:string"/> |
|
123 |
<xs:element name="LieuNaissance" maxOccurs="2"> |
|
124 |
<xs:complexType> |
|
125 |
<xs:sequence> |
|
126 |
<xs:element name="Code" type="xs:string"/> |
|
127 |
<xs:element name="Nom" type="xs:string"/> |
|
128 |
</xs:sequence> |
|
129 |
</xs:complexType> |
|
130 |
</xs:element> |
|
131 |
</xs:sequence> |
|
132 |
</xs:complexType> |
|
133 |
</xs:element> |
|
134 |
</xs:sequence> |
|
135 |
</xs:complexType> |
|
136 |
</xs:element> |
|
137 |
</xs:schema> |
passerelle/apps/sp_fr/forms.py | ||
---|---|---|
1 |
# passerelle - uniform access to multiple data sources and services |
|
2 |
# Copyright (C) 2019 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django import forms |
|
18 | ||
19 |
from . import models, fields |
|
20 | ||
21 | ||
22 |
class MappingForm(forms.ModelForm): |
|
23 | ||
24 |
def __init__(self, *args, **kwargs): |
|
25 |
super(MappingForm, self).__init__(*args, **kwargs) |
|
26 |
if self.instance.procedure and self.instance and self.instance.formdef: |
|
27 |
choices = [('', '--------')] + [(v, v) for v in self.instance.variables] |
|
28 |
for i, field in enumerate(self.schema_fields()): |
|
29 |
label = field.label |
|
30 |
label += ' (%s)' % (field.varname or 'NO VARNAME') |
|
31 |
base_name = str(field.varname or i) |
|
32 |
initial = self.instance.rules.get('fields', {}).get(base_name) |
|
33 |
self.fields['field_%s' % base_name] = fields.VariableAndExpressionField( |
|
34 |
label=label, |
|
35 |
choices=choices, |
|
36 |
initial=initial, |
|
37 |
required=False) |
|
38 | ||
39 |
def table_fields(self): |
|
40 |
return [field for field in self if field.name.startswith('field_')] |
|
41 | ||
42 |
def schema_fields(self): |
|
43 |
if self.instance and self.instance.formdef: |
|
44 |
schema = self.instance.formdef.schema |
|
45 |
for i, field in enumerate(schema.fields): |
|
46 |
if field.type in ('page', 'comment', 'title', 'subtitle'): |
|
47 |
continue |
|
48 |
yield field |
|
49 | ||
50 |
def save(self, commit=True): |
|
51 |
fields = {} |
|
52 |
for key in self.cleaned_data: |
|
53 |
if not key.startswith('field_'): |
|
54 |
continue |
|
55 |
if not self.cleaned_data[key]: |
|
56 |
continue |
|
57 |
real_key = key[len('field_'):] |
|
58 |
value = self.cleaned_data[key].copy() |
|
59 |
value['label'] = self.fields[key].label |
|
60 |
fields[real_key] = value |
|
61 |
self.instance.rules['fields'] = fields |
|
62 |
return super(MappingForm, self).save(commit=commit) |
|
63 | ||
64 |
class Meta: |
|
65 |
model = models.Mapping |
|
66 |
fields = [ |
|
67 |
'procedure', |
|
68 |
'formdef', |
|
69 |
] |
passerelle/apps/sp_fr/models.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# passerelle - uniform access to multiple data sources and services |
|
3 |
# Copyright (C) 2019 Entr'ouvert |
|
4 |
# |
|
5 |
# This program is free software: you can redistribute it and/or modify it |
|
6 |
# under the terms of the GNU Affero General Public License as published |
|
7 |
# by the Free Software Foundation, either version 3 of the License, or |
|
8 |
# (at your option) any later version. |
|
9 |
# |
|
10 |
# This program is distributed in the hope that it will be useful, |
|
11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
13 |
# GNU Affero General Public License for more details. |
|
14 |
# |
|
15 |
# You should have received a copy of the GNU Affero General Public License |
|
16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
17 | ||
18 |
import re |
|
19 |
import os |
|
20 |
import stat |
|
21 |
import zipfile |
|
22 |
import collections |
|
23 |
import base64 |
|
24 |
import datetime |
|
25 |
import unicodedata |
|
26 | ||
27 |
from lxml import etree as ET |
|
28 | ||
29 |
from django.core.urlresolvers import reverse |
|
30 |
from django.core.files import File |
|
31 |
from django.db import models, transaction |
|
32 |
from django.template import engines |
|
33 |
from django.utils import six |
|
34 |
from django.utils.translation import ugettext_lazy as _ |
|
35 | ||
36 |
from jsonfield import JSONField |
|
37 | ||
38 |
from passerelle.base.models import BaseResource |
|
39 |
from passerelle.utils.api import endpoint |
|
40 |
from passerelle.utils.sftp import SFTPField |
|
41 |
from passerelle.utils.wcs import FormDefField, get_wcs_choices |
|
42 |
from passerelle.utils.xsd import Schema |
|
43 |
from passerelle.utils.xml import text_content |
|
44 | ||
45 | ||
46 |
PROCEDURE_DOC = 'DOC' |
|
47 |
PROCEDURE_RCO = 'RCO' |
|
48 |
PROCEDURE_DDPACS = 'DDPACS' |
|
49 |
PROCEDURES = [ |
|
50 |
(PROCEDURE_DOC, _('Request for construction site opening')), |
|
51 |
(PROCEDURE_RCO, _('Request for mandatory citizen census')), |
|
52 |
(PROCEDURE_DDPACS, _('Pre-request for citizen solidarity pact')), |
|
53 |
] |
|
54 | ||
55 |
FILE_PATTERN = re.compile(r'^(?P<identifier>.*)-(?P<procedure>[A-Z]+)-(?P<sequence>\d+).zip$') |
|
56 |
ENT_PATTERN = re.compile(r'^.*-ent-\d+(?:-.*)?.xml$') |
|
57 |
DOC_PATTERN = re.compile(r'^.*-doc-\d+-XML-\d+(?:-.*)?\.xml$') |
|
58 |
DOC_PDF_PATTERN = re.compile(r'^.*-doc-\d+-PDF-\d+(?:-.*)?\.pdf$') |
|
59 |
PJ_PATTERN = re.compile(r'^.*-pj-(?P<type>[^-]+)-\d+\.(?P<extension>[^.]+)$') |
|
60 |
NSMAP = { |
|
61 |
'dgme-metier': 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier' |
|
62 |
} |
|
63 |
ROUTAGE_XPATH = ET.XPath( |
|
64 |
('dgme-metier:Routage/dgme-metier:Donnee/dgme-metier:Valeur/text()'), |
|
65 |
namespaces=NSMAP) |
|
66 | ||
67 |
ET.register_namespace('dgme-metier', 'http://finances.gouv.fr/dgme/gf/composants/teledemarchexml/donnee/metier') |
|
68 | ||
69 | ||
70 |
def simplify(s): |
|
71 |
'''Simplify XML node tag names because XSD from DGME are garbage''' |
|
72 |
if not s: |
|
73 |
return '' |
|
74 |
if not isinstance(s, six.text_type): |
|
75 |
s = six.text_type(s, 'utf-8', 'ignore') |
|
76 |
s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') |
|
77 |
s = re.sub(r'[^\w\s\'-_]', '', s) |
|
78 |
s = s.replace('-', '_') |
|
79 |
s = re.sub(r'[\s\']+', '', s) |
|
80 |
return s.strip().lower() |
|
81 | ||
82 | ||
83 |
class Resource(BaseResource): |
|
84 |
category = _('Business Process Connectors') |
|
85 | ||
86 |
input_sftp = SFTPField( |
|
87 |
verbose_name=_('Input SFTP URL'), |
|
88 |
null=True) |
|
89 | ||
90 |
output_sftp = SFTPField( |
|
91 |
verbose_name=_('Output SFTP URL'), |
|
92 |
null=True) |
|
93 | ||
94 |
def check_status(self): |
|
95 |
with self.input_sftp.client() as sftp: |
|
96 |
sftp.listdir() |
|
97 |
with self.output_sftp.client() as sftp: |
|
98 |
sftp.listdir() |
|
99 |
get_wcs_choices(session=self.requests) |
|
100 | ||
101 |
@endpoint(name='ping', description=_('Check Solis API availability')) |
|
102 |
def ping(self, request): |
|
103 |
self.check_status() |
|
104 |
return {'err': 0} |
|
105 | ||
106 |
def run_loop(self, count=1): |
|
107 |
with transaction.atomic(): |
|
108 |
# lock resource |
|
109 |
r = Resource.objects.select_for_update(skip_locked=True).filter(pk=self.pk) |
|
110 |
if not r: |
|
111 |
# already locked |
|
112 |
self.logger.info('did nothing') |
|
113 |
return |
|
114 |
with self.input_sftp.client() as sftp: |
|
115 |
try: |
|
116 |
sftp.lstat('DONE') |
|
117 |
except IOError: |
|
118 |
sftp.mkdir('DONE') |
|
119 | ||
120 |
try: |
|
121 |
sftp.lstat('FAILED') |
|
122 |
except IOError: |
|
123 |
sftp.mkdir('FAILED') |
|
124 | ||
125 |
def helper(): |
|
126 |
for file_stat in sftp.listdir_attr(): |
|
127 |
if stat.S_ISDIR(file_stat.st_mode): |
|
128 |
continue |
|
129 |
yield file_stat.filename |
|
130 | ||
131 |
for filename, i in zip(helper(), range(count)): |
|
132 |
m = FILE_PATTERN.match(filename) |
|
133 |
if not m: |
|
134 |
self.logger.info('file "%s" did not match pattern %s, moving to FAILED/', filename, FILE_PATTERN) |
|
135 |
sftp.rename(filename, 'FAILED/' + filename) |
|
136 |
continue |
|
137 |
procedure = m.group('procedure') |
|
138 |
try: |
|
139 |
mapping = self.mappings.get(procedure=procedure) |
|
140 |
except Mapping.DoesNotExist: |
|
141 |
self.logger.info('no mapping for procedure "%s" for file "%s", moving to FAILED/', |
|
142 |
procedure, filename) |
|
143 |
continue |
|
144 | ||
145 |
handler = self.FileHandler( |
|
146 |
resource=self, |
|
147 |
sftp=sftp, |
|
148 |
filename=filename, |
|
149 |
identifier=m.group('identifier'), |
|
150 |
procedure=procedure, |
|
151 |
sequence=m.group('sequence'), |
|
152 |
mapping=mapping) |
|
153 |
try: |
|
154 |
move, error = handler() |
|
155 |
except Exception: |
|
156 |
self.logger.exception('handling of file "%s" failed', filename) |
|
157 |
# sftp.rename(filename, 'FAILED/' + filename) |
|
158 |
else: |
|
159 |
if move and error: |
|
160 |
self.logger.error('handling of file "%s" failed: %s', filename, error) |
|
161 |
# sftp.rename(filename, 'FAILED/' + filename) |
|
162 |
else: |
|
163 |
if error: |
|
164 |
self.logger.warning('handling of file "%s" failed: %s', filename, error) |
|
165 |
elif move: |
|
166 |
sftp.rename(filename, 'DONE/' + filename) |
|
167 | ||
168 |
class FileHandler(object): |
|
169 |
def __init__(self, resource, sftp, filename, identifier, procedure, sequence, mapping): |
|
170 |
self.resource = resource |
|
171 |
self.sftp = sftp |
|
172 |
self.filename = filename |
|
173 |
self.identifier = identifier |
|
174 |
self.procedure = procedure |
|
175 |
self.sequence = sequence |
|
176 |
self.mapping = mapping |
|
177 |
self.variables = list(self.mapping.variables) |
|
178 |
self.request = Request.objects.filter(resource=resource, filename=filename).first() |
|
179 | ||
180 |
def __call__(self): |
|
181 |
if not self.request: |
|
182 |
with self.sftp.open(self.filename) as fd: |
|
183 |
with transaction.atomic(): |
|
184 |
self.request = Request.objects.create( |
|
185 |
resource=self.resource, |
|
186 |
filename=self.filename) |
|
187 |
self.request.state = Request.STATE_RECEIVED |
|
188 |
self.request.archive.save(self.filename, File(fd)) |
|
189 |
if self.request.state == Request.STATE_RECEIVED: |
|
190 |
with self.request.archive as fd: |
|
191 |
# error during processing are fatal, we want to log them |
|
192 |
data = self.process(fd) |
|
193 |
try: |
|
194 |
backoffice_url = self.transfer(data) |
|
195 |
except Exception as e: |
|
196 |
return False, 'error during transfer to w.c.s %s' % e |
|
197 |
self.request.url = backoffice_url |
|
198 |
self.request.state = Request.STATE_TRANSFERED |
|
199 |
self.request.save() |
|
200 | ||
201 |
if self.request.state == Request.STATE_TRANSFERED: |
|
202 |
try: |
|
203 |
self.response() |
|
204 |
except Exception as e: |
|
205 |
return False, 'error during response to service-public.fr %s' % e |
|
206 |
self.request.state = Request.STATE_RETURNED |
|
207 |
self.request.save() |
|
208 | ||
209 |
def process(self, fd): |
|
210 |
try: |
|
211 |
archive = zipfile.ZipFile(fd) |
|
212 |
except Exception: |
|
213 |
return False, 'could not load zipfile' |
|
214 |
# sort files |
|
215 |
doc_files = [] |
|
216 |
ent_files = [] |
|
217 |
attachments = {} |
|
218 |
for name in archive.namelist(): |
|
219 |
if ENT_PATTERN.match(name): |
|
220 |
ent_files.append(name) |
|
221 |
elif DOC_PATTERN.match(name): |
|
222 |
doc_files.append(name) |
|
223 |
elif DOC_PDF_PATTERN.match(name): |
|
224 |
attachments['doc_pdf'] = [name] |
|
225 |
else: |
|
226 |
m = PJ_PATTERN.match(name) |
|
227 |
if m: |
|
228 |
attachments.setdefault(m.group('type'), []).append(name) |
|
229 | ||
230 |
if len(ent_files) != 1: |
|
231 |
return False, 'too many/few ent files found: %s' % ent_files |
|
232 |
if len(doc_files) != 1: |
|
233 |
return False, 'too many/few doc files found: %s' % doc_files |
|
234 | ||
235 |
for key in attachments: |
|
236 |
if len(attachments[key]) > 1: |
|
237 |
return False, 'too many attachments of kind %s: %s' % (key, attachments[key]) |
|
238 |
name = attachments[key][0] |
|
239 |
with archive.open(attachments[key][0]) as zip_fd: |
|
240 |
content = zip_fd.read() |
|
241 |
attachments[key] = { |
|
242 |
'filename': name, |
|
243 |
'content': base64.b64encode(content).decode('ascii'), |
|
244 |
'content_type': 'application/octet-stream', |
|
245 |
} |
|
246 |
if self.procedure == 'RCO' and not attachments: |
|
247 |
return False, 'no attachments but RCO requires them' |
|
248 | ||
249 |
ent_file = ent_files[0] |
|
250 |
doc_file = doc_files[0] |
|
251 | ||
252 |
with archive.open(ent_file) as fd: |
|
253 |
document = ET.parse(fd) |
|
254 | ||
255 |
insee_codes = ROUTAGE_XPATH(document) |
|
256 |
if len(insee_codes) != 1: |
|
257 |
return False, 'too many/few insee codes found: %s' % insee_codes |
|
258 |
insee_code = insee_codes[0] |
|
259 | ||
260 |
data = {'insee_code': insee_code} |
|
261 |
data.update(attachments) |
|
262 | ||
263 |
with archive.open(doc_file) as fd: |
|
264 |
document = ET.parse(fd) |
|
265 |
data.update(self.extract_data(document)) |
|
266 |
if hasattr(self, 'update_data_%s' % self.procedure): |
|
267 |
getattr(self, 'update_data_%s' % self.procedure)(data) |
|
268 |
return data |
|
269 | ||
270 |
def transfer(self, data): |
|
271 |
formdef = self.mapping.formdef |
|
272 |
formdef.session = self.resource.requests |
|
273 | ||
274 |
with formdef.submit() as submitter: |
|
275 |
submitter.submission_channel = 'web' |
|
276 |
submitter.submission_context = { |
|
277 |
'mdel_procedure': self.procedure, |
|
278 |
'mdel_identifier': self.identifier, |
|
279 |
'mdel_sequence': self.sequence, |
|
280 |
} |
|
281 |
fields = self.mapping.rules.get('fields', {}) |
|
282 |
for name in fields: |
|
283 |
field = fields[name] |
|
284 |
variable = field['variable'] |
|
285 |
expression = field['expression'] |
|
286 |
value = data.get(variable) |
|
287 |
if expression.strip(): |
|
288 |
template = engines['django'].from_string(expression) |
|
289 |
context = data.copy() |
|
290 |
context['value'] = value |
|
291 |
value = template.render(context) |
|
292 |
submitter.set(name, value) |
|
293 |
return submitter.result.backoffice_url |
|
294 | ||
295 |
def response(self): |
|
296 |
raise NotImplementedError |
|
297 | ||
298 |
def get_data(self, data, name): |
|
299 |
# prevent error in manual mapping |
|
300 |
assert name in self.variables, 'variable "%s" is unknown' % name |
|
301 |
return data.get(name, '') |
|
302 | ||
303 |
def update_data_DOC(self, data): |
|
304 |
def get(name): |
|
305 |
return self.get_data(data, name) |
|
306 | ||
307 |
numero_permis_construire = get('doc_declarant_designation_permis_numero_permis_construire') |
|
308 |
numero_permis_amenager = get('doc_declarant_designation_permis_numero_permis_amenager') |
|
309 |
data['type_permis'] = u'Un permis de construire' if numero_permis_construire else u'Un permis d\'aménager' |
|
310 |
data['numero_permis'] = numero_permis_construire or numero_permis_amenager |
|
311 |
particulier = get('doc_declarant_identite_type_personne').strip().lower() == 'true' |
|
312 |
data['type_declarant'] = u'Un particulier' if particulier else u'Une personne morale' |
|
313 |
if particulier: |
|
314 |
data['nom'] = get('doc_declarant_identite_personne_physique_nom') |
|
315 |
data['prenoms'] = get('doc_declarant_identite_personne_physique_prenom') |
|
316 |
else: |
|
317 |
data['nom'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_nom') |
|
318 |
data['prenoms'] = get('doc_declarant_identite_personne_morale_representant_personne_morale_prenom') |
|
319 |
mapping = { |
|
320 |
'1000': 'Monsieur', |
|
321 |
'1001': 'Madame', |
|
322 |
'1002': 'Madame et Monsieur', |
|
323 |
} |
|
324 |
if particulier: |
|
325 |
data['civilite_particulier'] = mapping.get(get('doc_declarant_identite_personne_physique_civilite'), '') |
|
326 |
else: |
|
327 |
data['civilite_pm'] = mapping.get( |
|
328 |
get('doc_declarant_identite_personne_morale_representant_personne_morale_civilite'), '') |
|
329 |
data['portee'] = (u'Pour la totalité des travaux' |
|
330 |
if get('doc_ouverture_chantier_totalite_travaux').lower().strip() == 'true' |
|
331 |
else u'Pour une tranche des travaux') |
|
332 | ||
333 |
def update_data_RCO(self, data): |
|
334 |
def get(name): |
|
335 |
return self.get_data(data, name) |
|
336 | ||
337 |
motif = ( |
|
338 |
get('recensementcitoyen_formalite_formalitemotifcode_1') |
|
339 |
or get('recensementcitoyen_formalite_formalitemotifcode_2') |
|
340 |
) |
|
341 |
data['motif'] = { |
|
342 |
'RECENSEMENT': '1', |
|
343 |
'EXEMPTION': '2' |
|
344 |
}[motif] |
|
345 |
if data['motif'] == '2': |
|
346 |
data['motif_exempte'] = ( |
|
347 |
u"Titulaire d'une carte d'invalidité de 80% minimum" |
|
348 |
if get('recensementcitoyen_formalite_formalitemotifcode_2') == 'INFIRME' |
|
349 |
else u"Autre situation") |
|
350 |
data['justificatif_exemption'] = get('je') |
|
351 |
data['double_nationalite'] = ( |
|
352 |
'Oui' |
|
353 |
if get('recensementcitoyen_personne_nationalite') |
|
354 |
else 'Non') |
|
355 |
data['residence_differente'] = ( |
|
356 |
'Oui' |
|
357 |
if get('recensementcitoyen_personne_adresseresidence_localite') |
|
358 |
else 'Non') |
|
359 |
data['civilite'] = ( |
|
360 |
'Monsieur' |
|
361 |
if get('recensementcitoyen_personne_civilite') == 'M' |
|
362 |
else 'Madame' |
|
363 |
) |
|
364 | ||
365 |
def get_lieu_naissance(variable, code): |
|
366 |
for idx in ['', '_1', '_2']: |
|
367 |
v = variable + idx |
|
368 |
if get(v + '_code') == code: |
|
369 |
return get(v + '_nom') |
|
370 | ||
371 |
data['cp_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'AUTRE') |
|
372 |
data['commune_naissance'] = get_lieu_naissance('recensementcitoyen_personne_lieunaissance', 'COMMUNE') |
|
373 |
data['justificatif_identite'] = get('ji') |
|
374 |
situation_matrimoniale = get('recensementcitoyen_personne_situationfamille_situationmatrimoniale') |
|
375 |
data['situation_familiale'] = { |
|
376 |
u'Célibataire': u'Célibataire', |
|
377 |
u'Marié': u'Marié(e)', |
|
378 |
}.get(situation_matrimoniale, u'Autres') |
|
379 |
if data['situation_familiale'] == u'Autres': |
|
380 |
data['situation_familiale_precision'] = situation_matrimoniale |
|
381 |
pupille = get('recensementcitoyen_personne_situationfamille_pupille') |
|
382 |
data['pupille'] = ( |
|
383 |
'Oui' |
|
384 |
if pupille |
|
385 |
else 'Non' |
|
386 |
) |
|
387 |
data['pupille_categorie'] = { |
|
388 |
'NATION': u"Pupille de la nation", |
|
389 |
'ETAT': u"Pupille de l'État", |
|
390 |
}.get(pupille) |
|
391 |
for idx in ['', '_1', '_2']: |
|
392 |
code = get('recensementcitoyen_personne_methodecontact%s_canalcode' % idx) |
|
393 |
uri = get('recensementcitoyen_personne_methodecontact%s_uri' % idx) |
|
394 |
if code == 'EMAIL': |
|
395 |
data['courriel'] = uri |
|
396 |
if code == 'TEL': |
|
397 |
data['telephone_fixe'] = uri |
|
398 |
data['justificatif_famille'] = data['jf'] |
|
399 |
data['filiation_inconnue_p1'] = not get('recensementcitoyen_filiationpere_nomfamille') |
|
400 |
data['filiation_inconnue_p2'] = not get('recensementcitoyen_filiationmere_nomfamille') |
|
401 |
data['cp_naissance_p1'] = get_lieu_naissance('recensementcitoyen_filiationpere_lieunaissance', 'AUTRE') |
|
402 |
data['cp_naissance_p2'] = get_lieu_naissance('recensementcitoyen_filiationmere_lieunaissance', 'AUTRE') |
|
403 |
data['commune_naissance_p1'] = get_lieu_naissance( |
|
404 |
'recensementcitoyen_filiationpere_lieunaissance', 'COMMUNE') |
|
405 |
data['commune_naissance_p2'] = get_lieu_naissance( |
|
406 |
'recensementcitoyen_filiationmere_lieunaissance', 'COMMUNE') |
|
407 |
for key in data: |
|
408 |
if key.endswith('_datenaissance') and data[key]: |
|
409 |
data[key] = ( |
|
410 |
datetime.datetime.strptime(data[key], '%d/%m/%Y') |
|
411 |
.date() |
|
412 |
.strftime('%Y-%m-%d') |
|
413 |
) |
|
414 | ||
415 |
def extract_data(self, document): |
|
416 |
'''Convert XML into a dictionnary of values''' |
|
417 |
root = document.getroot() |
|
418 | ||
419 |
def tag_name(node): |
|
420 |
return simplify(ET.QName(node.tag).localname) |
|
421 | ||
422 |
def helper(path, node): |
|
423 |
if len(node): |
|
424 |
tags = collections.Counter(tag_name(child) for child in node) |
|
425 |
counter = collections.Counter() |
|
426 |
for child in node: |
|
427 |
name = tag_name(child) |
|
428 |
if tags[name] > 1: |
|
429 |
counter[name] += 1 |
|
430 |
name += '_%s' % counter[name] |
|
431 |
for p, value in helper(path + [name], child): |
|
432 |
yield p, value |
|
433 |
else: |
|
434 |
yield path, text_content(node) |
|
435 |
return {'_'.join(path): value for path, value in helper([tag_name(root)], root)} |
|
436 | ||
437 |
class Meta: |
|
438 |
verbose_name = _('Service-Public.fr') |
|
439 | ||
440 | ||
441 |
class SPFRMessage(object): |
|
442 |
@classmethod |
|
443 |
def from_file(self, resource, filename, fd): |
|
444 |
m = re.match(r'(.*)-([A-Z]+)-(\d+).zip', filename) |
|
445 |
if not m: |
|
446 |
resource.logger.warning('found file with an unknown pattern %s moving in DONE/', filename) |
|
447 |
return None |
|
448 |
mdel_number, procedure, sequence = m.groups() |
|
449 | ||
450 | ||
451 |
def default_rule(): |
|
452 |
return {} |
|
453 | ||
454 | ||
455 |
class Mapping(models.Model): |
|
456 |
resource = models.ForeignKey( |
|
457 |
Resource, |
|
458 |
verbose_name=_('Resource'), |
|
459 |
related_name='mappings') |
|
460 | ||
461 |
procedure = models.CharField( |
|
462 |
verbose_name=_('Procedure'), |
|
463 |
choices=PROCEDURES, |
|
464 |
unique=True, |
|
465 |
max_length=8) |
|
466 | ||
467 |
formdef = FormDefField( |
|
468 |
verbose_name=_('Formulaire')) |
|
469 | ||
470 |
rules = JSONField( |
|
471 |
verbose_name=_('Rules'), |
|
472 |
default=default_rule) |
|
473 | ||
474 |
def get_absolute_url(self): |
|
475 |
return reverse('sp-fr-mapping-edit', kwargs=dict( |
|
476 |
slug=self.resource.slug, |
|
477 |
pk=self.pk)) |
|
478 | ||
479 |
@property |
|
480 |
def xsd(self): |
|
481 |
doc = ET.parse(os.path.join(os.path.dirname(__file__), '%s.XSD' % self.procedure)) |
|
482 |
schema = Schema() |
|
483 |
schema.visit(doc.getroot()) |
|
484 |
return schema |
|
485 | ||
486 |
@property |
|
487 |
def variables(self): |
|
488 |
yield 'insee_code' |
|
489 |
for path, xsd_type in self.xsd.paths(): |
|
490 |
names = [simplify(tag.localname) for tag in path] |
|
491 |
yield '_'.join(names) |
|
492 |
if hasattr(self, 'variables_%s' % self.procedure): |
|
493 |
for variable in getattr(self, 'variables_%s' % self.procedure): |
|
494 |
yield variable |
|
495 | ||
496 |
@property |
|
497 |
def variables_DOC(self): |
|
498 |
yield 'type_permis' |
|
499 |
yield 'numero_permis' |
|
500 |
yield 'type_declarant' |
|
501 |
yield 'nom' |
|
502 |
yield 'prenoms' |
|
503 |
yield 'civilite_particulier' |
|
504 |
yield 'civilite_pm' |
|
505 |
yield 'portee' |
|
506 | ||
507 |
@property |
|
508 |
def variables_RCO(self): |
|
509 |
yield 'motif' |
|
510 |
yield 'motif_exemple' |
|
511 |
yield 'justificatif_exemption' |
|
512 |
yield 'double_nationalite' |
|
513 |
yield 'residence_differente' |
|
514 |
yield 'civilite' |
|
515 |
yield 'cp_naissance' |
|
516 |
yield 'commune_naissance' |
|
517 |
yield 'je' |
|
518 |
yield 'ji' |
|
519 |
yield 'situation_familiale' |
|
520 |
yield 'situation_familiale_precision' |
|
521 |
yield 'pupille' |
|
522 |
yield 'pupille_categorie' |
|
523 |
yield 'courriel' |
|
524 |
yield 'telephone_fixe' |
|
525 |
yield 'jf' |
|
526 |
yield 'filiation_inconnue_p1' |
|
527 |
yield 'filiation_inconnue_p2' |
|
528 |
yield 'cp_naissance_p1' |
|
529 |
yield 'cp_naissance_p2' |
|
530 |
yield 'commune_naissance_p1' |
|
531 |
yield 'commune_naissance_p2' |
|
532 | ||
533 | ||
534 |
class Request(models.Model): |
|
535 |
# To prevent mixing errors from analysing archive from s-p.fr and errors |
|
536 |
# from pushing to w.c.s we separate processing with three steps: |
|
537 |
# - receiving, i.e. copying zipfile from SFTP and storing them locally |
|
538 |
# - processing, i.e. openeing the zipfile and extracting content as we need it |
|
539 |
# - transferring, pushing content as a new form in w.c.s. |
|
540 |
STATE_RECEIVED = 'received' |
|
541 |
STATE_TRANSFERED = 'transfered' |
|
542 |
STATE_RETURNED = 'returned' |
|
543 |
STATE_ERROR = 'error' |
|
544 |
STATES = [ |
|
545 |
(STATE_RECEIVED, _('Received')), |
|
546 |
(STATE_TRANSFERED, _('Transfered')), |
|
547 |
(STATE_ERROR, _('Transfered')), |
|
548 |
(STATE_RETURNED, _('Returned')), |
|
549 |
] |
|
550 | ||
551 |
resource = models.ForeignKey( |
|
552 |
Resource, |
|
553 |
verbose_name=_('Resource')) |
|
554 | ||
555 |
created = models.DateTimeField( |
|
556 |
verbose_name=_('Created'), |
|
557 |
auto_now_add=True) |
|
558 | ||
559 |
modified = models.DateTimeField( |
|
560 |
verbose_name=_('Created'), |
|
561 |
auto_now=True) |
|
562 | ||
563 |
filename = models.CharField( |
|
564 |
verbose_name=_('Identifier'), |
|
565 |
max_length=128) |
|
566 | ||
567 |
archive = models.FileField( |
|
568 |
verbose_name=_('Archive'), |
|
569 |
max_length=256) |
|
570 | ||
571 |
state = models.CharField( |
|
572 |
verbose_name=_('State'), |
|
573 |
choices=STATES, |
|
574 |
default=STATE_RECEIVED, |
|
575 |
max_length=16) |
|
576 | ||
577 |
url = models.URLField( |
|
578 |
verbose_name=_('URL'), |
|
579 |
blank=True) |
|
580 | ||
581 |
class Meta: |
|
582 |
unique_together = ( |
|
583 |
('resource', 'filename'), |
|
584 |
) |
passerelle/apps/sp_fr/templates/passerelle/widgets/variable_and_expression_widget.html | ||
---|---|---|
1 |
<div class="variable-widget"> |
|
2 |
{% include widget.subwidgets.0.template_name with widget=widget.subwidgets.0 %} |
|
3 |
</div> |
|
4 |
<div class="expression-widget"> |
|
5 |
{% include widget.subwidgets.1.template_name with widget=widget.subwidgets.1 %} |
|
6 |
</div> |
passerelle/apps/sp_fr/templates/sp_fr/mapping_confirm_delete.html | ||
---|---|---|
1 |
{% extends "passerelle/manage/resource_child_confirm_delete.html" %} |
passerelle/apps/sp_fr/templates/sp_fr/mapping_form.html | ||
---|---|---|
1 |
{% extends "passerelle/manage/resource_child_form.html" %} |
|
2 |
{% load i18n %} |
|
3 | ||
4 | ||
5 |
{% block form %} |
|
6 |
{% if form.errors %} |
|
7 |
<div class="errornotice"> |
|
8 |
<p>{% trans "There were errors processing your form." %}</p> |
|
9 |
{% for error in form.non_field_errors %} |
|
10 |
<p>{{ error }}</p> |
|
11 |
{% endfor %} |
|
12 |
{% for field in form %} |
|
13 |
{% if field.is_hidden and field.errors %} |
|
14 |
<p> |
|
15 |
{% for error in field.errors %} |
|
16 |
{% blocktrans with name=field.name %}(Hidden field {{name}}) {{ error }}{% endblocktrans %} |
|
17 |
{% if not forloop.last %}<br>{% endif %} |
|
18 |
{% endfor %} |
|
19 |
</p> |
|
20 |
{% endif %} |
|
21 |
{% endfor %} |
|
22 |
</div> |
|
23 |
{% endif %} |
|
24 |
{% include "gadjo/widget.html" with field=form.procedure %} |
|
25 | ||
26 |
{% include "gadjo/widget.html" with field=form.formdef%} |
|
27 | ||
28 |
<table class="main"> |
|
29 |
<thead> |
|
30 |
<tr> |
|
31 |
<td>Label</td> |
|
32 |
<td>Variable</td> |
|
33 |
</tr> |
|
34 |
</thead> |
|
35 |
<tbody> |
|
36 |
{% for field in form.table_fields %} |
|
37 |
<tr> |
|
38 |
<td>{{ field.label_tag }}</td> |
|
39 |
<td>{{ field }}</td> |
|
40 |
</tr> |
|
41 |
{% endfor %} |
|
42 |
</tbody> |
|
43 |
</table> |
|
44 |
{% endblock %} |
passerelle/apps/sp_fr/templates/sp_fr/resource_detail.html | ||
---|---|---|
1 |
{% extends "passerelle/manage/service_view.html" %} |
|
2 |
{% load i18n passerelle %} |
|
3 | ||
4 |
{% block description %} |
|
5 |
<p> |
|
6 |
{% blocktrans %} |
|
7 |
Connector to forms published by <a href="https://www.service-public.fr/">service-public.fr</a> |
|
8 |
{% endblocktrans %} |
|
9 |
<a href="{% url "sp-fr-run" slug=object.slug %}">Run</a> |
|
10 |
</p> |
|
11 |
{{ block.super }} |
|
12 |
{% endblock %} |
|
13 | ||
14 | ||
15 |
{% block extra-sections %} |
|
16 |
<div id="mappings" class="section"> |
|
17 |
<h3>{% trans "Mappings" %} <a class="button" href="{% url "sp-fr-mapping-new" slug=object.slug %}">{% trans "Add" %}</a></h3> |
|
18 |
<ul> |
|
19 | ||
20 |
{% for mapping in object.mappings.all %} |
|
21 |
<li> |
|
22 |
<fieldset class="gadjo-foldable gadjo-folded" id="sp-fr-mapping-{{ mapping.pk}}"> |
|
23 |
<legend class="gadjo-foldable-widget"> |
|
24 |
<a href="{% url "sp-fr-mapping-edit" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% blocktrans with procedure=mapping.get_procedure_display formdef=mapping.formdef.title %}From procedure {{ procedure }} to form {{ formdef }}{% endblocktrans %}</a> |
|
25 |
</legend> |
|
26 |
<div class="gadjo-folding"> |
|
27 |
{% for key, value in mapping.rules.fields.items %} |
|
28 |
{% if value %} |
|
29 |
<p>{{ value.label }} : {{ value.variable }} {% if value.expression %}({% trans "with expression" %} <tt>{{ value.expression }}</tt>){% endif %}</p> |
|
30 |
{% endif %} |
|
31 |
{% endfor %} |
|
32 |
<a rel="popup" class="delete" href="{% url "sp-fr-mapping-delete" connector=object.get_connector_slug slug=object.slug pk=mapping.pk %}">{% trans "Delete" %}</a> |
|
33 |
</div> |
|
34 |
</fieldset> |
|
35 |
</li> |
|
36 |
{% endfor %} |
|
37 |
</ul> |
|
38 |
</div> |
|
39 |
<div id="requests" class="section"> |
|
40 |
<h3>{% trans "Mappings" %} <a class="button" href="{% url "sp-fr-mapping-new" slug=object.slug %}">{% trans "Add" %}</a></h3> |
|
41 |
<table class="main"> |
|
42 |
<thead> |
|
43 |
<tr> |
|
44 |
<td>Id</td> |
|
45 |
<td>Created</td> |
|
46 |
<td>Modified</td> |
|
47 |
<td>State</td> |
|
48 |
<td>Filename</td> |
|
49 |
<td>Slug</td> |
|
50 |
<td>Form Id</td> |
|
51 |
</tr> |
|
52 |
</thead> |
|
53 |
<tbody> |
|
54 |
{% for req in object.request_set.all %} |
|
55 |
<tr> |
|
56 |
<td>{{ req.id }}</td> |
|
57 |
<td>{{ req.created }}</td> |
|
58 |
<td>{{ req.modified }}</td> |
|
59 |
<td>{{ req.get_state_display }}</td> |
|
60 |
<td>{{ req.filename }}</td> |
|
61 |
<td>{{ req.content.formdef_slug }}</td> |
|
62 |
<td>{{ req.content.formdata_id }}</td> |
|
63 |
</tr> |
|
64 |
{% endfor %} |
|
65 | ||
66 |
</tbody> |
|
67 |
</table> |
|
68 |
</div> |
|
69 |
{% endblock %} |
passerelle/apps/sp_fr/urls.py | ||
---|---|---|
1 |
# passerelle - uniform access to multiple data sources and services |
|
2 |
# Copyright (C) 2019 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django.conf.urls import url |
|
18 | ||
19 |
from . import views |
|
20 | ||
21 |
management_urlpatterns = [ |
|
22 |
url(r'^(?P<slug>[\w,-]+)/mapping/new/$', |
|
23 |
views.MappingNew.as_view(), name='sp-fr-mapping-new'), |
|
24 |
url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/$', |
|
25 |
views.MappingEdit.as_view(), name='sp-fr-mapping-edit'), |
|
26 |
url(r'^(?P<slug>[\w,-]+)/mapping/(?P<pk>\d+)/delete/$', |
|
27 |
views.MappingDelete.as_view(), name='sp-fr-mapping-delete'), |
|
28 |
url(r'^(?P<slug>[\w,-]+)/run/$', |
|
29 |
views.run, name='sp-fr-run'), |
|
30 |
] |
passerelle/apps/sp_fr/views.py | ||
---|---|---|
1 |
# passerelle - uniform access to multiple data sources and services |
|
2 |
# Copyright (C) 2019 Entr'ouvert |
|
3 |
# |
|
4 |
# This program is free software: you can redistribute it and/or modify it |
|
5 |
# under the terms of the GNU Affero General Public License as published |
|
6 |
# by the Free Software Foundation, either version 3 of the License, or |
|
7 |
# (at your option) any later version. |
|
8 |
# |
|
9 |
# This program is distributed in the hope that it will be useful, |
|
10 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 |
# GNU Affero General Public License for more details. |
|
13 |
# |
|
14 |
# You should have received a copy of the GNU Affero General Public License |
|
15 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | ||
17 |
from django.views.generic import UpdateView, CreateView, DeleteView |
|
18 |
from django.shortcuts import get_object_or_404 |
|
19 |
from django.http import HttpResponseRedirect |
|
20 | ||
21 |
from passerelle.base.mixins import ResourceChildViewMixin |
|
22 | ||
23 |
from . import models, forms |
|
24 | ||
25 | ||
26 |
class StayIfChanged(object): |
|
27 |
has_changed = False |
|
28 | ||
29 |
def form_valid(self, form): |
|
30 |
if set(form.changed_data) & set(['procedure', 'formdef']): |
|
31 |
self.has_changed = True |
|
32 |
return super(StayIfChanged, self).form_valid(form) |
|
33 | ||
34 |
def get_success_url(self): |
|
35 |
if self.has_changed: |
|
36 |
return self.get_changed_url() |
|
37 |
return super(StayIfChanged, self).get_success_url() |
|
38 | ||
39 |
def get_changed_url(self): |
|
40 |
return '' |
|
41 | ||
42 | ||
43 |
class MappingNew(StayIfChanged, ResourceChildViewMixin, CreateView): |
|
44 |
model = models.Mapping |
|
45 |
form_class = forms.MappingForm |
|
46 | ||
47 |
def form_valid(self, form): |
|
48 |
form.instance.resource = self.resource |
|
49 |
return super(MappingNew, self).form_valid(form) |
|
50 | ||
51 |
def get_changed_url(self): |
|
52 |
return self.object.get_absolute_url() |
|
53 | ||
54 | ||
55 |
class MappingEdit(StayIfChanged, ResourceChildViewMixin, UpdateView): |
|
56 |
model = models.Mapping |
|
57 |
form_class = forms.MappingForm |
|
58 | ||
59 | ||
60 |
class MappingDelete(ResourceChildViewMixin, DeleteView): |
|
61 |
model = models.Mapping |
|
62 | ||
63 | ||
64 |
def run(request, connector, slug): |
|
65 |
resource = get_object_or_404(models.Resource, slug=slug) |
|
66 |
resource.run_loop(1000) |
|
67 |
return HttpResponseRedirect(resource.get_absolute_url()) |
passerelle/settings.py | ||
---|---|---|
135 | 135 |
'passerelle.apps.feeds', |
136 | 136 |
'passerelle.apps.gdc', |
137 | 137 |
'passerelle.apps.jsondatastore', |
138 |
'passerelle.apps.sp_fr', |
|
138 | 139 |
'passerelle.apps.mobyt', |
139 | 140 |
'passerelle.apps.okina', |
140 | 141 |
'passerelle.apps.opengis', |
passerelle/static/css/style.css | ||
---|---|---|
184 | 184 |
.log-dialog table td { |
185 | 185 |
vertical-align: top; |
186 | 186 |
} |
187 |
.expression-widget input { |
|
188 |
width: 100%; |
|
189 |
} |
|
190 |
.variable-widget select { |
|
191 |
width: 100%; |
|
192 |
} |
tests/test_sp_fr.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
# passerelle - uniform access to multiple data sources and services |
|
3 |
# Copyright (C) 2019 Entr'ouvert |
|
4 |
# |
|
5 |
# This program is free software: you can redistribute it and/or modify it |
|
6 |
# under the terms of the GNU Affero General Public License as published |
|
7 |
# by the Free Software Foundation, either version 3 of the License, or |
|
8 |
# (at your option) any later version. |
|
9 |
# |
|
10 |
# This program is distributed in the hope that it will be useful, |
|
11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
13 |
# GNU Affero General Public License for more details. |
|
14 |
# |
|
15 |
# You should have received a copy of the GNU Affero General Public License |
|
16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
17 | ||
18 |
import pytest |
|
19 | ||
20 |
from passerelle.apps.sp_fr.models import Resource |
|
21 | ||
22 |
import utils |
|
23 | ||
24 | ||
25 |
DUMMY_CONTENT = { |
|
26 |
'DILA': { |
|
27 |
'a.zip': 'a', |
|
28 |
} |
|
29 |
} |
|
30 | ||
31 | ||
32 |
@pytest.fixture |
|
33 |
def spfr(settings, db, sftpserver): |
|
34 |
settings.KNOWN_SERVICES = { |
|
35 |
'wcs': { |
|
36 |
'eservices': { |
|
37 |
'title': u'Démarches', |
|
38 |
'url': 'https://demarches-hautes-alpes.test.entrouvert.org/', |
|
39 |
'secret': '9a1f62b680c1cabe73cefcbc5ff4cd4f95c24c3be8dd930adb80d8aaa33bfe67', |
|
40 |
'orig': 'passerelle-hautes-alpes.test.entrouvert.org', |
|
41 |
} |
|
42 |
} |
|
43 |
} |
|
44 |
yield utils.make_resource( |
|
45 |
Resource, |
|
46 |
title='Test 1', |
|
47 |
slug='test1', |
|
48 |
description='Connecteur de test', |
|
49 |
input_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver), |
|
50 |
output_sftp='sftp://john:doe@{server.host}:{server.port}/DILA/'.format(server=sftpserver) |
|
51 |
) |
|
52 | ||
53 | ||
54 |
def test_resource(spfr): |
|
55 |
from passerelle.utils.wcs import get_wcs_choices |
|
56 | ||
57 |
assert get_wcs_choices() == [] |
|
58 | ||
59 | ||
60 |
def test_sftp_access(spfr, sftpserver): |
|
61 |
with sftpserver.serve_content(DUMMY_CONTENT): |
|
62 |
with spfr.input_sftp as input_sftp: |
|
63 |
assert input_sftp.listdir() == ['a.zip'] |
|
64 |
with spfr.output_sftp as output_sftp: |
|
65 |
assert output_sftp.listdir() == ['a.zip'] |
|
0 |
- |