Projet

Général

Profil

0001-family-orleans-connector-uses-invoices-dir-path-1585.patch

Voir les différences:

Subject: [PATCH] family: orleans connector uses invoices dir path (#15855)

Instead of copying pdf files into the archive.
 passerelle/apps/family/loaders/concerto_orleans.py |  12 +++--
 .../management/commands/import_orleans_data.py     |  42 ++++++----------
 tests/data/family_data_orleans.zip                 | Bin 29632 -> 0 bytes
 tests/data/orleans/factures/1502033794.pdf         |  49 +++++++++++++++++++
 tests/data/orleans/factures/1602023571.pdf         |  49 +++++++++++++++++++
 tests/data/orleans/factures/1602025774.pdf         |  49 +++++++++++++++++++
 tests/data/orleans/factures/1602031264.pdf         |  49 +++++++++++++++++++
 tests/data/orleans/factures/1602031791.pdf         |  49 +++++++++++++++++++
 tests/data/orleans/family_data_orleans.zip         | Bin 0 -> 3849 bytes
 tests/test_family.py                               |  54 +++++++++------------
 10 files changed, 290 insertions(+), 63 deletions(-)
 delete mode 100644 tests/data/family_data_orleans.zip
 create mode 100644 tests/data/orleans/factures/1502033794.pdf
 create mode 100644 tests/data/orleans/factures/1602023571.pdf
 create mode 100644 tests/data/orleans/factures/1602025774.pdf
 create mode 100644 tests/data/orleans/factures/1602031264.pdf
 create mode 100644 tests/data/orleans/factures/1602031791.pdf
 create mode 100644 tests/data/orleans/family_data_orleans.zip
passerelle/apps/family/loaders/concerto_orleans.py
22 22
from django.utils import timezone
23 23
from django.core.exceptions import ValidationError
24 24
from django.utils.translation import ugettext_lazy as _
25
from django.core.files.storage import DefaultStorage
25 26

  
26 27
from ..models import Family, Adult, Child, Invoice
27 28
from ..models import dict_cherry_pick, get_datetime
......
105 106

  
106 107
    def __init__(self, connector):
107 108
        self.connector = connector
108
        self.invoice_filename_pattern = 'invoices/facture_%s.pdf'
109 109

  
110 110
    def clean(self, archive):
111 111
        for filename in ('extract_prcit_personne.csv', 'extract_prcit_famille.csv',
......
167 167
                        external_id=child_data['external_id'], defaults=child_data)
168 168

  
169 169
                for invoice_data in family_data.get('invoices') or []:
170
                    invoice_filename = self.invoice_filename_pattern % invoice_data['external_id']
170
                    storage = DefaultStorage()
171
                    invoices_dir = storage.path('family-%s/invoices' % self.connector.id)
172
                    invoice_filename = '%s.pdf' % invoice_data['external_id']
173
                    invoice_path = os.path.join(invoices_dir, invoice_filename)
171 174
                    # create invoice object only if associated pdf exists
172
                    if invoice_filename in archive.namelist():
175
                    if os.path.exists(invoice_path):
173 176
                        invoice, created = Invoice.objects.update_or_create(resource=self.connector,
174
                                            family=family, external_id=invoice_data['external_id'], defaults=invoice_data)
175
                        invoice.write_pdf(archive.read(invoice_filename))
177
                            family=family, external_id=invoice_data['external_id'], defaults=invoice_data)
176 178

  
177 179
        except Exception, e:
178 180
            self.connector.logger.error('Error occured while importing data: %s', e)
passerelle/apps/family/management/commands/import_orleans_data.py
23 23

  
24 24
from django.core.files.base import File
25 25
from django.core.management.base import BaseCommand, CommandError
26
from django.core.files.storage import DefaultStorage
26 27

  
27
from family.models import GenericFamily
28

  
29
LOCK_FILENAME = os.path.join(tempfile.gettempdir(), 'import-orleans-data.lock')
28
from family.models import GenericFamily, Invoice
30 29

  
31 30

  
32 31
class Command(BaseCommand):
33 32
    option_list = BaseCommand.option_list + (
34
        make_option('-a', '--archive-name', dest='archive_name',
35
                    help="""Archive name containing data files""",
33
        make_option('-a', '--archive-file', dest='archive_file',
34
                    help="""Archive containing data files""",
36 35
                    default='exports_prcit.zip'
37 36
        ),
38
        make_option('-d', '--data-directory', dest='data_dir',
39
                    help="""Directory containing archive and invoice files"""
40
        ),
41 37
        make_option('-c', '--connector', dest='connector',
42 38
                    help="""Slug of the connector to import data into"""
43 39
        )
44 40
    )
45 41

  
46 42
    def handle(self, *args, **options):
47
        if not options['data_dir']:
48
            raise CommandError('No data dir specified.')
49

  
50
        if not os.path.exists(options['data_dir']):
51
            raise CommandError('Directory %s does not exist.' % options['data_dir'])
52

  
53
        archive_path = os.path.join(options['data_dir'], options['archive_name'])
54 43

  
55
        if not os.path.exists(archive_path):
56
            raise CommandError('File %s does not exist.' % archive_path)
44
        if not os.path.exists(options['archive_file']):
45
            raise CommandError('File %s does not exist.' % options['archive_file'])
57 46

  
58 47
        try:
59 48
            connector = GenericFamily.objects.get(slug=options['connector'])
60 49
        except GenericFamily.DoesNotExist:
61 50
            return
51

  
52
        storage = DefaultStorage()
53
        lock_filename = storage.path('family-%s/import-orleans-data.lock' % connector.id)
62 54
        try:
63
            fd = open(LOCK_FILENAME, 'w')
55
            fd = open(lock_filename, 'w')
64 56
            fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
65
            invoices_dir = os.path.join(options['data_dir'], 'factures')
66
            with zipfile.ZipFile(archive_path, 'a') as archive:
67

  
68
                for invoice_file in os.listdir(invoices_dir):
69
                    if invoice_file.startswith('facture_'):
70
                        archive.write(os.path.join(invoices_dir, invoice_file),
71
                                      'invoices/%s' % invoice_file)
72
            connector.archive.save(options['archive_name'], File(file(archive_path)))
73 57
        except IOError:
74 58
            raise CommandError('Command already running.')
59

  
60
        try:
61
            archive_name = os.path.basename(options['archive_file'])
62
            connector.archive.save(archive_name, File(file(options['archive_file'])))
75 63
        except Exception, e:
76 64
            raise CommandError('Error occured: %s' % e)
77 65
        finally:
78 66
            fd.close()
79
            os.unlink(LOCK_FILENAME)
67
            os.unlink(lock_filename)
tests/data/orleans/factures/1502033794.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20160830174340-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20160830174340-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 117 >>
25
stream
26
Gap@D0a`Fb'LVU]muUBel;&kg/1:darrtnXTbSP0^SOID_oiE2-*/+#0IcjPTSFRrDumOY<bojnht;Ru"Id]=q7Y+a,<r>=%RO'hKjN-/ra<tU(ag%1~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001087 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\363\)\177\203\257\221]\211>l\034\003\251>\030X) (\363\)\177\203\257\221]\211>l\034\003\251>\030X)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1136
49
%%EOF
tests/data/orleans/factures/1602023571.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<1,_!Ke5Q^q;@(YOlVZU%!W]fE'CY~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/data/orleans/factures/1602025774.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 114 >>
25
stream
26
Gap@D0b-H6'Eri?4%tK_s4=Pk$'k'g5W0=>D1qd"OdA`to+ZJJ+(KoMBP'[R[\LS1fYV0Ep'UcQ<Z"6/6ntn8]$CiQ`JNnIn0N6ceb:65*kL;DpA~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001084 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1133
49
%%EOF
tests/data/orleans/factures/1602031264.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<2Dd0Fd8UCn;@(YOlVZU%!W]eq'CG~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/data/orleans/factures/1602031791.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<2Dhm"e5Q^q;@(YOlVZU%!W]g$'Cu~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/test_family.py
5 5
from uuid import uuid4
6 6
import zipfile
7 7
import logging
8
import shutil
8 9

  
9 10
from django.core.exceptions import ValidationError
10 11
from django.core.files import File
......
15 16
from django.core.files import File
16 17
from django.core.management import call_command
17 18
from django.core.management.base import CommandError
19
from django.core.files.storage import default_storage
18 20

  
19 21
from family.models import GenericFamily, Family, FamilyLink
20 22
from family.models import Invoice, Adult, Child, DATETIME_FORMAT
......
220 222
def test_orleans_concerto_loader():
221 223
    # all related objects will also be deleted
222 224
    Family.objects.all().delete()
223
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'family_data_orleans.zip')
225
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'orleans',
226
                            'family_data_orleans.zip')
224 227
    resource = GenericFamily(title='test orleans',
225 228
            slug='test-orleans', archive=filepath, file_format='concerto_orleans')
226 229
    from family.loaders.concerto_orleans import Loader
......
274 277
    assert Family.objects.filter(resource=resource).count() == 18
275 278
    assert Adult.objects.all().count() == 31
276 279
    assert Child.objects.all().count() == 35
277
    assert Invoice.objects.filter(resource=resource).count() == 26
278
    assert Invoice.objects.filter(resource=resource, total_amount=0).count() == 2
279
    assert Invoice.objects.filter(online_payment=True).count() == 26
280
    assert Invoice.objects.filter(resource=resource).count() == 0
280 281

  
281 282
def test_orleans_data_import_command():
282 283
    with pytest.raises(CommandError) as error:
283 284
        call_command('import_orleans_data')
284
    assert str(error.value) == 'No data dir specified.'
285

  
286
    with pytest.raises(CommandError) as error:
287
        call_command('import_orleans_data', data_dir='/tmp/orleans_data')
288
    assert str(error.value) == 'Directory /tmp/orleans_data does not exist.'
285
    assert str(error.value) == 'File exports_prcit.zip does not exist.'
289 286

  
290 287
    cur_dir = os.path.dirname(__file__)
291
    data_dir = os.path.join(cur_dir, 'data')
292
    with pytest.raises(CommandError) as error:
293
        call_command('import_orleans_data', data_dir=data_dir)
294
    assert str(error.value) == 'File %s/exports_prcit.zip does not exist.' % data_dir
288
    data_dir = os.path.join(cur_dir, 'data', 'orleans')
295 289

  
296 290
    resource = GenericFamily.objects.create(title='test orleans',
297 291
                archive=os.path.join(data_dir, 'family_data_orleans.zip'),
......
301 295
    Family.objects.filter(resource=resource).delete()
302 296
    Invoice.objects.filter(resource=resource).delete()
303 297

  
304
    call_command('import_orleans_data', data_dir=data_dir,
305
                 archive_name='family_data_orleans.zip')
298
    call_command('import_orleans_data',
299
                 archive_file=os.path.join(data_dir, 'family_data_orleans.zip'))
306 300
    assert Family.objects.filter(resource=resource).count() == 0
307 301
    assert Invoice.objects.filter(resource=resource).count() == 0
308 302

  
309
    with pytest.raises(Exception) as error:
310
        call_command('import_orleans_data', data_dir=data_dir,
311
                     archive_name='family_data_orleans.zip',
312
                     connector='test-orleans')
313
    assert "No such file or directory: '%s/factures'" % data_dir in str(error.value)
314

  
315
    # create temporary expected dir for invoices
316
    invoices_dir = os.path.join(data_dir, 'factures')
317
    os.mkdir(invoices_dir)
318
    call_command('import_orleans_data', data_dir=data_dir,
319
                 archive_name='family_data_orleans.zip',
303
    resource_invoices_dir = default_storage.path('family-%s/invoices' % resource.id)
304
    # cleanup previosly created invoices dir, if exists
305
    if os.path.exists(resource_invoices_dir) and os.path.isdir(resource_invoices_dir):
306
        shutil.rmtree(resource_invoices_dir)
307
        os.symlink(os.path.join(data_dir, 'factures'), resource_invoices_dir)
308

  
309
    call_command('import_orleans_data',
310
                 archive_file=os.path.join(data_dir, 'family_data_orleans.zip'),
320 311
                 connector='test-orleans')
321
    os.rmdir(invoices_dir)
312
    os.unlink(resource_invoices_dir)
313

  
314
    assert Family.objects.filter(resource=resource).count() == 18
315
    assert Adult.objects.all().count() == 31
316
    assert Child.objects.all().count() == 35
317
    assert Invoice.objects.filter(resource=resource).count() == 5
322 318

  
323
    assert Family.objects.filter(resource=resource).count() > 0
324
    assert Adult.objects.all().count() > 0
325
    assert Child.objects.all().count() > 0
326
    assert Invoice.objects.filter(resource=resource).count() > 0
327 319

  
328 320
def test_incorrect_orleans_data(caplog):
329 321
    filepath = os.path.join(os.path.dirname(__file__), 'data',
330
-