Projet

Général

Profil

0001-family-orleans-connector-uses-invoices-dir-path-1585.patch

Voir les différences:

Subject: [PATCH] family: orleans connector uses invoices dir path (#15855)

Instead of copying pdf files into the archive.
 passerelle/apps/family/loaders/concerto_orleans.py |  12 ++--
 .../management/commands/import_orleans_data.py     |  19 +++---
 tests/data/family_data_orleans.zip                 | Bin 29632 -> 0 bytes
 tests/data/orleans/factures/1602023571.pdf         |  49 +++++++++++++++
 tests/data/orleans/factures/1602025774.pdf         |  49 +++++++++++++++
 tests/data/orleans/factures/1602028362.pdf         |  49 +++++++++++++++
 tests/data/orleans/factures/1602031264.pdf         |  49 +++++++++++++++
 tests/data/orleans/factures/1602031791.pdf         |  49 +++++++++++++++
 tests/data/orleans/family_data_orleans.zip         | Bin 0 -> 3849 bytes
 tests/test_family.py                               |  67 ++++++++++++---------
 10 files changed, 300 insertions(+), 43 deletions(-)
 delete mode 100644 tests/data/family_data_orleans.zip
 create mode 100644 tests/data/orleans/factures/1602023571.pdf
 create mode 100644 tests/data/orleans/factures/1602025774.pdf
 create mode 100644 tests/data/orleans/factures/1602028362.pdf
 create mode 100644 tests/data/orleans/factures/1602031264.pdf
 create mode 100644 tests/data/orleans/factures/1602031791.pdf
 create mode 100644 tests/data/orleans/family_data_orleans.zip
passerelle/apps/family/loaders/concerto_orleans.py
22 22
from django.utils import timezone
23 23
from django.core.exceptions import ValidationError
24 24
from django.utils.translation import ugettext_lazy as _
25
from django.core.files.storage import DefaultStorage
25 26

  
26 27
from ..models import Family, Adult, Child, Invoice
27 28
from ..models import dict_cherry_pick, get_datetime
......
105 106

  
106 107
    def __init__(self, connector):
107 108
        self.connector = connector
108
        self.invoice_filename_pattern = 'invoices/facture_%s.pdf'
109 109

  
110 110
    def clean(self, archive):
111 111
        for filename in ('extract_prcit_personne.csv', 'extract_prcit_famille.csv',
......
167 167
                        external_id=child_data['external_id'], defaults=child_data)
168 168

  
169 169
                for invoice_data in family_data.get('invoices') or []:
170
                    invoice_filename = self.invoice_filename_pattern % invoice_data['external_id']
170
                    storage = DefaultStorage()
171
                    invoices_dir = storage.path('family-%s/invoices' % self.connector.id)
172
                    invoice_filename = '%s.pdf' % invoice_data['external_id']
173
                    invoice_path = os.path.join(invoices_dir, invoice_filename)
171 174
                    # create invoice object only if associated pdf exists
172
                    if invoice_filename in archive.namelist():
175
                    if os.path.exists(invoice_path):
173 176
                        invoice, created = Invoice.objects.update_or_create(resource=self.connector,
174
                                            family=family, external_id=invoice_data['external_id'], defaults=invoice_data)
175
                        invoice.write_pdf(archive.read(invoice_filename))
177
                            family=family, external_id=invoice_data['external_id'], defaults=invoice_data)
176 178

  
177 179
        except Exception, e:
178 180
            self.connector.logger.error('Error occured while importing data: %s', e)
passerelle/apps/family/management/commands/import_orleans_data.py
23 23

  
24 24
from django.core.files.base import File
25 25
from django.core.management.base import BaseCommand, CommandError
26
from django.core.files.storage import DefaultStorage
26 27

  
27
from family.models import GenericFamily
28

  
29
LOCK_FILENAME = os.path.join(tempfile.gettempdir(), 'import-orleans-data.lock')
28
from family.models import GenericFamily, Invoice
30 29

  
31 30

  
32 31
class Command(BaseCommand):
......
59 58
            connector = GenericFamily.objects.get(slug=options['connector'])
60 59
        except GenericFamily.DoesNotExist:
61 60
            return
61

  
62
        storage = DefaultStorage()
63
        LOCK_FILENAME = storage.path('family-%s/import-orleans-data.lock' % connector.id)
62 64
        try:
63 65
            fd = open(LOCK_FILENAME, 'w')
64 66
            fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB)
65
            invoices_dir = os.path.join(options['data_dir'], 'factures')
66
            with zipfile.ZipFile(archive_path, 'a') as archive:
67

  
68
                for invoice_file in os.listdir(invoices_dir):
69
                    if invoice_file.startswith('facture_'):
70
                        archive.write(os.path.join(invoices_dir, invoice_file),
71
                                      'invoices/%s' % invoice_file)
72
            connector.archive.save(options['archive_name'], File(file(archive_path)))
73 67
        except IOError:
74 68
            raise CommandError('Command already running.')
69

  
70
        try:
71
            connector.archive.save(options['archive_name'], File(file(archive_path)))
75 72
        except Exception, e:
76 73
            raise CommandError('Error occured: %s' % e)
77 74
        finally:
tests/data/orleans/factures/1602023571.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<1,_!Ke5Q^q;@(YOlVZU%!W]fE'CY~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/data/orleans/factures/1602025774.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 114 >>
25
stream
26
Gap@D0b-H6'Eri?4%tK_s4=Pk$'k'g5W0=>D1qd"OdA`to+ZJJ+(KoMBP'[R[\LS1fYV0Ep'UcQ<Z"6/6ntn8]$CiQ`JNnIn0N6ceb:65*kL;DpA~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001084 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1133
49
%%EOF
tests/data/orleans/factures/1602028362.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 114 >>
25
stream
26
Gap@D0a`Fb&-R>'J!a+coIc86/SJ^@+-<.LO@JmB#*FURAlb$LrKo?67_^A:Ylc#X!F(u:<CQ-/#%84^1A.rLqmgl,MAJL3kSrm,g@'Zp?FlgDo)~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001084 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1133
49
%%EOF
tests/data/orleans/factures/1602031264.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<2Dd0Fd8UCn;@(YOlVZU%!W]eq'CG~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/data/orleans/factures/1602031791.pdf
1
%PDF-1.3
2
%???? ReportLab Generated PDF document http://www.reportlab.com
3
1 0 obj
4
<< /F1 2 0 R >>
5
endobj
6
2 0 obj
7
<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
8
endobj
9
3 0 obj
10
<< /Contents 7 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 6 0 R /Resources << /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans <<  >> 
11
  /Type /Page >>
12
endobj
13
4 0 obj
14
<< /Outlines 8 0 R /PageMode /UseNone /Pages 6 0 R /Type /Catalog >>
15
endobj
16
5 0 obj
17
<< /Author (anonymous) /CreationDate (D:20161118142737-01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20161118142737-01'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
18
  /Subject (unspecified) /Title (untitled) /Trapped /False >>
19
endobj
20
6 0 obj
21
<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
22
endobj
23
7 0 obj
24
<< /Filter [ /ASCII85Decode /FlateDecode ] /Length 115 >>
25
stream
26
GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_TN+I0SHG`2GgV30Mi%s<R>\X9J.bG8l[Q%1GCR<2Dhm"e5Q^q;@(YOlVZU%!W]g$'Cu~>endstream
27
endobj
28
8 0 obj
29
<< /Count 0 /Type /Outlines >>
30
endobj
31
xref
32
0 9
33
0000000000 65535 f
34
0000000075 00000 n
35
0000000109 00000 n
36
0000000219 00000 n
37
0000000426 00000 n
38
0000000513 00000 n
39
0000000813 00000 n
40
0000000875 00000 n
41
0000001085 00000 n
42
trailer
43
<< /ID 
44
 % ReportLab generated PDF document -- digest (http://www.reportlab.com)
45
 [(\225\345b\271C#420}9\311\365k\266\254) (\225\345b\271C#420}9\311\365k\266\254)]
46
 /Info 5 0 R /Root 4 0 R /Size 9 >>
47
startxref
48
1134
49
%%EOF
tests/test_family.py
5 5
from uuid import uuid4
6 6
import zipfile
7 7
import logging
8
import shutil
8 9

  
9 10
from django.core.exceptions import ValidationError
10 11
from django.core.files import File
......
15 16
from django.core.files import File
16 17
from django.core.management import call_command
17 18
from django.core.management.base import CommandError
19
from django.core.files.storage import default_storage
18 20

  
19 21
from family.models import GenericFamily, Family, FamilyLink
20 22
from family.models import Invoice, Adult, Child, DATETIME_FORMAT
......
220 222
def test_orleans_concerto_loader():
221 223
    # all related objects will also be deleted
222 224
    Family.objects.all().delete()
223
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'family_data_orleans.zip')
225
    filepath = os.path.join(os.path.dirname(__file__), 'data', 'orleans',
226
                            'family_data_orleans.zip')
224 227
    resource = GenericFamily(title='test orleans',
225 228
            slug='test-orleans', archive=filepath, file_format='concerto_orleans')
226 229
    from family.loaders.concerto_orleans import Loader
......
274 277
    assert Family.objects.filter(resource=resource).count() == 18
275 278
    assert Adult.objects.all().count() == 31
276 279
    assert Child.objects.all().count() == 35
277
    assert Invoice.objects.filter(resource=resource).count() == 26
278
    assert Invoice.objects.filter(resource=resource, total_amount=0).count() == 2
279
    assert Invoice.objects.filter(online_payment=True).count() == 26
280
    assert Invoice.objects.filter(resource=resource).count() == 0
280 281

  
281 282
def test_orleans_data_import_command():
282 283
    with pytest.raises(CommandError) as error:
......
288 289
    assert str(error.value) == 'Directory /tmp/orleans_data does not exist.'
289 290

  
290 291
    cur_dir = os.path.dirname(__file__)
291
    data_dir = os.path.join(cur_dir, 'data')
292
    data_dir = os.path.join(cur_dir, 'data', 'orleans')
292 293
    with pytest.raises(CommandError) as error:
293 294
        call_command('import_orleans_data', data_dir=data_dir)
294 295
    assert str(error.value) == 'File %s/exports_prcit.zip does not exist.' % data_dir
......
297 298
                archive=os.path.join(data_dir, 'family_data_orleans.zip'),
298 299
                slug='test-orleans', file_format='concerto_orleans')
299 300

  
300
    # cleanup data before launching import
301
    Family.objects.filter(resource=resource).delete()
302
    Invoice.objects.filter(resource=resource).delete()
301
    resource_invoices_dir = default_storage.path('family-%s/invoices' % resource.id)
302
    if os.path.exists(resource_invoices_dir):
303
        shutil.rmtree(resource_invoices_dir)
304
        os.symlink(os.path.join(data_dir, 'factures'), resource_invoices_dir)
305

  
306
    try:
307
        # cleanup data before launching import
308
        Family.objects.filter(resource=resource).delete()
309
        Invoice.objects.filter(resource=resource).delete()
310

  
311
        call_command('import_orleans_data', data_dir=data_dir,
312
                     archive_name='family_data_orleans.zip')
313
        assert Family.objects.filter(resource=resource).count() == 0
314
        assert Invoice.objects.filter(resource=resource).count() == 0
315

  
316
        non_existing_data_dir = os.path.join(cur_dir, 'nonexisting')
317
        with pytest.raises(CommandError) as error:
318
            call_command('import_orleans_data', data_dir=non_existing_data_dir,
319
                         archive_name='family_data_orleans.zip',
320
                         connector='test-orleans')
321
        assert 'Directory %s does not exist' % non_existing_data_dir in str(error.value)
322

  
323
        Family.objects.filter(resource=resource).delete()
324
        Invoice.objects.filter(resource=resource).delete()
325

  
303 326

  
304
    call_command('import_orleans_data', data_dir=data_dir,
305
                 archive_name='family_data_orleans.zip')
306
    assert Family.objects.filter(resource=resource).count() == 0
307
    assert Invoice.objects.filter(resource=resource).count() == 0
308 327

  
309
    with pytest.raises(Exception) as error:
310 328
        call_command('import_orleans_data', data_dir=data_dir,
311 329
                     archive_name='family_data_orleans.zip',
312 330
                     connector='test-orleans')
313
    assert "No such file or directory: '%s/factures'" % data_dir in str(error.value)
314

  
315
    # create temporary expected dir for invoices
316
    invoices_dir = os.path.join(data_dir, 'factures')
317
    os.mkdir(invoices_dir)
318
    call_command('import_orleans_data', data_dir=data_dir,
319
                 archive_name='family_data_orleans.zip',
320
                 connector='test-orleans')
321
    os.rmdir(invoices_dir)
322

  
323
    assert Family.objects.filter(resource=resource).count() > 0
324
    assert Adult.objects.all().count() > 0
325
    assert Child.objects.all().count() > 0
326
    assert Invoice.objects.filter(resource=resource).count() > 0
331

  
332
        assert Family.objects.filter(resource=resource).count() == 18
333
        assert Adult.objects.all().count() == 31
334
        assert Child.objects.all().count() == 35
335
        assert Invoice.objects.filter(resource=resource).count() == 5
336
    except:
337
        pass
338
    finally:
339
        os.unlink(resource_invoices_dir)
327 340

  
328 341
def test_incorrect_orleans_data(caplog):
329 342
    filepath = os.path.join(os.path.dirname(__file__), 'data',
330
-