Projet

Général

Profil

0002-rewrite-file-validation-fixes-10444.patch

Benjamin Dauvergne, 30 mars 2016 11:01

Télécharger (46,4 ko)

Voir les différences:

Subject: [PATCH 2/2] rewrite file validation (fixes #10444)

- metadata fields are now directly added to the form, as part of the
  FileWithPreviewWidget sub-widgets (it's a CompositeWidget)
- existing validated metadatas are proposed for prefilling
- if prefilled no file is uploadee, and a special upload class NoUpload is used
  instead of a PicklableUpload, it only contains the metadatas.
- prefilled fields are disabled,
- on upload of a new file all prefilled fields are emptied and re-enabled.
- rewrittent tests around file validation
 tests/test_backoffice_pages.py            | 250 +++++++++++++++++++++++++-----
 tests/test_form_pages.py                  | 110 +++++++++----
 tests/test_formdata.py                    | 109 +++++++++++++
 wcs/backoffice/management.py              |   2 +-
 wcs/fields.py                             |  17 +-
 wcs/file_validation.py                    | 114 +++++++-------
 wcs/forms/common.py                       |  60 ++++---
 wcs/qommon/form.py                        |  80 ++++++++--
 wcs/qommon/static/css/qommon.css          |  10 ++
 wcs/qommon/static/js/qommon.fileupload.js |  45 +++++-
 10 files changed, 638 insertions(+), 159 deletions(-)
tests/test_backoffice_pages.py
15 15
from quixote import cleanup, get_publisher
16 16
from wcs.qommon import errors, sessions
17 17
from qommon.ident.password_accounts import PasswordAccount
18
from qommon.misc import json_loads
18 19
from wcs.qommon.http_request import HTTPRequest
19 20
from wcs.roles import Role
20 21
from wcs.workflows import (Workflow, CommentableWorkflowStatusItem,
......
1577 1578
    assert resp.body == 'foo(%s);' % menu_json_str
1578 1579
    assert resp.headers['content-type'] == 'application/javascript'
1579 1580

  
1581
def test_backoffice_file_field_fargo_no_metadata(pub, fargo_url):
1582
    document_type = {
1583
        'id': 'justificatif-de-domicile',
1584
        'fargo': True,
1585
        'mimetypes': ['application/pdf'],
1586
        'label': 'Justificatif de domicile',
1587
    }
1588
    user = create_user(pub, is_admin=True)
1589
    user.name_identifiers = ['12345']
1590
    user.store()
1591
    FormDef.wipe()
1592
    formdef = FormDef()
1593
    formdef.name = 'form title'
1594
    formdef.fields = [fields.FileField(
1595
        id='0', label='1st field', type='file',
1596
        document_type=document_type)]
1597
    formdef.store()
1598
    formdef.data_class().wipe()
1599
    upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf')
1600
    digest = hashlib.sha256('%PDF-1.4').hexdigest()
1601
    app = login(get_app(pub))
1602
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
1603
        resp = app.get('/form-title/')
1604
        assert fargo_get.call_count == 0
1605
    resp.forms[0]['f0$file'] = upload
1606
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
1607
        resp = resp.forms[0].submit('submit')
1608
        assert fargo_get.call_count == 0
1609
    assert 'Check values then click submit.' in resp.body
1610
    resp = resp.forms[0].submit('submit').follow()
1611
    assert formdef.data_class().count() == 1
1612
    formdata = formdef.data_class().select()[0]
1613
    form_id = formdata.id
1614
    assert not hasattr(formdata.data['0'], 'metadata')
1615
    assert not '0_structured' in formdata.data
1616
    resp = app.get('/backoffice/management/form-title/%s/' % form_id)
1617
    assert not 'Validate' in resp.body
1618
    with mock.patch('wcs.file_validation.http_post_request') as http_post_request:
1619
        resp = app.get('/backoffice/management/form-title/%s/validate?field_id=0' % form_id)
1620
        assert http_post_request.call_count == 0
1621
    resp = resp.follow()
1622
    assert not 'Valid ' in resp.body
1623
    assert not 'Validate' in resp.body
1624

  
1625

  
1580 1626
def test_backoffice_file_field_validation(pub, fargo_url):
1627
    document_type = {
1628
        'id': 'justificatif-de-domicile',
1629
        'fargo': True,
1630
        'mimetypes': ['application/pdf'],
1631
        'label': 'Justificatif de domicile',
1632
        'metadata': [
1633
            {'label': 'Nom', 'name': 'nom'},
1634
            {'label': 'Prénom(s)', 'name': 'prenoms'},
1635
            {'label': 'Numéro', 'name': 'numero'},
1636
            {'label': 'Rue', 'name': 'rue'},
1637
            {'label': 'Code postal', 'name': 'code-postal'},
1638
            {'label': 'Ville', 'name': 'ville'},
1639
        ],
1640
    }
1641
    metadata = {
1642
        'nom': 'Doe',
1643
        'prenoms': 'John',
1644
        'numero': '169',
1645
        'rue': 'rue du château',
1646
        'code-postal': '75014',
1647
        'ville': 'PARIS',
1648
    }
1581 1649
    user = create_user(pub, is_admin=True)
1582 1650
    user.name_identifiers = ['12345']
1583 1651
    user.store()
......
1586 1654
    formdef.name = 'form title'
1587 1655
    formdef.fields = [fields.FileField(
1588 1656
        id='0', label='1st field', type='file',
1589
        document_type={
1590
            'id': 'justificatif-de-domicile',
1591
            'fargo': True,
1592
            'mimetypes': ['application/pdf'],
1593
            'label': 'PDF files',
1594
        })
1595
    ]
1657
        document_type=document_type)]
1596 1658
    formdef.store()
1597 1659
    formdef.data_class().wipe()
1598 1660
    upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf')
1599 1661
    digest = hashlib.sha256('%PDF-1.4').hexdigest()
1600 1662
    app = login(get_app(pub))
1601
    resp = app.get('/form-title/')
1663
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
1664
        fargo_get.return_value = {'result': 1, 'data': {'results': []}}
1665
        resp = app.get('/form-title/')
1666
        fargo_get.assert_called_once_with(
1667
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
1602 1668
    resp.forms[0]['f0$file'] = upload
1603
    resp = resp.forms[0].submit('submit')
1669
    for key in metadata:
1670
        resp.forms[0]['f0$%s' % key] = metadata[key]
1671
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
1672
        fargo_get.return_value = {'result': 1, 'data': {'results': []}}
1673
        resp = resp.forms[0].submit('submit')
1674
        fargo_get.assert_called_once_with(
1675
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
1676
    for key in metadata:
1677
        assert 'value="%s"' % metadata[key] in resp.body
1604 1678
    assert 'Check values then click submit.' in resp.body
1605
    resp = resp.forms[0].submit('submit')
1606
    assert resp.status_int == 302
1679
    resp = resp.forms[0].submit('submit').follow()
1680
    for metadata_field in document_type['metadata']:
1681

  
1682
        fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']])
1683
        assert fragment in resp.body
1607 1684
    assert formdef.data_class().count() == 1
1608
    form_id = formdef.data_class().select()[0].id
1609
    with mock.patch('wcs.file_validation.http_get_page') as http_get_page:
1610
        json_response = json.dumps({
1611
            'err': 0,
1612
            'data': {
1613
                'type': 'justificatif-de-domicile',
1614
                'label': 'Justificatif de domicile',
1615
                'creator': 'Jean Bono',
1616
                'created': '2014-01-01T01:01:01',
1617
                'start': '2014-01-01T01:01:01',
1618
                'end': '2014-01-01T01:01:01',
1619
                'metadata': [{
1620
                        'name': 'code_postal',
1621
                        'label': 'Code postal',
1622
                        'value': '13400',
1623
                }],
1624
            },
1685
    formdata = formdef.data_class().select()[0]
1686
    form_id = formdata.id
1687
    assert formdata.data['0'].metadata == metadata
1688
    assert formdata.data['0_structured'] == metadata
1689
    resp = app.get('/backoffice/management/form-title/%s/' % form_id)
1690
    assert 'Validate' in resp.body
1691
    for metadata_field in document_type['metadata']:
1692

  
1693
        fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']])
1694
        assert fragment in resp.body
1695
    with mock.patch('wcs.file_validation.http_post_request') as http_post_request:
1696
        payload = {
1697
            'user_nameid': '12345',
1698
            'origin': 'example.net',
1699
            'creator': 'admin',
1700
            'content_hash': digest,
1701
        }
1702
        payload.update(metadata)
1703
        result = {
1704
            'result': 1,
1705
            'data': payload.copy()
1706
        }
1707
        result['data'].update({
1708
            'url': 'zob',
1709
            'created': '1970-01-01T10:10:10',
1710
            'start': '1970-01-01',
1711
            'end': '1978-01-01',
1712
            'display': 'John Doe, 169 rue du château, 75014 PARIS'
1625 1713
        })
1626
        http_get_page.return_value = None, 200, json_response, None
1627
        resp = app.get('/backoffice/management/form-title/%s/' % form_id)
1628
        http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/%s/justificatif-de-domicile/' % digest)
1629
        assert 'class="value valid"' in resp.body
1630
        assert 'Justificatif de domicile validated by Jean Bono on 2014-01-01T01:01:01' in resp.body
1631
        assert 'Code postal: 13400' in resp.body
1632
        assert 'Valid from 2014-01-01T01:01:01 to 2014-01-01T01:01:01' in resp.body
1714
        http_post_request.return_value = None, 201, json.dumps(result), None
1715
        resp = app.get('/backoffice/management/form-title/%s/validate?field_id=0' % form_id)
1716
        assert http_post_request.call_count == 1
1717
        assert http_post_request.call_args[0][0] == 'http://fargo.example.net/api/validation/justificatif-de-domicile/'
1718
        assert json_loads(http_post_request.call_args[0][1]) == payload
1719
        assert http_post_request.call_args[1] == {'headers': {'Content-Type': 'application/json'}}
1720
    resp = resp.follow()
1721

  
1722
    assert 'Valid from 1970-01-01 to 1978-01-01' in resp.body
1723

  
1724

  
1725
def test_backoffice_file_validation_no_upload(pub, fargo_url):
1726
    document_type = {
1727
        'id': 'justificatif-de-domicile',
1728
        'fargo': True,
1729
        'mimetypes': ['application/pdf'],
1730
        'label': 'Justificatif de domicile',
1731
        'metadata': [
1732
            {'label': 'Nom', 'name': 'nom'},
1733
            {'label': 'Prénom(s)', 'name': 'prenoms'},
1734
            {'label': 'Numéro', 'name': 'numero'},
1735
            {'label': 'Rue', 'name': 'rue'},
1736
            {'label': 'Code postal', 'name': 'code-postal'},
1737
            {'label': 'Ville', 'name': 'ville'},
1738
        ],
1739
    }
1740
    metadata = {
1741
        'nom': 'Doe',
1742
        'prenoms': 'John',
1743
        'numero': '169',
1744
        'rue': 'rue du château',
1745
        'code-postal': '75014',
1746
        'ville': 'PARIS',
1747
    }
1748
    validation = {
1749
        'url': 'zob',
1750
        'creator': 'admin',
1751
        'created': '1970-01-01T10:10:10',
1752
        'start': '1970-01-01',
1753
        'end': '1978-01-01',
1754
        'display': 'John Doe, 169 rue du château, 75014 PARIS'
1755
    }
1756
    validation.update(metadata)
1757

  
1758
    user = create_user(pub, is_admin=True)
1759
    user.name_identifiers = ['12345']
1760
    user.store()
1761
    FormDef.wipe()
1762
    formdef = FormDef()
1763
    formdef.name = 'form title'
1764
    formdef.fields = [fields.FileField(
1765
        id='0', label='1st field', type='file',
1766
        document_type=document_type)]
1767
    formdef.store()
1768
    formdef.data_class().wipe()
1769
    app = login(get_app(pub))
1770
    return_value = {'result': 1, 'data': {'results': [validation]}}
1771
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
1772
        fargo_get.return_value = return_value
1773
        resp = app.get('/form-title/')
1774
        fargo_get.assert_called_once_with(
1775
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
1776
        assert validation['display'] in resp.body
1777
    resp.forms[0]['f0$validation_url'] = 'zob'
1778
    for key in metadata:
1779
        resp.forms[0]['f0$%s' % key] = metadata[key]
1780
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get, \
1781
            mock.patch('wcs.file_validation.http_get_page') as http_get_page:
1782
        fargo_get.return_value = return_value
1783
        return_value = {
1784
            'result': 1,
1785
            'data': validation,
1786
        }
1787
        http_get_page.return_value = None, 200, json.dumps(return_value), None
1788
        resp = resp.forms[0].submit('submit')
1789
        fargo_get.assert_called_once_with(
1790
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
1791
        http_get_page.assert_called_with('zob')
1792
        for key in metadata:
1793
            assert 'value="%s"' % metadata[key] in resp.body
1794
        assert 'Check values then click submit.' in resp.body
1795
        resp = resp.forms[0].submit('submit').follow()
1796
    assert formdef.data_class().count() == 1
1797
    formdata = formdef.data_class().select()[0]
1798
    form_id = formdata.id
1799
    assert formdata.data['0'].metadata == validation
1800
    assert formdata.data['0_structured'] == validation
1801
    for metadata_field in document_type['metadata']:
1802

  
1803
        fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']])
1804
        assert fragment in resp.body
1805
    resp = app.get('/backoffice/management/form-title/%s/' % form_id)
1806
    assert not 'Validate' in resp.body
1807
    for metadata_field in document_type['metadata']:
1808

  
1809
        fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']])
1810
        assert fragment in resp.body
1811
    assert 'Valid from 1970-01-01 to 1978-01-01' in resp.body
1812

  
1633 1813

  
1634 1814
def test_360_user_view(pub):
1635 1815
    if not pub.is_using_postgresql():
tests/test_form_pages.py
1
# -*- coding: utf-8 -*-
1 2
import json
2 3
import pytest
3 4
import hashlib
......
2288 2289
    assert formdef.data_class().select()[0].data['1'] == 'foobar3'
2289 2290

  
2290 2291
def test_file_field_validation(pub, fargo_url):
2292
    document_type = {
2293
        'id': 'justificatif-de-domicile',
2294
        'fargo': True,
2295
        'label': 'Justificatif de domicile',
2296
        'metadata': [
2297
            {'label': 'Nom', 'name': 'nom'},
2298
            {'label': 'Prénom(s)', 'name': 'prenoms'},
2299
            {'label': 'Numéro', 'name': 'numero'},
2300
            {'label': 'Rue', 'name': 'rue'},
2301
            {'label': 'Code postal', 'name': 'code-postal'},
2302
            {'label': 'Ville', 'name': 'ville'},
2303
        ],
2304
    }
2305
    metadata = {
2306
        'nom': 'Doe',
2307
        'prenoms': 'John',
2308
        'numero': '169',
2309
        'rue': 'rue du château',
2310
        'code-postal': '75014',
2311
        'ville': 'PARIS',
2312
    }
2291 2313
    user = create_user(pub)
2292 2314
    user.name_identifiers = ['12345']
2293 2315
    user.store()
......
2296 2318
    formdef.name = 'form title'
2297 2319
    formdef.fields = [fields.FileField(
2298 2320
        id='0', label='1st field', type='file',
2299
        document_type={
2300
            'id': 'justificatif-de-domicile',
2301
            'fargo': True,
2302
            'mimetypes': ['application/pdf'],
2303
            'label': 'PDF files',
2304
        })
2321
        document_type=document_type)
2305 2322
    ]
2306 2323
    formdef.store()
2307 2324
    formdef.data_class().wipe()
2308 2325
    upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf')
2309
    digest = hashlib.sha256('%PDF-1.4').hexdigest()
2310 2326
    app = login(get_app(pub), username='foo', password='foo')
2311
    resp = app.get('/form-title/')
2327
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
2328
        fargo_get.return_value = {'result': 1, 'data': {'results': []}}
2329
        resp = app.get('/form-title/')
2330
        fargo_get.assert_called_once_with(
2331
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
2312 2332
    resp.forms[0]['f0$file'] = upload
2333
    for key in metadata:
2334
        resp.forms[0]['f0$%s' % key] = metadata[key]
2335
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
2336
        fargo_get.return_value = {'result': 1, 'data': {'results': []}}
2337
        resp = resp.forms[0].submit('submit')
2338
        fargo_get.assert_called_once_with(
2339
            'api/validation/justificatif-de-domicile/?user_nameid=12345')
2340
    assert 'Check values then click submit.' in resp.body
2313 2341
    resp = resp.forms[0].submit('submit')
2342
    assert resp.status_int == 302
2343
    resp = resp.follow()
2344
    assert 'The form has been recorded' in resp.body
2345
    assert formdef.data_class().count() == 1
2346
    formdata = formdef.data_class().select()[0]
2347
    assert formdata.data['0'].metadata == metadata
2348
    assert formdata.data['0_structured'] == metadata
2349

  
2350

  
2351
def test_file_field_fargo_no_metadata(pub, fargo_url):
2352
    document_type = {
2353
        'id': 'justificatif-de-domicile',
2354
        'fargo': True,
2355
        'label': 'Justificatif de domicile',
2356
    }
2357
    user = create_user(pub)
2358
    user.name_identifiers = ['12345']
2359
    user.store()
2360
    FormDef.wipe()
2361
    formdef = FormDef()
2362
    formdef.name = 'form title'
2363
    formdef.fields = [fields.FileField(
2364
        id='0', label='1st field', type='file',
2365
        document_type=document_type)
2366
    ]
2367
    formdef.store()
2368
    formdef.data_class().wipe()
2369
    upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf')
2370
    app = login(get_app(pub), username='foo', password='foo')
2371
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
2372
        resp = app.get('/form-title/')
2373
        assert fargo_get.call_count == 0
2374
    resp.forms[0]['f0$file'] = upload
2375
    with mock.patch('wcs.file_validation.fargo_get') as fargo_get:
2376
        resp = resp.forms[0].submit('submit')
2377
        assert fargo_get.call_count == 0
2314 2378
    assert 'Check values then click submit.' in resp.body
2315 2379
    resp = resp.forms[0].submit('submit')
2316 2380
    assert resp.status_int == 302
2317
    with mock.patch('wcs.file_validation.http_get_page') as http_get_page:
2318
        json_response = json.dumps({
2319
            'err': 0,
2320
            'data': {
2321
                'type': 'justificatif-de-domicile',
2322
                'label': 'Justificatif de domicile',
2323
                'creator': 'Jean Bono',
2324
                'created': '2014-01-01T01:01:01',
2325
                'start': '2014-01-01T01:01:01',
2326
                'end': '2014-01-01T01:01:01',
2327
                'metadata': [{
2328
                        'name': 'code_postal',
2329
                        'label': 'Code postal',
2330
                        'value': '13400',
2331
                }],
2332
            },
2333
        })
2334
        http_get_page.return_value = None, 200, json_response, None
2335
        resp = resp.follow()
2336
        http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/%s/justificatif-de-domicile/' % digest)
2337
        http_get_page.reset_mock()
2338
        assert 'The form has been recorded' in resp.body
2339
        assert 'class="value valid"' in resp.body
2381
    resp = resp.follow()
2382
    assert 'The form has been recorded' in resp.body
2383
    assert formdef.data_class().count() == 1
2384
    formdata = formdef.data_class().select()[0]
2385
    assert not hasattr(formdata.data['0'], 'metadata')
2386
    assert not '0_structured' in formdata.data
2387

  
2340 2388

  
2341 2389
def test_form_string_field_autocomplete(pub):
2342 2390
    formdef = create_formdef()
tests/test_formdata.py
1
# -*- coding: utf-8 -*-
1 2
import pytest
2 3
import sys
3 4
import shutil
......
11 12
from wcs.formdata import Evolution
12 13
from wcs.workflows import Workflow, WorkflowCriticalityLevel
13 14
from wcs.wf.anonymise import AnonymiseWorkflowStatusItem
15
from wcs.qommon.form import NoUpload
16
import mock
14 17

  
15 18
from utilities import create_temporary_pub, clean_temporary_pub
16 19

  
......
148 151
    assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0')
149 152
    assert isinstance(substvars.get('form_var_foo_raw'), Upload)
150 153

  
154

  
155
def test_file_field_fargo_no_metadata(pub):
156
    document_types = {
157
        'justificatif-de-domicile': {
158
            'id': 'justificatif-de-domicile',
159
            'fargo': True,
160
            'label': 'Justificatif de domicile',
161
        }
162
    }
163
    formdef.data_class().wipe()
164
    formdef.fields = [fields.FileField(id='0', label='file', varname='foo',
165
                                       document_type=document_types['justificatif-de-domicile'])]
166
    formdef.store()
167
    formdata = formdef.data_class()()
168
    upload = Upload('test.txt', 'text/plain', 'ascii')
169
    upload.receive(['first line', 'second line'])
170
    formdata.data = {'0': upload}
171
    formdata.id = 1
172
    substvars = formdata.get_substitution_variables()
173
    assert substvars.get('form_var_foo') == 'test.txt'
174
    assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0')
175
    assert isinstance(substvars.get('form_var_foo_raw'), Upload)
176

  
177

  
178
def test_file_field_with_metadata(pub):
179
    document_types = {
180
        'justificatif-de-domicile': {
181
            'id': 'justificatif-de-domicile',
182
            'fargo': True,
183
            'label': 'Justificatif de domicile',
184
            'metadata': [
185
                {'label': 'Nom', 'name': 'nom'},
186
                {'label': 'Prénom(s)', 'name': 'prenoms'},
187
                {'label': 'Numéro', 'name': 'numero'},
188
                {'label': 'Rue', 'name': 'rue'},
189
                {'label': 'Code postal', 'name': 'code-postal'},
190
                {'label': 'Ville', 'name': 'ville'},
191
            ],
192
        }
193
    }
194
    formdef.data_class().wipe()
195
    formdef.fields = [fields.FileField(id='0', label='file', varname='foo',
196
                                       document_type=document_types['justificatif-de-domicile'])]
197
    formdef.store()
198
    formdata = formdef.data_class()()
199
    upload = Upload('test.txt', 'text/plain', 'ascii')
200
    upload.receive(['first line', 'second line'])
201
    upload.metadata = {
202
        'nom': 'Doe',
203
        'prenoms': 'John',
204
        'numero': '169',
205
        'rue': 'rue du château',
206
        'code-postal': '75014',
207
        'ville': 'PARIS',
208
    }
209
    formdata.data = {'0': upload, '0_structured': upload.metadata}
210
    formdata.id = 1
211
    substvars = formdata.get_substitution_variables()
212
    assert substvars.get('form_var_foo') == 'test.txt'
213
    assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0')
214
    assert isinstance(substvars.get('form_var_foo_raw'), Upload)
215
    for key in upload.metadata:
216
        assert substvars.get('form_var_foo_%s' % key) == upload.metadata[key]
217

  
218

  
219
def test_file_field_no_file_with_metadata(pub):
220
    document_types = {
221
        'justificatif-de-domicile': {
222
            'id': 'justificatif-de-domicile',
223
            'fargo': True,
224
            'label': 'Justificatif de domicile',
225
            'metadata': [
226
                {'label': 'Nom', 'name': 'nom'},
227
                {'label': 'Prénom(s)', 'name': 'prenoms'},
228
                {'label': 'Numéro', 'name': 'numero'},
229
                {'label': 'Rue', 'name': 'rue'},
230
                {'label': 'Code postal', 'name': 'code-postal'},
231
                {'label': 'Ville', 'name': 'ville'},
232
            ],
233
        }
234
    }
235
    formdef.data_class().wipe()
236
    formdef.fields = [fields.FileField(id='0', label='file', varname='foo',
237
                                       document_type=document_types['justificatif-de-domicile'])]
238
    formdef.store()
239
    formdata = formdef.data_class()()
240
    metadata = {
241
        'nom': 'Doe',
242
        'prenoms': 'John',
243
        'numero': '169',
244
        'rue': 'rue du château',
245
        'code-postal': '75014',
246
        'ville': 'PARIS',
247
    }
248
    with mock.patch('wcs.file_validation.get_validation', return_value=metadata):
249
        upload = NoUpload('http://whatever.com/')
250
    formdata.data = {'0': upload, '0_structured': upload.metadata}
251
    formdata.id = 1
252
    substvars = formdata.get_substitution_variables()
253
    assert isinstance(substvars.get('form_var_foo'), NoUpload)
254
    assert not 'form_var_foo_url' in substvars
255
    assert not 'form_var_foo_raw' in substvars
256
    for key in upload.metadata:
257
        assert substvars.get('form_var_foo_%s' % key) == upload.metadata[key]
258

  
259

  
151 260
def test_get_submitter(pub):
152 261
    formdef.data_class().wipe()
153 262
    formdef.fields = [fields.StringField(id='0', label='email', varname='foo',
wcs/backoffice/management.py
1623 1623

  
1624 1624

  
1625 1625
class FormBackOfficeStatusPage(FormStatusPage):
1626
    _q_exports = ['', 'download', 'json', 'action', 'inspect']
1626
    _q_exports = ['', 'download', 'json', 'action', 'inspect', 'validate']
1627 1627
    form_page_class = FormFillPage
1628 1628

  
1629 1629
    def html_top(self, title = None):
wcs/fields.py
720 720
        self.document_type = self.document_type or {}
721 721

  
722 722
    @property
723
    def metadata(self):
724
        return self.document_type.get('metadata', [])
725

  
726
    @property
723 727
    def file_type(self):
724 728
        return (self.document_type or {}).get('mimetypes', [])
725 729

  
......
752 756
                'document_type', 'max_file_size', 'allow_portfolio_picking']
753 757

  
754 758
    def get_view_value(self, value):
755
        return htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % (
759
        r = TemplateIO(html=True)
760
        if not getattr(value, 'no_file', False):
761
            r += htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % (
756 762
                value.base_filename, self.id, value)
763
        for meta_field in self.metadata:
764
            metadata_value = getattr(value, 'metadata', {}).get(meta_field['name'], '')
765
            r += htmltext('<p>%s&nbsp;: %s</p>') % (meta_field['label'], metadata_value)
766
        return r.getvalue()
757 767

  
758 768
    def get_csv_value(self, value, hint=None):
759 769
        if not value:
......
788 798
            value = get_request().get_field(self.field_key)
789 799
            if value and hasattr(value, 'token'):
790 800
                get_request().form[self.field_key + '$token'] = value.token
801
        kwargs['document_type'] = self.document_type
791 802

  
792 803
    def get_document_types(self):
793 804
        document_types = {
......
867 878
        if self.document_type and self.document_type.get('fargo'):
868 879
            self.document_type['fargo'] = self.document_type['fargo'] == 'True'
869 880

  
881
    def store_structured_value(self, data, field_id):
882
        value = data.get(field_id)
883
        return getattr(value, 'metadata', {})
884

  
870 885

  
871 886
register_field_class(FileField)
872 887

  
wcs/file_validation.py
19 19
import hashlib
20 20
import urllib
21 21

  
22
from qommon.misc import http_get_page, json_loads
23
from quixote import get_publisher, get_response
24
from quixote.html import htmltext
22
from qommon.misc import http_get_page, json_loads, http_post_request
23
from quixote import get_publisher, get_request
25 24

  
26 25

  
27 26
def has_file_validation():
28 27
    return get_publisher().get_site_option('fargo_url') is not None
29 28

  
29

  
30 30
def fargo_get(path):
31 31
    fargo_url = get_publisher().get_site_option('fargo_url')
32 32
    url = urlparse.urljoin(fargo_url, path)
......
35 35
        return json_loads(data)
36 36
    return None
37 37

  
38

  
38 39
def sha256_of_upload(upload):
39 40
    return hashlib.sha256(upload.get_content()).hexdigest()
40 41

  
42

  
41 43
def get_document_types():
42 44
    if not has_file_validation():
43 45
        return {}
44 46
    response = fargo_get('/document-types/')
45
    publisher = get_publisher()
46 47
    if response.get('err') == 0:
47 48
        result = {}
48 49
        for schema in response['data']:
......
52 53
                'fargo': True,
53 54
            }
54 55
            if 'mimetypes' in schema:
55
                d['mimetypes'] = shema['mimetypes']
56
                d['mimetypes'] = schema['mimetypes']
56 57
            result[d['id']] = d
57

  
58
            d['metadata'] = schema.get('metadata', [])
58 59
        return result
59 60
    return {}
60 61

  
61
def validation_path(filled, field, upload):
62
    user = filled.get_user()
63
    if not user:
64
        return None
65
    if not user.name_identifiers:
66
        return None
67
    if not field.document_type or not field.document_type.get('fargo'):
68
        return None
69
    name_id = user.name_identifiers[0]
70
    sha_256 = sha256_of_upload(upload)
71
    document_type = field.document_type['id']
72
    path = '%s/%s/%s/' % (
73
        urllib.quote(name_id),
74
        urllib.quote(sha_256),
75
        urllib.quote(document_type),
76
    )
77
    return path
78

  
79
def validate_upload(filled, field, upload):
62

  
63
def get_validation(url):
64
    response, status, data, auth_header = http_get_page(url)
65
    if status == 200:
66
        return json_loads(data)['data']
67
    return None
68

  
69

  
70
def get_validations(document_type):
71
    request = get_request()
72
    document_type_id = document_type['id']
73
    qs = {}
74
    if not request.user:
75
        return []
76
    if request.user.name_identifiers:
77
        qs['user_nameid'] = request.user.name_identifiers[0]
78
    elif request.user.email:
79
        qs['user_email'] = request.user.email
80
    else:
81
        return []
82
    path = 'api/validation/%s/' % urllib.quote(document_type_id)
83
    validations = fargo_get(path + '?%s' % urllib.urlencode(qs))
84
    if validations and validations.get('data', {}).get('results', []):
85
        return validations['data']['results']
86
    return []
87

  
88

  
89
def is_valid(filled, field, upload):
80 90
    '''Check validation of the uploaded file with Fargo'''
81
    path = validation_path(filled, field, upload)
82
    if not path:
83
        return None
84
    response = fargo_get('metadata/' + path)
85
    if response is None:
86
        return None
87
    if response['err'] == 1:
88
        return False
89
    return response['data']
90

  
91
def get_document_type_label(document_type):
92
    return get_document_types().get(document_type, {}).get('label')
93

  
94
def validation_link(filled, field, upload):
91
    return 'url' in getattr(upload, 'metadata', {})
92

  
93

  
94
def validate(filled, field, upload):
95 95
    '''Compute link to Fargo to validate the given document'''
96
    path = validation_path(filled, field, upload)
97
    if not path:
98
        return ''
99
    get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css')
100
    get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js'])
101
    label = get_document_type_label(field.document_type['id'])
96
    document_type_id = field.document_type['id']
97
    path = 'api/validation/%s/' % urllib.quote(document_type_id)
102 98
    fargo_url = get_publisher().get_site_option('fargo_url')
103
    url = urlparse.urljoin(fargo_url, 'validation/' + path)
104
    next_url = get_publisher().get_frontoffice_url() + '/reload-top'
105
    url += '?next=%s' % urllib.quote(next_url)
106
    title = _('Validate as a %s') % label
107
    return htmltext('<a data-title="%(title)s" data-width="800" '
108
                    'data-height="500" href="%(url)s">%(title)s</a>') % {
109
        'title': title,
110
        'url': url,
111
    }
99
    url = urlparse.urljoin(fargo_url, path)
100
    payload = {}
101
    if filled.user:
102
        if filled.user.name_identifiers:
103
            payload['user_nameid'] = filled.user.name_identifiers[0]
104
        else:
105
            payload['user_email'] = filled.user.email
106
    payload['origin'] = get_request().get_server()
107
    payload['creator'] = get_request().user.display_name
108
    payload['content_hash'] = sha256_of_upload(upload)
109
    for meta_field in field.metadata:
110
        payload[meta_field['name']] = upload.metadata.get(meta_field['name'], '')
111
    headers = {'Content-Type': 'application/json'}
112
    response, status, response_payload, auth_header = http_post_request(url, json.dumps(payload),
113
                                                                        headers=headers)
114
    if status == 201:
115
        upload.metadata = json_loads(response_payload)['data']
116
        filled.data['%s_structured' % field.id] = upload.metadata
117
        filled.store()
wcs/forms/common.py
97 97
    def html_top(self, title = None):
98 98
        template.html_top(title = title, default_org = _('Forms'))
99 99

  
100
    def validate(self):
101
        if not file_validation.has_file_validation():
102
            return redirect('.')
103
        field_id = get_request().form.get('field_id')
104
        if not field_id:
105
            return redirect('.')
106
        for field in self.formdef.fields:
107
            if field.id == field_id:
108
                break
109
        else:
110
            return redirect('.')
111
        if field.key != 'file':
112
            return redirect('.')
113
        if not field.document_type.get('fargo', False):
114
            return redirect('.')
115
        if not field.document_type.get('metadata', []):
116
            return redirect('.')
117
        value = self.filled.data.get(field_id)
118
        file_validation.validate(self.filled, field, value)
119
        return redirect('.')
120

  
100 121
    def __init__(self, formdef, filled, register_workflow_subdirs=True):
101 122
        get_publisher().substitutions.feed(filled)
102 123
        self.formdef = formdef
......
612 633

  
613 634
    def display_file_field(self, form_url, field, value):
614 635
        r = TemplateIO(html=True)
615
        status = None
616
        if file_validation.has_file_validation():
617
            status = file_validation.validate_upload(self.filled, field, value)
618
            if status is False:
636
        validated = None
637
        is_fargo_dt = field.document_type.get('fargo', False)
638
        has_metadata = bool(field.document_type.get('metadata', []))
639
        if file_validation.has_file_validation() and is_fargo_dt and has_metadata:
640
            validated = file_validation.is_valid(self.filled, field, value)
641
            if validated is False:
619 642
                extra_class = ' invalid'
620
            elif status is None:
621
                extra_class = ''
622 643
            else:
623 644
                extra_class = ' valid'
624 645
            r += htmltext('<div class="value%s">' % extra_class)
......
627 648
        s = field.get_view_value(value)
628 649
        s = s.replace(str('[download]'), str('%sdownload' % form_url))
629 650
        r += s
630
        if status is not None and get_request().is_in_backoffice():
631
            if status is False:
632
                r += htmltext(' <span class="validation-status">%s</span>') % _('not validated')
651
        if validated is not None and get_request().is_in_backoffice():
633 652
            r += htmltext('<div class="file-validation">')
634
            if status:
635
                r += htmltext('<p class="validation-status">%s</p>') % _(
636
                        '%(label)s validated by %(creator)s on %(created)s') % status
637
                r += htmltext('<ul>')
638
                for meta in status['metadata']:
639
                    r += htmltext(_('<li>%(label)s: %(value)s</li>')) % {
640
                        'label': meta['label'],
641
                        'value': meta['value']
642
                    }
643
                r += htmltext('</ul>')
653
            if validated:
654
                r += htmltext('<p class="validation-validated">%s</p>') % _(
655
                    'validated by %(creator)s on %(created)s') % value.metadata
644 656
                r += htmltext('<p>%s</p>') % (_('Valid from %(start)s to %(end)s') % {
645
                    'start': status['start'],
646
                    'end': status['end'],
657
                    'start': value.metadata['start'],
658
                    'end': value.metadata['end'],
647 659
                })
648
            else:
649
                r += file_validation.validation_link(self.filled, field, value)
660
            elif self.filled.user:
661
                r += htmltext('<form method="post" action="./validate?field_id=%s">'
662
                              '<button>%s</button></form>') % (
663
                    field.id, _('Validate'))
650 664
            r += htmltext('</div>')
651 665
        r += htmltext('</div>')
652 666
        return str(r)
wcs/qommon/form.py
59 59
import misc
60 60
from strftime import strftime
61 61
from publisher import get_cfg
62
from wcs import file_validation
62 63

  
63 64
QuixoteForm = Form
64 65

  
......
581 582
            self.value = None
582 583

  
583 584

  
585
class NoUpload(object):
586
    no_file = True
587
    metadata = None
588

  
589
    def __init__(self, validation_url):
590
        self.metadata = file_validation.get_validation(validation_url)
591

  
592

  
584 593
class FileWithPreviewWidget(CompositeWidget):
585 594
    """Widget that proposes a File Upload widget but that stores the file
586 595
    ondisk so it has a "readonly" mode where the filename is shown."""
......
589 598
    max_file_size = None
590 599
    file_type = None
591 600

  
592
    max_file_size_bytes = None # will be filled automatically
601
    max_file_size_bytes = None  # will be filled automatically
602

  
603
    PARTIAL_FILL_ERROR = _('This field is normally not required, but you must leave it completely '
604
                           'empty or fill it, You cannot fill it partially')
593 605

  
594 606
    def __init__(self, name, value=None, **kwargs):
595 607
        CompositeWidget.__init__(self, name, value, **kwargs)
596 608
        self.value = value
609
        self.document_type = kwargs.pop('document_type', None) or {}
597 610
        self.preview = kwargs.get('readonly')
598 611
        self.max_file_size = kwargs.pop('max_file_size', None)
599 612
        self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True)
......
609 622
                # this could be used for client size validation of file size
610 623
                attrs['data-max-file-size'] = str(self.max_file_size_bytes)
611 624
            self.add(FileWidget, 'file', render_br=False, attrs=attrs)
625
        if self.document_type.get('metadata'):
626
            if self.preview:
627
                self.add(HiddenWidget, 'validation_url')
628
            else:
629
                validations = file_validation.get_validations(self.document_type)
630
                if validations:
631
                    options = [('', _('Known documents'), '')]
632
                    options += [(v['url'], v['display'], v['url']) for v in validations]
633
                    self.add(SingleSelectWidget, 'validation_url', options=options,
634
                             attrs={'data-validations': json.dumps(validations)})
635
            for meta_field in self.metadata:
636
                subvalue = getattr(value, 'metadata', {}).get(meta_field['name'])
637
                self.add(StringWidget, meta_field['name'],
638
                         title=meta_field['label'],
639
                         required=meta_field.get('required', True) and self.required,
640
                         readonly=self.preview,
641
                         value=subvalue)
642
                self.get_widget(meta_field['name']).extra_css_class = 'subwidget'
612 643
        if value:
613 644
            self.set_value(value)
614 645

  
......
624 655
                self.get_widget('token').set_value(self.value.token)
625 656

  
626 657
    def render_content(self):
627
        get_response().add_javascript(['jquery.js', 'jquery-ui.js',
628
                        'jquery.iframe-transport.js', 'jquery.fileupload.js',
629
                        'qommon.fileupload.js'])
658
        get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'jquery.iframe-transport.js',
659
                                       'jquery.fileupload.js', 'qommon.fileupload.js'])
630 660

  
631 661
        temp = get_session().get_tempfile(self.get('token')) or {}
632 662

  
......
637 667
        r += htmltext('<div class="fileprogress" style="display: none;">')
638 668
        r += htmltext(' <div class="bar">%s</div>' % _('Upload in progress...'))
639 669
        r += htmltext('</div>')
640
        r += htmltext('<div class="fileinfo"><span class="filename">%s</span>' % temp.get('base_filename', ''))
670
        r += htmltext('<div class="fileinfo"><span class="filename">%s</span>'
671
                      % temp.get('base_filename', ''))
641 672
        if not self.preview:
642
            r += htmltext(' <a href="#" class="remove" title="%s">%s</a>' % (
643
                                    _('Remove this file'),
644
                                    _('remove')))
673
            r += htmltext(' <a href="#" class="remove" title="%s">%s</a>'
674
                          % (_('Remove this file'), _('remove')))
645 675
        elif temp:
646 676
            filetype = mimetypes.guess_type(temp.get('orig_filename', ''))
647 677
            if filetype and filetype[0] and filetype[0].startswith('image'):
648
                r += htmltext('<img alt="" src="tempfile?t=%s&thumbnail=1" />' % \
649
                                             self.get('token'))
678
                r += htmltext('<img alt="" src="tempfile?t=%s&thumbnail=1" />' %
679
                              self.get('token'))
650 680

  
651 681
        r += htmltext('</div>')
652 682
        return r.getvalue()
653 683

  
684
    @property
685
    def metadata(self):
686
        return self.document_type.get('metadata', [])
687

  
654 688
    def _parse(self, request):
655 689
        self.value = None
656 690
        if self.get('token'):
657 691
            token = self.get('token')
692
        elif self.get('validation_url'):
693
            self.value = NoUpload(self.get('validation_url'))
694
            return
658 695
        elif self.get('file'):
659 696
            token = get_session().add_tempfile(self.get('file'))
660 697
            request.form[self.get_widget('token').get_name()] = token
......
663 700

  
664 701
        session = get_session()
665 702
        if token and session.tempfiles and session.tempfiles.has_key(token):
666
            temp = session.tempfiles[token]
667 703
            self.value = session.get_tempfile_content(token)
668 704

  
669 705
        if self.value is None:
670
            # there's no file, the other checks are irrelevant.
706
            # there's no file, check all metadata field are empty too
707
            # if not file and required metadata field become required
708
            if (not self.required
709
                    and any([self.get(meta_field['name']) for meta_field in self.metadata])):
710
                self.set_error(self.PARTIAL_FILL_ERROR)
711
                self.get_widget('file').required = True
712
                for meta_field in self.metadata:
713
                    required = meta_field.get('required', True)
714
                    if required:
715
                        self.get_widget(meta_field['name']).required = True
671 716
            return
672 717

  
718
        # There is some file, check all required metadata files have been filled
719
        for meta_field in self.metadata:
720
            required = meta_field.get('required', True)
721
            if required:
722
                self.get_widget(meta_field['name']).required = True
723

  
724
        if self.metadata:
725
            self.value.metadata = {}
726
            for meta_field in self.metadata:
727
                self.value.metadata[meta_field['name']] = self.get(meta_field['name'])
728

  
673 729
        # Don't trust the browser supplied MIME type, update the Upload object
674 730
        # with a MIME type created with magic (or based on the extension if the
675 731
        # module is missing).
wcs/qommon/static/css/qommon.css
79 79
	vertical-align: middle;
80 80
}
81 81

  
82
/* nested widgets */
83
.widget .subwidget {
84
	padding-left: 2em;
85
}
86
div.FileWithPreviewWidget .validation {
87
	display: inline-block;
88
	font-size: xx-small;
89
	margin-right: 2em;
90
}
91

  
82 92
div.form .title, form.quixote .title {
83 93
	font-weight: bold;
84 94
}
wcs/qommon/static/js/qommon.fileupload.js
6 6
        } else {
7 7
            $(base_widget).find('.fileinfo').hide();
8 8
        }
9
        function disable() {
10
            base_widget.find('.subwidget input').prop('disabled', true);
11
            $('form').on('submit', function () {
12
                $('input').prop('disabled', false);
13
            });
14
        }
15
        function enable() {
16
            if (base_widget.find('.subwidget input:disabled').length) {
17
                base_widget.find('.subwidget input').val('');
18
                base_widget.find('.subwidget input').prop('disabled', false);
19
            }
20
        }
9 21
        $(this).find('input[type=file]').fileupload({
10 22
            dataType: 'json',
11 23
            add: function (e, data) {
......
19 31
                $(base_widget).find('.fileprogress').hide();
20 32
                $(base_widget).find('.filename').text(data.result[0].name);
21 33
                $(base_widget).find('.fileinfo').show();
22
                $(base_widget).find('input[type=hidden]').val(data.result[0].token);
34
                $(base_widget).find('input[name$="$token"]').val(data.result[0].token);
35
                $(base_widget).find('select[name$="$validation_url"]').val('');
36
                enable();
23 37
                $(base_widget).parents('form').find('input[name=submit]').prop('disabled', false);
24 38
                $(this).hide();
25 39
            },
......
29 43
            }
30 44
        });
31 45
        $(this).find('a.remove').click(function() {
32
            $(base_widget).find('input[type=hidden]').val('');
46
            $(base_widget).find('input[name$="$token"]').val('');
33 47
            $(base_widget).find('.fileinfo').hide();
34 48
            $(base_widget).find('input[type=file]').show();
35 49
            return false;
......
38 52
            $(base_widget).find('input[type=file]').click();
39 53
            return false;
40 54
        });
55
        if ($(this).find('select[name$="$validation_url"] option:selected').val()) {
56
            disable();
57
        }
58
        $(this).find('select[name$="$validation_url"]').on('change', function() {
59
            var url = $(this).find(':selected').val();
60
            if (url) {
61
                var validations = $(this).data('validations');
62

  
63
                for (var i = 0; i < validations.length; i++) {
64
                    if (validations[i].url == url) {
65
                        base_widget.find('a.remove').trigger('click');
66
                        for (var item in validations[i]) {
67
                            if (! item) {
68
                                continue;
69
                            }
70
                            var $input = base_widget.find('input[name$="$' + item + '"]');
71
                            if ($input.length) {
72
                                $input.val(validations[i][item]);
73
                            }
74
                        }
75
                        disable();
76
                    }
77
                }
78
            } else {
79
                enable();
80
            }
81
        });
41 82
    });
42 83
});
43
-