From 649c0f71fc25456d648622e79415b952853039f7 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Tue, 29 Mar 2016 12:57:49 +0200 Subject: [PATCH 2/2] rewrite file validation (fixes #10444) - metadata fields are now directly added to the form, as part of the FileWithPreviewWidget sub-widgets (it's a CompositeWidget) - existing validated metadatas are proposed for prefilling - if prefilled no file is uploadee, and a special upload class NoUpload is used instead of a PicklableUpload, it only contains the metadatas. - prefilled fields are disabled, - on upload of a new file all prefilled fields are emptied and re-enabled. - modify XML import/export to encode metadata sub-field as JSON in exports - add test on import/export of FileField with metadata - rewritten tests around file validation --- tests/test_backoffice_pages.py | 250 +++++++++++++++++++++++++----- tests/test_form_pages.py | 110 +++++++++---- tests/test_formdata.py | 109 +++++++++++++ tests/test_formdef_import.py | 15 ++ wcs/backoffice/management.py | 2 +- wcs/fields.py | 24 ++- wcs/file_validation.py | 114 +++++++------- wcs/forms/common.py | 60 ++++--- wcs/qommon/form.py | 83 ++++++++-- wcs/qommon/static/css/qommon.css | 10 ++ wcs/qommon/static/js/qommon.fileupload.js | 45 +++++- 11 files changed, 663 insertions(+), 159 deletions(-) diff --git a/tests/test_backoffice_pages.py b/tests/test_backoffice_pages.py index e441817..4c00d57 100644 --- a/tests/test_backoffice_pages.py +++ b/tests/test_backoffice_pages.py @@ -15,6 +15,7 @@ import mock from quixote import cleanup, get_publisher from wcs.qommon import errors, sessions from qommon.ident.password_accounts import PasswordAccount +from qommon.misc import json_loads from wcs.qommon.http_request import HTTPRequest from wcs.roles import Role from wcs.workflows import (Workflow, CommentableWorkflowStatusItem, @@ -1577,7 +1578,74 @@ def test_menu_json(pub): assert resp.body == 'foo(%s);' % menu_json_str assert resp.headers['content-type'] == 'application/javascript' +def test_backoffice_file_field_fargo_no_metadata(pub, fargo_url): + document_type = { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf'], + 'label': 'Justificatif de domicile', + } + user = create_user(pub, is_admin=True) + user.name_identifiers = ['12345'] + user.store() + FormDef.wipe() + formdef = FormDef() + formdef.name = 'form title' + formdef.fields = [fields.FileField( + id='0', label='1st field', type='file', + document_type=document_type)] + formdef.store() + formdef.data_class().wipe() + upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf') + digest = hashlib.sha256('%PDF-1.4').hexdigest() + app = login(get_app(pub)) + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + resp = app.get('/form-title/') + assert fargo_get.call_count == 0 + resp.forms[0]['f0$file'] = upload + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + resp = resp.forms[0].submit('submit') + assert fargo_get.call_count == 0 + assert 'Check values then click submit.' in resp.body + resp = resp.forms[0].submit('submit').follow() + assert formdef.data_class().count() == 1 + formdata = formdef.data_class().select()[0] + form_id = formdata.id + assert not hasattr(formdata.data['0'], 'metadata') + assert not '0_structured' in formdata.data + resp = app.get('/backoffice/management/form-title/%s/' % form_id) + assert not 'Validate' in resp.body + with mock.patch('wcs.file_validation.http_post_request') as http_post_request: + resp = app.get('/backoffice/management/form-title/%s/validate?field_id=0' % form_id) + assert http_post_request.call_count == 0 + resp = resp.follow() + assert not 'Valid ' in resp.body + assert not 'Validate' in resp.body + + def test_backoffice_file_field_validation(pub, fargo_url): + document_type = { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf'], + 'label': 'Justificatif de domicile', + 'metadata': [ + {'label': 'Nom', 'name': 'nom'}, + {'label': 'Prénom(s)', 'name': 'prenoms'}, + {'label': 'Numéro', 'name': 'numero'}, + {'label': 'Rue', 'name': 'rue'}, + {'label': 'Code postal', 'name': 'code-postal'}, + {'label': 'Ville', 'name': 'ville'}, + ], + } + metadata = { + 'nom': 'Doe', + 'prenoms': 'John', + 'numero': '169', + 'rue': 'rue du château', + 'code-postal': '75014', + 'ville': 'PARIS', + } user = create_user(pub, is_admin=True) user.name_identifiers = ['12345'] user.store() @@ -1586,50 +1654,162 @@ def test_backoffice_file_field_validation(pub, fargo_url): formdef.name = 'form title' formdef.fields = [fields.FileField( id='0', label='1st field', type='file', - document_type={ - 'id': 'justificatif-de-domicile', - 'fargo': True, - 'mimetypes': ['application/pdf'], - 'label': 'PDF files', - }) - ] + document_type=document_type)] formdef.store() formdef.data_class().wipe() upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf') digest = hashlib.sha256('%PDF-1.4').hexdigest() app = login(get_app(pub)) - resp = app.get('/form-title/') + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + fargo_get.return_value = {'result': 1, 'data': {'results': []}} + resp = app.get('/form-title/') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') resp.forms[0]['f0$file'] = upload - resp = resp.forms[0].submit('submit') + for key in metadata: + resp.forms[0]['f0$%s' % key] = metadata[key] + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + fargo_get.return_value = {'result': 1, 'data': {'results': []}} + resp = resp.forms[0].submit('submit') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') + for key in metadata: + assert 'value="%s"' % metadata[key] in resp.body assert 'Check values then click submit.' in resp.body - resp = resp.forms[0].submit('submit') - assert resp.status_int == 302 + resp = resp.forms[0].submit('submit').follow() + for metadata_field in document_type['metadata']: + + fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']]) + assert fragment in resp.body assert formdef.data_class().count() == 1 - form_id = formdef.data_class().select()[0].id - with mock.patch('wcs.file_validation.http_get_page') as http_get_page: - json_response = json.dumps({ - 'err': 0, - 'data': { - 'type': 'justificatif-de-domicile', - 'label': 'Justificatif de domicile', - 'creator': 'Jean Bono', - 'created': '2014-01-01T01:01:01', - 'start': '2014-01-01T01:01:01', - 'end': '2014-01-01T01:01:01', - 'metadata': [{ - 'name': 'code_postal', - 'label': 'Code postal', - 'value': '13400', - }], - }, + formdata = formdef.data_class().select()[0] + form_id = formdata.id + assert formdata.data['0'].metadata == metadata + assert formdata.data['0_structured'] == metadata + resp = app.get('/backoffice/management/form-title/%s/' % form_id) + assert 'Validate' in resp.body + for metadata_field in document_type['metadata']: + + fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']]) + assert fragment in resp.body + with mock.patch('wcs.file_validation.http_post_request') as http_post_request: + payload = { + 'user_nameid': '12345', + 'origin': 'example.net', + 'creator': 'admin', + 'content_hash': digest, + } + payload.update(metadata) + result = { + 'result': 1, + 'data': payload.copy() + } + result['data'].update({ + 'url': 'zob', + 'created': '1970-01-01T10:10:10', + 'start': '1970-01-01', + 'end': '1978-01-01', + 'display': 'John Doe, 169 rue du château, 75014 PARIS' }) - http_get_page.return_value = None, 200, json_response, None - resp = app.get('/backoffice/management/form-title/%s/' % form_id) - http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/%s/justificatif-de-domicile/' % digest) - assert 'class="value valid"' in resp.body - assert 'Justificatif de domicile validated by Jean Bono on 2014-01-01T01:01:01' in resp.body - assert 'Code postal: 13400' in resp.body - assert 'Valid from 2014-01-01T01:01:01 to 2014-01-01T01:01:01' in resp.body + http_post_request.return_value = None, 201, json.dumps(result), None + resp = app.get('/backoffice/management/form-title/%s/validate?field_id=0' % form_id) + assert http_post_request.call_count == 1 + assert http_post_request.call_args[0][0] == 'http://fargo.example.net/api/validation/justificatif-de-domicile/' + assert json_loads(http_post_request.call_args[0][1]) == payload + assert http_post_request.call_args[1] == {'headers': {'Content-Type': 'application/json'}} + resp = resp.follow() + + assert 'Valid from 1970-01-01 to 1978-01-01' in resp.body + + +def test_backoffice_file_validation_no_upload(pub, fargo_url): + document_type = { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf'], + 'label': 'Justificatif de domicile', + 'metadata': [ + {'label': 'Nom', 'name': 'nom'}, + {'label': 'Prénom(s)', 'name': 'prenoms'}, + {'label': 'Numéro', 'name': 'numero'}, + {'label': 'Rue', 'name': 'rue'}, + {'label': 'Code postal', 'name': 'code-postal'}, + {'label': 'Ville', 'name': 'ville'}, + ], + } + metadata = { + 'nom': 'Doe', + 'prenoms': 'John', + 'numero': '169', + 'rue': 'rue du château', + 'code-postal': '75014', + 'ville': 'PARIS', + } + validation = { + 'url': 'zob', + 'creator': 'admin', + 'created': '1970-01-01T10:10:10', + 'start': '1970-01-01', + 'end': '1978-01-01', + 'display': 'John Doe, 169 rue du château, 75014 PARIS' + } + validation.update(metadata) + + user = create_user(pub, is_admin=True) + user.name_identifiers = ['12345'] + user.store() + FormDef.wipe() + formdef = FormDef() + formdef.name = 'form title' + formdef.fields = [fields.FileField( + id='0', label='1st field', type='file', + document_type=document_type)] + formdef.store() + formdef.data_class().wipe() + app = login(get_app(pub)) + return_value = {'result': 1, 'data': {'results': [validation]}} + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + fargo_get.return_value = return_value + resp = app.get('/form-title/') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') + assert validation['display'] in resp.body + resp.forms[0]['f0$validation_url'] = 'zob' + for key in metadata: + resp.forms[0]['f0$%s' % key] = metadata[key] + with mock.patch('wcs.file_validation.fargo_get') as fargo_get, \ + mock.patch('wcs.file_validation.http_get_page') as http_get_page: + fargo_get.return_value = return_value + return_value = { + 'result': 1, + 'data': validation, + } + http_get_page.return_value = None, 200, json.dumps(return_value), None + resp = resp.forms[0].submit('submit') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') + http_get_page.assert_called_with('zob') + for key in metadata: + assert 'value="%s"' % metadata[key] in resp.body + assert 'Check values then click submit.' in resp.body + resp = resp.forms[0].submit('submit').follow() + assert formdef.data_class().count() == 1 + formdata = formdef.data_class().select()[0] + form_id = formdata.id + assert formdata.data['0'].metadata == validation + assert formdata.data['0_structured'] == validation + for metadata_field in document_type['metadata']: + + fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']]) + assert fragment in resp.body + resp = app.get('/backoffice/management/form-title/%s/' % form_id) + assert not 'Validate' in resp.body + for metadata_field in document_type['metadata']: + + fragment = '%s : %s' % (metadata_field['label'], metadata[metadata_field['name']]) + assert fragment in resp.body + assert 'Valid from 1970-01-01 to 1978-01-01' in resp.body + def test_360_user_view(pub): if not pub.is_using_postgresql(): diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py index 8fd1586..8eec48e 100644 --- a/tests/test_form_pages.py +++ b/tests/test_form_pages.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import json import pytest import hashlib @@ -2346,6 +2347,27 @@ def test_form_autosave(pub): assert formdef.data_class().select()[0].data['1'] == 'foobar3' def test_file_field_validation(pub, fargo_url): + document_type = { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'label': 'Justificatif de domicile', + 'metadata': [ + {'label': 'Nom', 'name': 'nom'}, + {'label': 'Prénom(s)', 'name': 'prenoms'}, + {'label': 'Numéro', 'name': 'numero'}, + {'label': 'Rue', 'name': 'rue'}, + {'label': 'Code postal', 'name': 'code-postal'}, + {'label': 'Ville', 'name': 'ville'}, + ], + } + metadata = { + 'nom': 'Doe', + 'prenoms': 'John', + 'numero': '169', + 'rue': 'rue du château', + 'code-postal': '75014', + 'ville': 'PARIS', + } user = create_user(pub) user.name_identifiers = ['12345'] user.store() @@ -2354,47 +2376,73 @@ def test_file_field_validation(pub, fargo_url): formdef.name = 'form title' formdef.fields = [fields.FileField( id='0', label='1st field', type='file', - document_type={ - 'id': 'justificatif-de-domicile', - 'fargo': True, - 'mimetypes': ['application/pdf'], - 'label': 'PDF files', - }) + document_type=document_type) ] formdef.store() formdef.data_class().wipe() upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf') - digest = hashlib.sha256('%PDF-1.4').hexdigest() app = login(get_app(pub), username='foo', password='foo') - resp = app.get('/form-title/') + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + fargo_get.return_value = {'result': 1, 'data': {'results': []}} + resp = app.get('/form-title/') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') resp.forms[0]['f0$file'] = upload + for key in metadata: + resp.forms[0]['f0$%s' % key] = metadata[key] + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + fargo_get.return_value = {'result': 1, 'data': {'results': []}} + resp = resp.forms[0].submit('submit') + fargo_get.assert_called_once_with( + 'api/validation/justificatif-de-domicile/?user_nameid=12345') + assert 'Check values then click submit.' in resp.body resp = resp.forms[0].submit('submit') + assert resp.status_int == 302 + resp = resp.follow() + assert 'The form has been recorded' in resp.body + assert formdef.data_class().count() == 1 + formdata = formdef.data_class().select()[0] + assert formdata.data['0'].metadata == metadata + assert formdata.data['0_structured'] == metadata + + +def test_file_field_fargo_no_metadata(pub, fargo_url): + document_type = { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'label': 'Justificatif de domicile', + } + user = create_user(pub) + user.name_identifiers = ['12345'] + user.store() + FormDef.wipe() + formdef = FormDef() + formdef.name = 'form title' + formdef.fields = [fields.FileField( + id='0', label='1st field', type='file', + document_type=document_type) + ] + formdef.store() + formdef.data_class().wipe() + upload = Upload('test.pdf', '%PDF-1.4', 'application/pdf') + app = login(get_app(pub), username='foo', password='foo') + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + resp = app.get('/form-title/') + assert fargo_get.call_count == 0 + resp.forms[0]['f0$file'] = upload + with mock.patch('wcs.file_validation.fargo_get') as fargo_get: + resp = resp.forms[0].submit('submit') + assert fargo_get.call_count == 0 assert 'Check values then click submit.' in resp.body resp = resp.forms[0].submit('submit') assert resp.status_int == 302 - with mock.patch('wcs.file_validation.http_get_page') as http_get_page: - json_response = json.dumps({ - 'err': 0, - 'data': { - 'type': 'justificatif-de-domicile', - 'label': 'Justificatif de domicile', - 'creator': 'Jean Bono', - 'created': '2014-01-01T01:01:01', - 'start': '2014-01-01T01:01:01', - 'end': '2014-01-01T01:01:01', - 'metadata': [{ - 'name': 'code_postal', - 'label': 'Code postal', - 'value': '13400', - }], - }, - }) - http_get_page.return_value = None, 200, json_response, None - resp = resp.follow() - http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/%s/justificatif-de-domicile/' % digest) - http_get_page.reset_mock() - assert 'The form has been recorded' in resp.body - assert 'class="value valid"' in resp.body + resp = resp.follow() + assert 'The form has been recorded' in resp.body + assert formdef.data_class().count() == 1 + formdata = formdef.data_class().select()[0] + assert not hasattr(formdata.data['0'], 'metadata') + assert not '0_structured' in formdata.data + def test_form_string_field_autocomplete(pub): formdef = create_formdef() diff --git a/tests/test_formdata.py b/tests/test_formdata.py index e07ffae..74614bd 100644 --- a/tests/test_formdata.py +++ b/tests/test_formdata.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import pytest import sys import shutil @@ -11,6 +12,8 @@ from wcs.formdef import FormDef from wcs.formdata import Evolution from wcs.workflows import Workflow, WorkflowCriticalityLevel from wcs.wf.anonymise import AnonymiseWorkflowStatusItem +from wcs.qommon.form import NoUpload +import mock from utilities import create_temporary_pub, clean_temporary_pub @@ -148,6 +151,112 @@ def test_file_field(pub): assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0') assert isinstance(substvars.get('form_var_foo_raw'), Upload) + +def test_file_field_fargo_no_metadata(pub): + document_types = { + 'justificatif-de-domicile': { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'label': 'Justificatif de domicile', + } + } + formdef.data_class().wipe() + formdef.fields = [fields.FileField(id='0', label='file', varname='foo', + document_type=document_types['justificatif-de-domicile'])] + formdef.store() + formdata = formdef.data_class()() + upload = Upload('test.txt', 'text/plain', 'ascii') + upload.receive(['first line', 'second line']) + formdata.data = {'0': upload} + formdata.id = 1 + substvars = formdata.get_substitution_variables() + assert substvars.get('form_var_foo') == 'test.txt' + assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0') + assert isinstance(substvars.get('form_var_foo_raw'), Upload) + + +def test_file_field_with_metadata(pub): + document_types = { + 'justificatif-de-domicile': { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'label': 'Justificatif de domicile', + 'metadata': [ + {'label': 'Nom', 'name': 'nom'}, + {'label': 'Prénom(s)', 'name': 'prenoms'}, + {'label': 'Numéro', 'name': 'numero'}, + {'label': 'Rue', 'name': 'rue'}, + {'label': 'Code postal', 'name': 'code-postal'}, + {'label': 'Ville', 'name': 'ville'}, + ], + } + } + formdef.data_class().wipe() + formdef.fields = [fields.FileField(id='0', label='file', varname='foo', + document_type=document_types['justificatif-de-domicile'])] + formdef.store() + formdata = formdef.data_class()() + upload = Upload('test.txt', 'text/plain', 'ascii') + upload.receive(['first line', 'second line']) + upload.metadata = { + 'nom': 'Doe', + 'prenoms': 'John', + 'numero': '169', + 'rue': 'rue du château', + 'code-postal': '75014', + 'ville': 'PARIS', + } + formdata.data = {'0': upload, '0_structured': upload.metadata} + formdata.id = 1 + substvars = formdata.get_substitution_variables() + assert substvars.get('form_var_foo') == 'test.txt' + assert substvars.get('form_var_foo_url').endswith('/foobar/1/download?f=0') + assert isinstance(substvars.get('form_var_foo_raw'), Upload) + for key in upload.metadata: + assert substvars.get('form_var_foo_%s' % key) == upload.metadata[key] + + +def test_file_field_no_file_with_metadata(pub): + document_types = { + 'justificatif-de-domicile': { + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'label': 'Justificatif de domicile', + 'metadata': [ + {'label': 'Nom', 'name': 'nom'}, + {'label': 'Prénom(s)', 'name': 'prenoms'}, + {'label': 'Numéro', 'name': 'numero'}, + {'label': 'Rue', 'name': 'rue'}, + {'label': 'Code postal', 'name': 'code-postal'}, + {'label': 'Ville', 'name': 'ville'}, + ], + } + } + formdef.data_class().wipe() + formdef.fields = [fields.FileField(id='0', label='file', varname='foo', + document_type=document_types['justificatif-de-domicile'])] + formdef.store() + formdata = formdef.data_class()() + metadata = { + 'nom': 'Doe', + 'prenoms': 'John', + 'numero': '169', + 'rue': 'rue du château', + 'code-postal': '75014', + 'ville': 'PARIS', + } + with mock.patch('wcs.file_validation.get_validation', return_value=metadata): + upload = NoUpload('http://whatever.com/') + formdata.data = {'0': upload, '0_structured': upload.metadata} + formdata.id = 1 + substvars = formdata.get_substitution_variables() + assert isinstance(substvars.get('form_var_foo'), NoUpload) + assert not 'form_var_foo_url' in substvars + assert not 'form_var_foo_raw' in substvars + for key in upload.metadata: + assert substvars.get('form_var_foo_%s' % key) == upload.metadata[key] + + def test_get_submitter(pub): formdef.data_class().wipe() formdef.fields = [fields.StringField(id='0', label='email', varname='foo', diff --git a/tests/test_formdef_import.py b/tests/test_formdef_import.py index 417534e..521618e 100644 --- a/tests/test_formdef_import.py +++ b/tests/test_formdef_import.py @@ -242,6 +242,21 @@ def test_file_field(): assert_xml_import_export_works(formdef) assert_json_import_export_works(formdef, include_id=True) assert_json_import_export_works(formdef) + formdef = FormDef() + formdef.name = 'foo' + formdef.fields = [fields.FileField(type='file', id='1', document_type={ + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'image/*'], + 'metadata': [ + {'name': 'nom', 'label': 'Nom'}, + {'name': 'rue', 'label': 'Rue'}, + ] + })] + assert_xml_import_export_works(formdef, include_id=True) + assert_xml_import_export_works(formdef) + assert_json_import_export_works(formdef, include_id=True) + assert_json_import_export_works(formdef) def test_unknown_data_source(): formdef = FormDef() diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py index 081bcaa..1ac8f75 100644 --- a/wcs/backoffice/management.py +++ b/wcs/backoffice/management.py @@ -1623,7 +1623,7 @@ class FormPage(Directory): class FormBackOfficeStatusPage(FormStatusPage): - _q_exports = ['', 'download', 'json', 'action', 'inspect'] + _q_exports = ['', 'download', 'json', 'action', 'inspect', 'validate'] form_page_class = FormFillPage def html_top(self, title = None): diff --git a/wcs/fields.py b/wcs/fields.py index 02edab9..784e49d 100644 --- a/wcs/fields.py +++ b/wcs/fields.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, see . +import json import time import random import re @@ -720,6 +721,10 @@ class FileField(WidgetField): self.document_type = self.document_type or {} @property + def metadata(self): + return self.document_type.get('metadata', []) + + @property def file_type(self): return (self.document_type or {}).get('mimetypes', []) @@ -752,8 +757,14 @@ class FileField(WidgetField): 'document_type', 'max_file_size', 'allow_portfolio_picking'] def get_view_value(self, value): - return htmltext('%s') % ( + r = TemplateIO(html=True) + if not getattr(value, 'no_file', False): + r += htmltext('%s') % ( value.base_filename, self.id, value) + for meta_field in self.metadata: + metadata_value = getattr(value, 'metadata', {}).get(meta_field['name'], '') + r += htmltext('

%s : %s

') % (meta_field['label'], metadata_value) + return r.getvalue() def get_csv_value(self, value, hint=None): if not value: @@ -788,6 +799,7 @@ class FileField(WidgetField): value = get_request().get_field(self.field_key) if value and hasattr(value, 'token'): get_request().form[self.field_key + '$token'] = value.token + kwargs['document_type'] = self.document_type def get_document_types(self): document_types = { @@ -854,9 +866,13 @@ class FileField(WidgetField): if self.document_type and self.document_type.get('mimetypes'): old_value = self.document_type['mimetypes'] self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes']) + if self.document_type and self.document_type.get('metadata'): + self.document_type['metadata'] = json.dumps(self.document_type['metadata']) result = super(FileField, self).export_to_xml(charset, include_id=include_id) if self.document_type and self.document_type.get('mimetypes'): self.document_type['mimetypes'] = old_value + if self.document_type and self.document_type.get('metadata'): + self.document_type['metadata'] = json.loads(self.document_type['metadata']) return result def init_with_xml(self, element, charset, include_id=False): @@ -866,6 +882,12 @@ class FileField(WidgetField): self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|') if self.document_type and self.document_type.get('fargo'): self.document_type['fargo'] = self.document_type['fargo'] == 'True' + if self.document_type and self.document_type.get('metadata'): + self.document_type['metadata'] = json.loads(self.document_type['metadata']) + + def store_structured_value(self, data, field_id): + value = data.get(field_id) + return getattr(value, 'metadata', {}) register_field_class(FileField) diff --git a/wcs/file_validation.py b/wcs/file_validation.py index 21e8dc5..7190574 100644 --- a/wcs/file_validation.py +++ b/wcs/file_validation.py @@ -19,14 +19,14 @@ import urlparse import hashlib import urllib -from qommon.misc import http_get_page, json_loads -from quixote import get_publisher, get_response -from quixote.html import htmltext +from qommon.misc import http_get_page, json_loads, http_post_request +from quixote import get_publisher, get_request def has_file_validation(): return get_publisher().get_site_option('fargo_url') is not None + def fargo_get(path): fargo_url = get_publisher().get_site_option('fargo_url') url = urlparse.urljoin(fargo_url, path) @@ -35,14 +35,15 @@ def fargo_get(path): return json_loads(data) return None + def sha256_of_upload(upload): return hashlib.sha256(upload.get_content()).hexdigest() + def get_document_types(): if not has_file_validation(): return {} response = fargo_get('/document-types/') - publisher = get_publisher() if response.get('err') == 0: result = {} for schema in response['data']: @@ -52,60 +53,65 @@ def get_document_types(): 'fargo': True, } if 'mimetypes' in schema: - d['mimetypes'] = shema['mimetypes'] + d['mimetypes'] = schema['mimetypes'] result[d['id']] = d - + d['metadata'] = schema.get('metadata', []) return result return {} -def validation_path(filled, field, upload): - user = filled.get_user() - if not user: - return None - if not user.name_identifiers: - return None - if not field.document_type or not field.document_type.get('fargo'): - return None - name_id = user.name_identifiers[0] - sha_256 = sha256_of_upload(upload) - document_type = field.document_type['id'] - path = '%s/%s/%s/' % ( - urllib.quote(name_id), - urllib.quote(sha_256), - urllib.quote(document_type), - ) - return path - -def validate_upload(filled, field, upload): + +def get_validation(url): + response, status, data, auth_header = http_get_page(url) + if status == 200: + return json_loads(data)['data'] + return None + + +def get_validations(document_type): + request = get_request() + document_type_id = document_type['id'] + qs = {} + if not request.user: + return [] + if request.user.name_identifiers: + qs['user_nameid'] = request.user.name_identifiers[0] + elif request.user.email: + qs['user_email'] = request.user.email + else: + return [] + path = 'api/validation/%s/' % urllib.quote(document_type_id) + validations = fargo_get(path + '?%s' % urllib.urlencode(qs)) + if validations and validations.get('data', {}).get('results', []): + return validations['data']['results'] + return [] + + +def is_valid(filled, field, upload): '''Check validation of the uploaded file with Fargo''' - path = validation_path(filled, field, upload) - if not path: - return None - response = fargo_get('metadata/' + path) - if response is None: - return None - if response['err'] == 1: - return False - return response['data'] - -def get_document_type_label(document_type): - return get_document_types().get(document_type, {}).get('label') - -def validation_link(filled, field, upload): + return 'url' in getattr(upload, 'metadata', {}) + + +def validate(filled, field, upload): '''Compute link to Fargo to validate the given document''' - path = validation_path(filled, field, upload) - if not path: - return '' - get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css') - get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js']) - label = get_document_type_label(field.document_type['id']) + document_type_id = field.document_type['id'] + path = 'api/validation/%s/' % urllib.quote(document_type_id) fargo_url = get_publisher().get_site_option('fargo_url') - url = urlparse.urljoin(fargo_url, 'validation/' + path) - next_url = get_publisher().get_frontoffice_url() + '/reload-top' - url += '?next=%s' % urllib.quote(next_url) - title = _('Validate as a %s') % label - return htmltext('%(title)s') % { - 'title': title, - 'url': url, - } + url = urlparse.urljoin(fargo_url, path) + payload = {} + if filled.user: + if filled.user.name_identifiers: + payload['user_nameid'] = filled.user.name_identifiers[0] + else: + payload['user_email'] = filled.user.email + payload['origin'] = get_request().get_server() + payload['creator'] = get_request().user.display_name + payload['content_hash'] = sha256_of_upload(upload) + for meta_field in field.metadata: + payload[meta_field['name']] = upload.metadata.get(meta_field['name'], '') + headers = {'Content-Type': 'application/json'} + response, status, response_payload, auth_header = http_post_request(url, json.dumps(payload), + headers=headers) + if status == 201: + upload.metadata = json_loads(response_payload)['data'] + filled.data['%s_structured' % field.id] = upload.metadata + filled.store() diff --git a/wcs/forms/common.py b/wcs/forms/common.py index 4195307..9a46cbf 100644 --- a/wcs/forms/common.py +++ b/wcs/forms/common.py @@ -97,6 +97,27 @@ class FormStatusPage(Directory): def html_top(self, title = None): template.html_top(title = title, default_org = _('Forms')) + def validate(self): + if not file_validation.has_file_validation(): + return redirect('.') + field_id = get_request().form.get('field_id') + if not field_id: + return redirect('.') + for field in self.formdef.fields: + if field.id == field_id: + break + else: + return redirect('.') + if field.key != 'file': + return redirect('.') + if not field.document_type.get('fargo', False): + return redirect('.') + if not field.document_type.get('metadata', []): + return redirect('.') + value = self.filled.data.get(field_id) + file_validation.validate(self.filled, field, value) + return redirect('.') + def __init__(self, formdef, filled, register_workflow_subdirs=True): get_publisher().substitutions.feed(filled) self.formdef = formdef @@ -612,13 +633,13 @@ class FormStatusPage(Directory): def display_file_field(self, form_url, field, value): r = TemplateIO(html=True) - status = None - if file_validation.has_file_validation(): - status = file_validation.validate_upload(self.filled, field, value) - if status is False: + validated = None + is_fargo_dt = field.document_type.get('fargo', False) + has_metadata = bool(field.document_type.get('metadata', [])) + if file_validation.has_file_validation() and is_fargo_dt and has_metadata: + validated = file_validation.is_valid(self.filled, field, value) + if validated is False: extra_class = ' invalid' - elif status is None: - extra_class = '' else: extra_class = ' valid' r += htmltext('
' % extra_class) @@ -627,26 +648,19 @@ class FormStatusPage(Directory): s = field.get_view_value(value) s = s.replace(str('[download]'), str('%sdownload' % form_url)) r += s - if status is not None and get_request().is_in_backoffice(): - if status is False: - r += htmltext(' %s') % _('not validated') + if validated is not None and get_request().is_in_backoffice(): r += htmltext('
') - if status: - r += htmltext('

%s

') % _( - '%(label)s validated by %(creator)s on %(created)s') % status - r += htmltext('
    ') - for meta in status['metadata']: - r += htmltext(_('
  • %(label)s: %(value)s
    • ')) % { - 'label': meta['label'], - 'value': meta['value'] - } - r += htmltext('
') + if validated: + r += htmltext('

%s

') % _( + 'validated by %(creator)s on %(created)s') % value.metadata r += htmltext('

%s

') % (_('Valid from %(start)s to %(end)s') % { - 'start': status['start'], - 'end': status['end'], + 'start': value.metadata['start'], + 'end': value.metadata['end'], }) - else: - r += file_validation.validation_link(self.filled, field, value) + elif self.filled.user: + r += htmltext('
' + '
') % ( + field.id, _('Validate')) r += htmltext('
') r += htmltext('
') return str(r) diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py index 19ab2f4..0305835 100644 --- a/wcs/qommon/form.py +++ b/wcs/qommon/form.py @@ -60,6 +60,7 @@ from quixote.util import randbytes import misc from strftime import strftime from publisher import get_cfg +from wcs import file_validation QuixoteForm = Form @@ -582,6 +583,14 @@ class UploadWidget(CompositeWidget): self.value = None +class NoUpload(object): + no_file = True + metadata = None + + def __init__(self, validation_url): + self.metadata = file_validation.get_validation(validation_url) + + class FileWithPreviewWidget(CompositeWidget): """Widget that proposes a File Upload widget but that stores the file ondisk so it has a "readonly" mode where the filename is shown.""" @@ -590,11 +599,12 @@ class FileWithPreviewWidget(CompositeWidget): max_file_size = None file_type = None - max_file_size_bytes = None # will be filled automatically + max_file_size_bytes = None # will be filled automatically def __init__(self, name, value=None, **kwargs): CompositeWidget.__init__(self, name, value, **kwargs) self.value = value + self.document_type = kwargs.pop('document_type', None) or {} self.preview = kwargs.get('readonly') self.max_file_size = kwargs.pop('max_file_size', None) self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True) @@ -610,6 +620,24 @@ class FileWithPreviewWidget(CompositeWidget): # this could be used for client size validation of file size attrs['data-max-file-size'] = str(self.max_file_size_bytes) self.add(FileWidget, 'file', render_br=False, attrs=attrs) + if self.document_type.get('metadata'): + if self.preview: + self.add(HiddenWidget, 'validation_url') + else: + validations = file_validation.get_validations(self.document_type) + if validations: + options = [('', _('Known documents'), '')] + options += [(v['url'], v['display'], v['url']) for v in validations] + self.add(SingleSelectWidget, 'validation_url', options=options, + attrs={'data-validations': json.dumps(validations)}) + for meta_field in self.metadata: + subvalue = getattr(value, 'metadata', {}).get(meta_field['name']) + self.add(StringWidget, meta_field['name'], + title=meta_field['label'], + required=meta_field.get('required', True) and self.required, + readonly=self.preview, + value=subvalue) + self.get_widget(meta_field['name']).extra_css_class = 'subwidget' if value: self.set_value(value) @@ -625,9 +653,8 @@ class FileWithPreviewWidget(CompositeWidget): self.get_widget('token').set_value(self.value.token) def render_content(self): - get_response().add_javascript(['jquery.js', 'jquery-ui.js', - 'jquery.iframe-transport.js', 'jquery.fileupload.js', - 'qommon.fileupload.js']) + get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'jquery.iframe-transport.js', + 'jquery.fileupload.js', 'qommon.fileupload.js']) temp = get_session().get_tempfile(self.get('token')) or {} @@ -638,24 +665,31 @@ class FileWithPreviewWidget(CompositeWidget): r += htmltext('') - r += htmltext('
%s' % temp.get('base_filename', '')) + r += htmltext('
%s' + % temp.get('base_filename', '')) if not self.preview: - r += htmltext(' %s' % ( - _('Remove this file'), - _('remove'))) + r += htmltext(' %s' + % (_('Remove this file'), _('remove'))) elif temp: filetype = mimetypes.guess_type(temp.get('orig_filename', '')) if filetype and filetype[0] and filetype[0].startswith('image'): - r += htmltext('' % \ - self.get('token')) + r += htmltext('' % + self.get('token')) r += htmltext('
') return r.getvalue() + @property + def metadata(self): + return self.document_type.get('metadata', []) + def _parse(self, request): self.value = None if self.get('token'): token = self.get('token') + elif self.get('validation_url'): + self.value = NoUpload(self.get('validation_url')) + return elif self.get('file'): token = get_session().add_tempfile(self.get('file')) request.form[self.get_widget('token').get_name()] = token @@ -664,13 +698,38 @@ class FileWithPreviewWidget(CompositeWidget): session = get_session() if token and session.tempfiles and session.tempfiles.has_key(token): - temp = session.tempfiles[token] self.value = session.get_tempfile_content(token) if self.value is None: - # there's no file, the other checks are irrelevant. + # there's no file, check all metadata field are empty too + # if not file and required metadata field become required + if (self.get_widget('file') + and not self.required + and any([self.get(meta_field['name']) for meta_field in self.metadata])): + self.get_widget('file').required = True + for meta_field in self.metadata: + required = meta_field.get('required', True) + if required: + widget = self.get_widget(meta_field['name']) + widget.required = True + if not self.get(meta_field['name']): + widget.set_error(self.REQUIRED_ERROR) return + # There is some file, check all required metadata files have been filled + for meta_field in self.metadata: + required = meta_field.get('required', True) + if required: + widget = self.get_widget(meta_field['name']) + widget.required = True + if not self.get(meta_field['name']): + widget.set_error(self.REQUIRED_ERROR) + + if self.metadata: + self.value.metadata = {} + for meta_field in self.metadata: + self.value.metadata[meta_field['name']] = self.get(meta_field['name']) + # Don't trust the browser supplied MIME type, update the Upload object # with a MIME type created with magic (or based on the extension if the # module is missing). diff --git a/wcs/qommon/static/css/qommon.css b/wcs/qommon/static/css/qommon.css index 34f1c5e..a9df4d8 100644 --- a/wcs/qommon/static/css/qommon.css +++ b/wcs/qommon/static/css/qommon.css @@ -79,6 +79,16 @@ div.SubmitWidget input, input[type=submit] { vertical-align: middle; } +/* nested widgets */ +.widget .subwidget { + padding-left: 2em; +} +div.FileWithPreviewWidget .validation { + display: inline-block; + font-size: xx-small; + margin-right: 2em; +} + div.form .title, form.quixote .title { font-weight: bold; } diff --git a/wcs/qommon/static/js/qommon.fileupload.js b/wcs/qommon/static/js/qommon.fileupload.js index cc6ee8c..3e25d28 100644 --- a/wcs/qommon/static/js/qommon.fileupload.js +++ b/wcs/qommon/static/js/qommon.fileupload.js @@ -6,6 +6,18 @@ $(function() { } else { $(base_widget).find('.fileinfo').hide(); } + function disable() { + base_widget.find('.subwidget input').prop('disabled', true); + $('form').on('submit', function () { + $('input').prop('disabled', false); + }); + } + function enable() { + if (base_widget.find('.subwidget input:disabled').length) { + base_widget.find('.subwidget input').val(''); + base_widget.find('.subwidget input').prop('disabled', false); + } + } $(this).find('input[type=file]').fileupload({ dataType: 'json', add: function (e, data) { @@ -19,7 +31,9 @@ $(function() { $(base_widget).find('.fileprogress').hide(); $(base_widget).find('.filename').text(data.result[0].name); $(base_widget).find('.fileinfo').show(); - $(base_widget).find('input[type=hidden]').val(data.result[0].token); + $(base_widget).find('input[name$="$token"]').val(data.result[0].token); + $(base_widget).find('select[name$="$validation_url"]').val(''); + enable(); $(base_widget).parents('form').find('input[name=submit]').prop('disabled', false); $(this).hide(); }, @@ -29,7 +43,7 @@ $(function() { } }); $(this).find('a.remove').click(function() { - $(base_widget).find('input[type=hidden]').val(''); + $(base_widget).find('input[name$="$token"]').val(''); $(base_widget).find('.fileinfo').hide(); $(base_widget).find('input[type=file]').show(); return false; @@ -38,5 +52,32 @@ $(function() { $(base_widget).find('input[type=file]').click(); return false; }); + if ($(this).find('select[name$="$validation_url"] option:selected').val()) { + disable(); + } + $(this).find('select[name$="$validation_url"]').on('change', function() { + var url = $(this).find(':selected').val(); + if (url) { + var validations = $(this).data('validations'); + + for (var i = 0; i < validations.length; i++) { + if (validations[i].url == url) { + base_widget.find('a.remove').trigger('click'); + for (var item in validations[i]) { + if (! item) { + continue; + } + var $input = base_widget.find('input[name$="$' + item + '"]'); + if ($input.length) { + $input.val(validations[i][item]); + } + } + disable(); + } + } + } else { + enable(); + } + }); }); }); -- 2.1.4