From 4c312191e1c2e209d4e96ad43eaf1ea739b7d934 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Mon, 28 Sep 2015 17:16:21 +0200 Subject: [PATCH 3/4] add support for file validation (#8402) --- tests/conftest.py | 26 ++++++++++ tests/test_backoffice_pages.py | 58 +++++++++++++++++++++ tests/test_form_pages.py | 54 +++++++++++++++++++ tests/test_formdef.py | 27 +++++++++- tests/test_formdef_import.py | 14 +++++ wcs/fields.py | 115 ++++++++++++++++++++++++++++++++++------- wcs/file_validation.py | 111 +++++++++++++++++++++++++++++++++++++++ wcs/forms/common.py | 59 ++++++++++++++++++--- wcs/qommon/static/js/fargo.js | 56 ++++++++++++++++++++ wcs/root.py | 6 ++- 10 files changed, 497 insertions(+), 29 deletions(-) create mode 100644 wcs/file_validation.py create mode 100644 wcs/qommon/static/js/fargo.js diff --git a/tests/conftest.py b/tests/conftest.py index 828a524..077eafc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,6 @@ +import os +import ConfigParser + import pytest def pytest_addoption(parser): @@ -7,3 +10,26 @@ def pytest_addoption(parser): def pytest_runtest_setup(item): if 'postgresql' in item.keywords and item.config.option.without_postgresql_tests is True: pytest.skip('skipped (PostgreSQL are disabled on command line)') + +@pytest.fixture +def fargo_url(request, pub): + config = ConfigParser.ConfigParser() + path = os.path.join(pub.app_dir, 'site-options.cfg') + url = 'http://fargo.example.net/' + if os.path.exists(path): + config.read([path]) + if not config.has_section('options'): + config.add_section('options') + config.set('options', 'fargo_url', url) + with file(path, 'w') as site_option: + config.write(site_option) + + def fin(): + config = ConfigParser.ConfigParser() + if os.path.exists(path): + config.read([path]) + config.remove_option('options', 'fargo_url') + with file(path, 'w') as site_option: + config.write(site_option) + request.addfinalizer(fin) + return url diff --git a/tests/test_backoffice_pages.py b/tests/test_backoffice_pages.py index b6700b7..f6fd62d 100644 --- a/tests/test_backoffice_pages.py +++ b/tests/test_backoffice_pages.py @@ -4,8 +4,11 @@ import os import re import shutil import StringIO +import hashlib import pytest +from webtest import Upload +import mock from quixote import cleanup, get_publisher from wcs.qommon import errors, sessions @@ -935,3 +938,58 @@ def test_backoffice_sidebar_user_context(pub): assert '>cat1<' in resp.body assert '>Misc<' in resp.body assert resp.body.index('>Misc<') < resp.body.index('>cat1<') + + +def test_backoffice_file_field_validation(pub, fargo_url): + user = create_user(pub, is_admin=True) + user.name_identifiers = ['12345'] + user.store() + FormDef.wipe() + formdef = FormDef() + formdef.name = 'form title' + formdef.fields = [fields.FileField( + id='0', label='1st field', type='file', + document_type={ + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf'], + 'label': 'PDF files', + }) + ] + formdef.store() + formdef.data_class().wipe() + upload = Upload('test.pdf', 'foobar', 'application/pdf') + digest = hashlib.sha256('foobar').hexdigest() + app = login(get_app(pub)) + resp = app.get('/form-title/') + resp.forms[0]['f0$file'] = upload + resp = resp.forms[0].submit('submit') + assert 'Check values then click submit.' in resp.body + resp = resp.forms[0].submit('submit') + assert resp.status_int == 302 + assert formdef.data_class().count() == 1 + form_id = formdef.data_class().select()[0].id + with mock.patch('wcs.file_validation.http_get_page') as http_get_page: + json_response = json.dumps({ + 'err': 0, + 'data': { + 'type': 'justificatif-de-domicile', + 'label': 'Justificatif de domicile', + 'creator': 'Jean Bono', + 'created': '2014-01-01T01:01:01', + 'start': '2014-01-01T01:01:01', + 'end': '2014-01-01T01:01:01', + 'metadata': [{ + 'name': 'code_postal', + 'label': 'Code postal', + 'value': '13400', + }], + }, + }) + http_get_page.return_value = None, 200, json_response, None + resp = app.get('/backoffice/management/form-title/%s/' % form_id) + http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2/justificatif-de-domicile/') + assert 'class="value valid"' in resp.body + assert 'Justificatif de domicile validated by Jean Bono on 2014-01-01T01:01:01' in resp.body + assert 'Code postal: 13400' in resp.body + assert 'Valid from 2014-01-01T01:01:01 to 2014-01-01T01:01:01' in resp.body diff --git a/tests/test_form_pages.py b/tests/test_form_pages.py index ad80d57..3456d6f 100644 --- a/tests/test_form_pages.py +++ b/tests/test_form_pages.py @@ -1,3 +1,4 @@ +import json import pytest import hashlib import os @@ -7,6 +8,7 @@ import time import zipfile import base64 from webtest import Upload +import mock from quixote.http_request import Upload as QuixoteUpload from wcs.qommon.form import UploadedFile @@ -1912,3 +1914,55 @@ def test_form_autosave(pub): assert formdef.data_class().count() == 1 assert formdef.data_class().select()[0].data['1'] == 'foobar3' assert formdef.data_class().select()[0].data['3'] == 'xxx3' + + +def test_file_field_validation(pub, fargo_url): + user = create_user(pub) + user.name_identifiers = ['12345'] + user.store() + FormDef.wipe() + formdef = FormDef() + formdef.name = 'form title' + formdef.fields = [fields.FileField( + id='0', label='1st field', type='file', + document_type={ + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf'], + 'label': 'PDF files', + }) + ] + formdef.store() + formdef.data_class().wipe() + upload = Upload('test.pdf', 'foobar', 'application/pdf') + digest = hashlib.sha256('foobar').hexdigest() + app = login(get_app(pub), username='foo', password='foo') + resp = app.get('/form-title/') + resp.forms[0]['f0$file'] = upload + resp = resp.forms[0].submit('submit') + assert 'Check values then click submit.' in resp.body + resp = resp.forms[0].submit('submit') + assert resp.status_int == 302 + with mock.patch('wcs.file_validation.http_get_page') as http_get_page: + json_response = json.dumps({ + 'err': 0, + 'data': { + 'type': 'justificatif-de-domicile', + 'label': 'Justificatif de domicile', + 'creator': 'Jean Bono', + 'created': '2014-01-01T01:01:01', + 'start': '2014-01-01T01:01:01', + 'end': '2014-01-01T01:01:01', + 'metadata': [{ + 'name': 'code_postal', + 'label': 'Code postal', + 'value': '13400', + }], + }, + }) + http_get_page.return_value = None, 200, json_response, None + resp = resp.follow() + http_get_page.assert_called_once_with('http://fargo.example.net/metadata/12345/c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2/justificatif-de-domicile/') + http_get_page.reset_mock() + assert 'The form has been recorded' in resp.body + assert 'class="value valid"' in resp.body diff --git a/tests/test_formdef.py b/tests/test_formdef.py index df01d26..53c22d6 100644 --- a/tests/test_formdef.py +++ b/tests/test_formdef.py @@ -4,11 +4,13 @@ import shutil import pytest +from mock import patch + from quixote import cleanup from wcs import formdef from wcs.formdef import FormDef from wcs.workflows import Workflow -from wcs.fields import StringField +from wcs.fields import StringField, FileField from utilities import create_temporary_pub @@ -132,3 +134,26 @@ def test_substitution_variables_object(): with pytest.raises(AttributeError): assert substs.foobar + +def test_file_field_migration(): + with patch('wcs.file_validation.get_document_types') as get_document_types: + get_document_types.return_value = { + 'justificatif-de-domicile': { + 'id': 'justificatif-de-domicile', + 'label': 'Justificatif de domicile', + 'fargo': True, + }, + } + FormDef.wipe() + formdef = FormDef() + formdef.name = 'foo' + file_type = ['image/*', 'application/pdf,application/vnd.oasis.opendocument.text,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.oasis.opendocument.spreadsheet,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'] + formdef.fields = [FileField(type='file', id='1', label='file')] + formdef.fields[0].__dict__['file_type'] = file_type + formdef.store() + formdef = FormDef.get(1) + assert 'file_type' not in formdef.fields[0].__dict__ + assert formdef.fields[0].document_type + assert formdef.fields[0].document_type['id'] == '_legacy' + assert formdef.fields[0].document_type['mimetypes'] == ['image/*', 'application/pdf,application/vnd.oasis.opendocument.text,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.oasis.opendocument.spreadsheet,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'] + assert formdef.fields[0].document_type['label'] == ','.join(file_type) diff --git a/tests/test_formdef_import.py b/tests/test_formdef_import.py index 40e89b6..ca495ff 100644 --- a/tests/test_formdef_import.py +++ b/tests/test_formdef_import.py @@ -227,3 +227,17 @@ def test_category_reference(): cat.store() assert FormDef.import_from_xml_tree(formdef_xml_with_id, include_id=False).category_id == '2' assert FormDef.import_from_xml_tree(formdef_xml_with_id, include_id=True).category_id is None + + +def test_file_field(): + formdef = FormDef() + formdef.name = 'foo' + formdef.fields = [fields.FileField(type='file', id='1', document_type={ + 'id': 'justificatif-de-domicile', + 'fargo': True, + 'mimetypes': ['application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'image/*'], + })] + assert_xml_import_export_works(formdef, include_id=True) + assert_xml_import_export_works(formdef) + assert_json_import_export_works(formdef, include_id=True) + assert_json_import_export_works(formdef) diff --git a/wcs/fields.py b/wcs/fields.py index b1977d1..b968508 100644 --- a/wcs/fields.py +++ b/wcs/fields.py @@ -30,6 +30,7 @@ from qommon import get_cfg, get_logger from qommon.strftime import strftime import data_sources +import file_validation class PrefillSelectionWidget(CompositeWidget): @@ -674,38 +675,43 @@ register_field_class(BoolField) class FileField(WidgetField): key = 'file' description = N_('File Upload') - file_type = [] + document_type = None max_file_size = None widget_class = FileWithPreviewWidget extra_attributes = ['file_type', 'max_file_size'] + def __init__(self, *args, **kwargs): + super(FileField, self).__init__(*args, **kwargs) + self.document_type = self.document_type or {} + + @property + def file_type(self): + return (self.document_type or {}).get('mimetypes', []) + def fill_admin_form(self, form): WidgetField.fill_admin_form(self, form) - file_types = [ - ('audio/*', _('Sound files')), - ('video/*', _('Video files')), - ('image/*', _('Image files'))] - filetypes_cfg = get_cfg('filetypes', {}) - if filetypes_cfg: - for file_type in filetypes_cfg.values(): - file_types.append(( - ','.join(file_type['mimetypes']), file_type['label'])) - if self.file_type: - known_file_types = [x[0] for x in file_types] - for file_type in self.file_type: - if not file_type in known_file_types: - file_types.append((file_type, file_type)) - form.add(CheckboxesWidget, 'file_type', title=_('File type suggestion'), - value=self.file_type, elements=file_types, inline=True, - advanced=not(self.file_type)) + document_types = self.get_document_types() + cur_dt = self.document_type + # SingleSelectWidget compare the value and not the keys, so if we want + # the current value not to be hidden, we must reset it with the corresponding + # value from settings based on the 'id' + document_type_id = self.document_type.get('id') + if document_type_id in document_types \ + and self.document_type != document_types[document_type_id]: + self.document_type = document_types[document_type_id] + options = [(None, '---', {})] + options += [(doc_type, doc_type['label'], key) for key, doc_type in document_types.iteritems()] + form.add(SingleSelectWidget, 'document_type', title=_('File type suggestion'), + value=self.document_type, options=options, + advanced=not(self.document_type)) form.add(FileSizeWidget, 'max_file_size', title=('Max file size'), value=self.max_file_size, advanced=not(self.max_file_size)) def get_admin_attributes(self): - return WidgetField.get_admin_attributes(self) + ['file_type', - 'max_file_size'] + return WidgetField.get_admin_attributes(self) + ['document_type', + 'max_file_size'] def get_view_value(self, value): return htmltext('%s') % ( @@ -732,6 +738,75 @@ class FileField(WidgetField): if value and hasattr(value, 'token'): get_request().form[self.field_key + '$token'] = value.token + def get_document_types(self): + document_types = { + '_audio': { + 'label': _('Sound files'), + 'mimetypes': ['audio/*'], + }, + '_video': { + 'label': _('Video files'), + 'mimetypes': ['video/*'], + }, + '_image': { + 'label': _('Image files'), + 'mimetypes': ['image/*'], + } + } + # Local document types + document_types.update(get_cfg('filetypes', {})) + # Remote documents types + document_types.update(file_validation.get_document_types()) + for key, document_type in document_types.iteritems(): + document_type['id'] = key + # add current file type if it does not exist anymore in the settings + cur_dt = self.document_type + if cur_dt and cur_dt['id'] not in document_types: + document_types[cur_dt['id']] = cur_dt + return document_types + + def migrate(self): + if 'file_type' in self.__dict__: + self.document_type = {} + if self.__dict__['file_type']: + file_type = self.__dict__['file_type'] + document_types = self.get_document_types() + for key, value in document_types.iteritems(): + if self.file_type == value.get('mimetypes'): + self.document_type = value.copy() + self.document_type['id'] = key + else: + # self.file_type is a combination of file type, we create a + # virtual one from them + self.document_type = { + 'id': '_legacy', + 'label': ','.join(file_type), + 'mimetypes': file_type, + } + del self.__dict__['file_type'] + return True + return False + + def export_to_xml(self, charset, include_id=False): + # convert some sub-fields to strings as export_to_xml() only supports + # dictionnaries with strings values + if self.document_type and self.document_type.get('mimetypes'): + old_value = self.document_type['mimetypes'] + self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes']) + result = super(FileField, self).export_to_xml(charset, include_id=include_id) + if self.document_type and self.document_type.get('mimetypes'): + self.document_type['mimetypes'] = old_value + return result + + def init_with_xml(self, element, charset, include_id=False): + super(FileField, self).init_with_xml(element, charset, include_id=include_id) + # translate fields flattened to strings + if self.document_type and self.document_type.get('mimetypes'): + self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|') + if self.document_type and self.document_type.get('fargo'): + self.document_type['fargo'] = self.document_type['fargo'] == 'True' + + register_field_class(FileField) diff --git a/wcs/file_validation.py b/wcs/file_validation.py new file mode 100644 index 0000000..21e8dc5 --- /dev/null +++ b/wcs/file_validation.py @@ -0,0 +1,111 @@ +# w.c.s. - web application for online forms +# Copyright (C) 2005-2010 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import json +import urlparse +import hashlib +import urllib + +from qommon.misc import http_get_page, json_loads +from quixote import get_publisher, get_response +from quixote.html import htmltext + + +def has_file_validation(): + return get_publisher().get_site_option('fargo_url') is not None + +def fargo_get(path): + fargo_url = get_publisher().get_site_option('fargo_url') + url = urlparse.urljoin(fargo_url, path) + response, status, data, auth_header = http_get_page(url) + if status == 200: + return json_loads(data) + return None + +def sha256_of_upload(upload): + return hashlib.sha256(upload.get_content()).hexdigest() + +def get_document_types(): + if not has_file_validation(): + return {} + response = fargo_get('/document-types/') + publisher = get_publisher() + if response.get('err') == 0: + result = {} + for schema in response['data']: + d = { + 'id': schema['name'], + 'label': schema['label'], + 'fargo': True, + } + if 'mimetypes' in schema: + d['mimetypes'] = shema['mimetypes'] + result[d['id']] = d + + return result + return {} + +def validation_path(filled, field, upload): + user = filled.get_user() + if not user: + return None + if not user.name_identifiers: + return None + if not field.document_type or not field.document_type.get('fargo'): + return None + name_id = user.name_identifiers[0] + sha_256 = sha256_of_upload(upload) + document_type = field.document_type['id'] + path = '%s/%s/%s/' % ( + urllib.quote(name_id), + urllib.quote(sha_256), + urllib.quote(document_type), + ) + return path + +def validate_upload(filled, field, upload): + '''Check validation of the uploaded file with Fargo''' + path = validation_path(filled, field, upload) + if not path: + return None + response = fargo_get('metadata/' + path) + if response is None: + return None + if response['err'] == 1: + return False + return response['data'] + +def get_document_type_label(document_type): + return get_document_types().get(document_type, {}).get('label') + +def validation_link(filled, field, upload): + '''Compute link to Fargo to validate the given document''' + path = validation_path(filled, field, upload) + if not path: + return '' + get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css') + get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js']) + label = get_document_type_label(field.document_type['id']) + fargo_url = get_publisher().get_site_option('fargo_url') + url = urlparse.urljoin(fargo_url, 'validation/' + path) + next_url = get_publisher().get_frontoffice_url() + '/reload-top' + url += '?next=%s' % urllib.quote(next_url) + title = _('Validate as a %s') % label + return htmltext('%(title)s') % { + 'title': title, + 'url': url, + } diff --git a/wcs/forms/common.py b/wcs/forms/common.py index 06f0b7e..adb6619 100644 --- a/wcs/forms/common.py +++ b/wcs/forms/common.py @@ -15,12 +15,15 @@ # along with this program; if not, see . import sys +import hashlib +import urlparse from quixote import get_publisher, get_request, get_response, get_session, redirect from quixote.directory import Directory from quixote.html import TemplateIO, htmltext -from wcs.fields import WidgetField +from wcs.fields import WidgetField, FileField +from wcs import file_validation from qommon import template from qommon import get_logger @@ -395,12 +398,15 @@ class FormStatusPage(Directory): continue r += htmltext('
%s ') % f.label - r += htmltext('
') - s = f.get_view_value(value) - s = s.replace(str('[download]'), str('%sdownload' % form_url)) - r += s - r += htmltext('
') - + if isinstance(f, FileField): + r += htmltext(self.display_file_field(form_url, f, value)) + else: # normal display + r += htmltext('
') + s = f.get_view_value(value) + s = s.replace(str('[download]'), str('%sdownload' % form_url)) + r += s + r += htmltext('
') + r += htmltext('') if on_page: r += htmltext('') @@ -529,6 +535,45 @@ class FormStatusPage(Directory): else: return redirect('files/%s/' % fn) + def display_file_field(self, form_url, field, value): + r = TemplateIO(html=True) + status = None + if file_validation.has_file_validation(): + status = file_validation.validate_upload(self.filled, field, value) + if status is False: + extra_class = ' invalid' + elif status is None: + extra_class = '' + else: + extra_class = ' valid' + r += htmltext('
' % extra_class) + else: + r += htmltext('
') + s = field.get_view_value(value) + s = s.replace(str('[download]'), str('%sdownload' % form_url)) + r += s + if status is not None and get_request().is_in_backoffice(): + r += htmltext('
') + if status: + r += htmltext(_('

%s validated by %s on %s

')) % ( + status['label'], status['creator'], status['created']) + r += htmltext('
    ') + for meta in status['metadata']: + r += htmltext(_('
  • %(label)s: %(value)s
    • ')) % { + 'label': meta['label'], + 'value': meta['value'] + } + r += htmltext('
') + r += htmltext('

%s

') % (_('Valid from %(start)s to %(end)s') % { + 'start': status['start'], + 'end': status['end'], + }) + else: + r += file_validation.validation_link(self.filled, field, value) + r += htmltext('
') + r += htmltext('
') + return str(r) + def _q_lookup(self, component): if component == 'files': self.check_receiver() diff --git a/wcs/qommon/static/js/fargo.js b/wcs/qommon/static/js/fargo.js new file mode 100644 index 0000000..af79ead --- /dev/null +++ b/wcs/qommon/static/js/fargo.js @@ -0,0 +1,56 @@ + +$(function() { + var iframe = $(''); + var dialog = $("
").append(iframe).appendTo("body").dialog({ + autoOpen: false, + modal: true, + resizable: false, + width: "auto", + height: "auto", + close: function () { + iframe.attr("src", ""); + } + }); + $('.file-validation a').click(function (e) { + e.preventDefault(); + var src = $(e.target).attr('href'); + var title = $(e.target).data("title"); + var width = $(e.target).data("width"); + var height = $(e.target).data("height"); + iframe.attr({ + width: parseInt(width), + height: parseInt(height), + src: src + }); + dialog.dialog("option", "title", title); + dialog.dialog("open"); + }); + $('p.use-file-from-fargo span').click(function(e) { + e.preventDefault(); + var base_widget = $(this).parents('.file-upload-widget'); + document.fargo_set_token = function (token, title) { + if (token) { + $(base_widget).find('.filename').text(title); + $(base_widget).find('.fileinfo').show(); + $(base_widget).find('input[type=hidden]').val(token); + $(base_widget).find('input[type=file]').hide(); + } + document.fargo_close_dialog(); + } + document.fargo_close_dialog = function () { + document.fargo_set_token = undefined; + dialog.dialog('close'); + } + var src = $(this).data('src'); + var title = $(this).data("title"); + var width = $(this).data("width"); + var height = $(this).data("height"); + iframe.attr({ + width: parseInt(width), + height: parseInt(height), + src: src + }); + dialog.dialog("option", "title", title); + dialog.dialog("open"); + }); +}); diff --git a/wcs/root.py b/wcs/root.py index 854f172..9382c9d 100644 --- a/wcs/root.py +++ b/wcs/root.py @@ -192,7 +192,7 @@ class RootDirectory(Directory): _q_exports = ['admin', 'backoffice', 'forms', 'login', 'logout', 'saml', 'ident', 'register', 'afterjobs', 'themes', 'myspace', 'user', 'roles', 'pages', ('tmp-upload', 'tmp_upload'), 'api', '__version__', - 'tryauth', 'auth', 'preview'] + 'tryauth', 'auth', 'preview', ('reload-top', 'reload_top')] api = ApiDirectory() themes = template.ThemesDirectory() @@ -308,6 +308,10 @@ class RootDirectory(Directory): # or a form ? return forms.root.RootDirectory()._q_lookup(component) + def reload_top(self): + get_response().filter = {} + return htmltext('') + admin = None backoffice = None -- 2.1.4