From 4c0e4027fe36560b74f298dd894853b6bcd9f39c Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Mon, 28 Mar 2016 03:38:36 +0200 Subject: [PATCH 2/2] rewrite file validation (fixes #10444) - metadata fields are now directly added to the form, as part of the FileWithPreviewWidget sub-widgets (it's a CompositeWidget) - existing validated metadatas are proposed for prefilling - if prefilled no file is uploadee, and a special upload class NoUpload is used instead of a PicklableUpload, it only contains the metadatas. - prefilled fields are disabled, - on upload of a new file all prefilled fields are emptied and re-enabled. --- wcs/fields.py | 17 ++++- wcs/file_validation.py | 114 ++++++++++++++++-------------- wcs/forms/common.py | 55 ++++++++------ wcs/qommon/form.py | 62 +++++++++++++++- wcs/qommon/static/css/qommon.css | 10 +++ wcs/qommon/static/js/qommon.fileupload.js | 43 ++++++++++- 6 files changed, 220 insertions(+), 81 deletions(-) diff --git a/wcs/fields.py b/wcs/fields.py index 02edab9..8ecee8a 100644 --- a/wcs/fields.py +++ b/wcs/fields.py @@ -720,6 +720,10 @@ class FileField(WidgetField): self.document_type = self.document_type or {} @property + def metadata(self): + return self.document_type.get('metadata', []) + + @property def file_type(self): return (self.document_type or {}).get('mimetypes', []) @@ -752,8 +756,14 @@ class FileField(WidgetField): 'document_type', 'max_file_size', 'allow_portfolio_picking'] def get_view_value(self, value): - return htmltext('%s') % ( + r = TemplateIO(html=True) + if not getattr(value, 'no_file', False): + r += htmltext('%s') % ( value.base_filename, self.id, value) + for meta_field in self.metadata: + metadata_value = getattr(value, 'metadata', {}).get(meta_field['name'], '') + r += htmltext('

%s : %s

') % (meta_field['label'], metadata_value) + return r.getvalue() def get_csv_value(self, value, hint=None): if not value: @@ -788,6 +798,7 @@ class FileField(WidgetField): value = get_request().get_field(self.field_key) if value and hasattr(value, 'token'): get_request().form[self.field_key + '$token'] = value.token + kwargs['document_type'] = self.document_type def get_document_types(self): document_types = { @@ -867,6 +878,10 @@ class FileField(WidgetField): if self.document_type and self.document_type.get('fargo'): self.document_type['fargo'] = self.document_type['fargo'] == 'True' + def store_structured_value(self, data, field_id): + value = data.get(field_id) + return getattr(value, 'metadata', {}) + register_field_class(FileField) diff --git a/wcs/file_validation.py b/wcs/file_validation.py index 21e8dc5..d1c8fbd 100644 --- a/wcs/file_validation.py +++ b/wcs/file_validation.py @@ -19,14 +19,14 @@ import urlparse import hashlib import urllib -from qommon.misc import http_get_page, json_loads -from quixote import get_publisher, get_response -from quixote.html import htmltext +from qommon.misc import http_get_page, json_loads, http_post_request +from quixote import get_publisher, get_request def has_file_validation(): return get_publisher().get_site_option('fargo_url') is not None + def fargo_get(path): fargo_url = get_publisher().get_site_option('fargo_url') url = urlparse.urljoin(fargo_url, path) @@ -35,14 +35,15 @@ def fargo_get(path): return json_loads(data) return None + def sha256_of_upload(upload): return hashlib.sha256(upload.get_content()).hexdigest() + def get_document_types(): if not has_file_validation(): return {} response = fargo_get('/document-types/') - publisher = get_publisher() if response.get('err') == 0: result = {} for schema in response['data']: @@ -52,60 +53,65 @@ def get_document_types(): 'fargo': True, } if 'mimetypes' in schema: - d['mimetypes'] = shema['mimetypes'] + d['mimetypes'] = schema['mimetypes'] result[d['id']] = d - + d['metadata'] = schema.get('metadata', []) return result return {} -def validation_path(filled, field, upload): - user = filled.get_user() - if not user: - return None - if not user.name_identifiers: - return None - if not field.document_type or not field.document_type.get('fargo'): - return None - name_id = user.name_identifiers[0] - sha_256 = sha256_of_upload(upload) - document_type = field.document_type['id'] - path = '%s/%s/%s/' % ( - urllib.quote(name_id), - urllib.quote(sha_256), - urllib.quote(document_type), - ) - return path - -def validate_upload(filled, field, upload): + +def get_validation(url): + response, status, data, auth_header = http_get_page(url) + if status == 200: + return json_loads(data)['data'] + return None + + +def get_validations(document_type): + request = get_request() + document_type_id = document_type['id'] + qs = {} + if not request.user: + return [] + if request.user.name_identifiers: + qs['user_nameid'] = request.user.name_identifiers[0] + elif request.user.email: + qs['user_email'] = request.user.email + else: + return [] + path = 'api/validation/%s/' % urllib.quote(document_type_id) + validations = fargo_get(path) + if validations and validations.get('data', {}).get('results', []): + return validations['data']['results'] + return [] + + +def is_valid(filled, field, upload): '''Check validation of the uploaded file with Fargo''' - path = validation_path(filled, field, upload) - if not path: - return None - response = fargo_get('metadata/' + path) - if response is None: - return None - if response['err'] == 1: - return False - return response['data'] - -def get_document_type_label(document_type): - return get_document_types().get(document_type, {}).get('label') - -def validation_link(filled, field, upload): + return 'url' in getattr(upload, 'metadata', {}) + + +def validate(filled, field, upload): '''Compute link to Fargo to validate the given document''' - path = validation_path(filled, field, upload) - if not path: - return '' - get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css') - get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js']) - label = get_document_type_label(field.document_type['id']) + document_type_id = field.document_type['id'] + path = 'api/validation/%s/' % urllib.quote(document_type_id) fargo_url = get_publisher().get_site_option('fargo_url') - url = urlparse.urljoin(fargo_url, 'validation/' + path) - next_url = get_publisher().get_frontoffice_url() + '/reload-top' - url += '?next=%s' % urllib.quote(next_url) - title = _('Validate as a %s') % label - return htmltext('%(title)s') % { - 'title': title, - 'url': url, - } + url = urlparse.urljoin(fargo_url, path) + payload = {} + if filled.user: + if filled.user.name_identifiers: + payload['user_nameid'] = filled.user.name_identifiers[0] + else: + payload['user_email'] = filled.user.email + payload['origin'] = get_request().get_server() + payload['creator'] = get_request().user.display_name + payload['content_hash'] = sha256_of_upload(upload) + for meta_field in field.metadata: + payload[meta_field['name']] = upload.metadata.get(meta_field['name'], '') + headers = {'Content-type': 'application/json'} + response, status, response_payload, auth_header = http_post_request(url, json.dumps(payload), + headers=headers) + if status == 201: + upload.metadata = json_loads(response_payload)['data'] + filled.data['%s_structured' % field.id] = upload.metadata + filled.store() diff --git a/wcs/forms/common.py b/wcs/forms/common.py index 4195307..dd48c86 100644 --- a/wcs/forms/common.py +++ b/wcs/forms/common.py @@ -97,6 +97,25 @@ class FormStatusPage(Directory): def html_top(self, title = None): template.html_top(title = title, default_org = _('Forms')) + def validate(self): + if not file_validation.has_file_validation(): + return redirect('.') + field_id = get_request().form.get('field_id') + if not field_id: + return redirect('.') + for field in self.formdef.fields: + if field.id == field_id: + break + else: + return redirect('.') + if field.key != 'file': + return redirect('.') + if not field.document_type.get('fargo', False): + return redirect('.') + value = self.filled.data.get(field_id) + file_validation.validate(self.filled, field, value) + return redirect('.') + def __init__(self, formdef, filled, register_workflow_subdirs=True): get_publisher().substitutions.feed(filled) self.formdef = formdef @@ -612,13 +631,12 @@ class FormStatusPage(Directory): def display_file_field(self, form_url, field, value): r = TemplateIO(html=True) - status = None - if file_validation.has_file_validation(): - status = file_validation.validate_upload(self.filled, field, value) - if status is False: + validated = None + is_fargo_dt = field.document_type.get('fargo', False) + if file_validation.has_file_validation() and is_fargo_dt: + validated = file_validation.is_valid(self.filled, field, value) + if validated is False: extra_class = ' invalid' - elif status is None: - extra_class = '' else: extra_class = ' valid' r += htmltext('
' % extra_class) @@ -627,26 +645,19 @@ class FormStatusPage(Directory): s = field.get_view_value(value) s = s.replace(str('[download]'), str('%sdownload' % form_url)) r += s - if status is not None and get_request().is_in_backoffice(): - if status is False: - r += htmltext(' %s') % _('not validated') + if validated is not None and get_request().is_in_backoffice(): r += htmltext('
') - if status: - r += htmltext('

%s

') % _( - '%(label)s validated by %(creator)s on %(created)s') % status - r += htmltext('
    ') - for meta in status['metadata']: - r += htmltext(_('
  • %(label)s: %(value)s
    • ')) % { - 'label': meta['label'], - 'value': meta['value'] - } - r += htmltext('
') + if validated: + r += htmltext('

%s

') % _( + 'validated by %(creator)s on %(created)s') % value.metadata r += htmltext('

%s

') % (_('Valid from %(start)s to %(end)s') % { - 'start': status['start'], - 'end': status['end'], + 'start': value.metadata['start'], + 'end': value.metadata['end'], }) else: - r += file_validation.validation_link(self.filled, field, value) + r += htmltext('
' + '
') % ( + field.id, _('Validate')) r += htmltext('
') r += htmltext('
') return str(r) diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py index 2ea1bc4..7d24f88 100644 --- a/wcs/qommon/form.py +++ b/wcs/qommon/form.py @@ -59,6 +59,7 @@ from quixote.util import randbytes import misc from strftime import strftime from publisher import get_cfg +from wcs import file_validation QuixoteForm = Form @@ -581,6 +582,14 @@ class UploadWidget(CompositeWidget): self.value = None +class NoUpload(object): + no_file = True + metadata = None + + def __init__(self, validation_url): + self.metadata = file_validation.get_validation(validation_url) + + class FileWithPreviewWidget(CompositeWidget): """Widget that proposes a File Upload widget but that stores the file ondisk so it has a "readonly" mode where the filename is shown.""" @@ -591,9 +600,13 @@ class FileWithPreviewWidget(CompositeWidget): max_file_size_bytes = None # will be filled automatically + PARTIAL_FILL_ERROR = _('This field is normally not required, but you must leave it completely ' + 'empty or fill it, You cannot fill it partially') + def __init__(self, name, value=None, **kwargs): CompositeWidget.__init__(self, name, value, **kwargs) self.value = value + self.document_type = kwargs.pop('document_type', None) or {} self.preview = kwargs.get('readonly') self.max_file_size = kwargs.pop('max_file_size', None) self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True) @@ -609,6 +622,24 @@ class FileWithPreviewWidget(CompositeWidget): # this could be used for client size validation of file size attrs['data-max-file-size'] = str(self.max_file_size_bytes) self.add(FileWidget, 'file', render_br=False, attrs=attrs) + if self.document_type.get('metadata'): + if self.preview: + self.add(HiddenWidget, 'validation_url') + else: + validations = file_validation.get_validations(self.document_type) + if validations: + options = [('', _('Known documents'), '')] + options += [(v['url'], v['display'], v['url']) for v in validations] + self.add(SingleSelectWidget, 'validation_url', options=options, + attrs={'data-validations': json.dumps(validations)}) + for meta_field in self.metadata: + subvalue = getattr(value, 'metadata', {}).get(meta_field['name']) + self.add(StringWidget, meta_field['name'], + title=meta_field['label'], + required=meta_field.get('required', True) and self.required, + readonly=self.preview, + value=subvalue) + self.get_widget(meta_field['name']).extra_css_class = 'subwidget' if value: self.set_value(value) @@ -651,10 +682,17 @@ class FileWithPreviewWidget(CompositeWidget): r += htmltext('') return r.getvalue() + @property + def metadata(self): + return self.document_type.get('metadata', []) + def _parse(self, request): self.value = None if self.get('token'): token = self.get('token') + elif self.get('validation_url'): + self.value = NoUpload(self.get('validation_url')) + return elif self.get('file'): token = get_session().add_tempfile(self.get('file')) request.form[self.get_widget('token').get_name()] = token @@ -663,13 +701,33 @@ class FileWithPreviewWidget(CompositeWidget): session = get_session() if token and session.tempfiles and session.tempfiles.has_key(token): - temp = session.tempfiles[token] self.value = session.get_tempfile_content(token) if self.value is None: - # there's no file, the other checks are irrelevant. + # there's no file, check all metadata field are empty too + # if not file and required metadata field become required + if any([self.get(meta_field['name']) for meta_field in self.metadata]): + self.set_error(self.PARTIAL_FILL_ERROR) + self.get_widget('file').set_error(self.REQUIRED_ERROR) + for meta_field in self.metadata: + required = meta_field.get('required', True) + if required and not self.get(meta_field['name']): + self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR) return + # There is some file, check all required metadata files have been filled + for meta_field in self.metadata: + required = meta_field.get('required', True) + if required and not self.get(meta_field['name']): + self.set_error(self.PARTIAL_FILL_ERROR) + self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR) + + + if self.metadata: + self.value.metadata = {} + for meta_field in self.metadata: + self.value.metadata[meta_field['name']] = self.get(meta_field['name']) + # Don't trust the browser supplied MIME type, update the Upload object # with a MIME type created with magic (or based on the extension if the # module is missing). diff --git a/wcs/qommon/static/css/qommon.css b/wcs/qommon/static/css/qommon.css index 34f1c5e..a9df4d8 100644 --- a/wcs/qommon/static/css/qommon.css +++ b/wcs/qommon/static/css/qommon.css @@ -79,6 +79,16 @@ div.SubmitWidget input, input[type=submit] { vertical-align: middle; } +/* nested widgets */ +.widget .subwidget { + padding-left: 2em; +} +div.FileWithPreviewWidget .validation { + display: inline-block; + font-size: xx-small; + margin-right: 2em; +} + div.form .title, form.quixote .title { font-weight: bold; } diff --git a/wcs/qommon/static/js/qommon.fileupload.js b/wcs/qommon/static/js/qommon.fileupload.js index cc6ee8c..3f062af 100644 --- a/wcs/qommon/static/js/qommon.fileupload.js +++ b/wcs/qommon/static/js/qommon.fileupload.js @@ -6,6 +6,16 @@ $(function() { } else { $(base_widget).find('.fileinfo').hide(); } + function disable() { + base_widget.find('.subwidget input').prop('disabled', true); + $('form').on('submit', function () { + $('input').prop('disabled', false); + }); + } + function enable() { + base_widget.find('.subwidget input').val(''); + base_widget.find('.subwidget input').prop('disabled', false); + } $(this).find('input[type=file]').fileupload({ dataType: 'json', add: function (e, data) { @@ -19,7 +29,9 @@ $(function() { $(base_widget).find('.fileprogress').hide(); $(base_widget).find('.filename').text(data.result[0].name); $(base_widget).find('.fileinfo').show(); - $(base_widget).find('input[type=hidden]').val(data.result[0].token); + $(base_widget).find('input[name$="$token"]').val(data.result[0].token); + $(base_widget).find('input[name$="$validation_url"]').val(''); + enable(); $(base_widget).parents('form').find('input[name=submit]').prop('disabled', false); $(this).hide(); }, @@ -29,7 +41,7 @@ $(function() { } }); $(this).find('a.remove').click(function() { - $(base_widget).find('input[type=hidden]').val(''); + $(base_widget).find('input[name$="$token"]').val(''); $(base_widget).find('.fileinfo').hide(); $(base_widget).find('input[type=file]').show(); return false; @@ -38,5 +50,32 @@ $(function() { $(base_widget).find('input[type=file]').click(); return false; }); + if ($(this).find('select[name$="$validation_url"] option:selected').val()) { + disable(); + } + $(this).find('select[name$="$validation_url"]').on('change', function() { + var url = $(this).find(':selected').val(); + if (url) { + var validations = $(this).data('validations'); + + for (var i = 0; i < validations.length; i++) { + if (validations[i].url == url) { + base_widget.find('a.remove').trigger('click'); + for (var item in validations[i]) { + if (! item) { + continue; + } + var $input = base_widget.find('input[name$="$' + item + '"]'); + if ($input.length) { + $input.val(validations[i][item]); + } + } + disable(); + } + } + } else { + enable(); + } + }); }); }); -- 2.1.4