0002-rewrite-file-validation-fixes-10444.patch
wcs/fields.py | ||
---|---|---|
720 | 720 |
self.document_type = self.document_type or {} |
721 | 721 | |
722 | 722 |
@property |
723 |
def metadata(self): |
|
724 |
return self.document_type.get('metadata', []) |
|
725 | ||
726 |
@property |
|
723 | 727 |
def file_type(self): |
724 | 728 |
return (self.document_type or {}).get('mimetypes', []) |
725 | 729 | |
... | ... | |
752 | 756 |
'document_type', 'max_file_size', 'allow_portfolio_picking'] |
753 | 757 | |
754 | 758 |
def get_view_value(self, value): |
755 |
return htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % ( |
|
759 |
r = TemplateIO(html=True) |
|
760 |
if not getattr(value, 'no_file', False): |
|
761 |
r += htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % ( |
|
756 | 762 |
value.base_filename, self.id, value) |
763 |
for meta_field in self.metadata: |
|
764 |
metadata_value = getattr(value, 'metadata', {}).get(meta_field['name'], '') |
|
765 |
r += htmltext('<p>%s : %s</p>') % (meta_field['label'], metadata_value) |
|
766 |
return r.getvalue() |
|
757 | 767 | |
758 | 768 |
def get_csv_value(self, value, hint=None): |
759 | 769 |
if not value: |
... | ... | |
788 | 798 |
value = get_request().get_field(self.field_key) |
789 | 799 |
if value and hasattr(value, 'token'): |
790 | 800 |
get_request().form[self.field_key + '$token'] = value.token |
801 |
kwargs['document_type'] = self.document_type |
|
791 | 802 | |
792 | 803 |
def get_document_types(self): |
793 | 804 |
document_types = { |
... | ... | |
867 | 878 |
if self.document_type and self.document_type.get('fargo'): |
868 | 879 |
self.document_type['fargo'] = self.document_type['fargo'] == 'True' |
869 | 880 | |
881 |
def store_structured_value(self, data, field_id): |
|
882 |
value = data.get(field_id) |
|
883 |
return getattr(value, 'metadata', {}) |
|
884 | ||
870 | 885 | |
871 | 886 |
register_field_class(FileField) |
872 | 887 |
wcs/file_validation.py | ||
---|---|---|
19 | 19 |
import hashlib |
20 | 20 |
import urllib |
21 | 21 | |
22 |
from qommon.misc import http_get_page, json_loads |
|
23 |
from quixote import get_publisher, get_response |
|
24 |
from quixote.html import htmltext |
|
22 |
from qommon.misc import http_get_page, json_loads, http_post_request |
|
23 |
from quixote import get_publisher, get_request |
|
25 | 24 | |
26 | 25 | |
27 | 26 |
def has_file_validation(): |
28 | 27 |
return get_publisher().get_site_option('fargo_url') is not None |
29 | 28 | |
29 | ||
30 | 30 |
def fargo_get(path): |
31 | 31 |
fargo_url = get_publisher().get_site_option('fargo_url') |
32 | 32 |
url = urlparse.urljoin(fargo_url, path) |
... | ... | |
35 | 35 |
return json_loads(data) |
36 | 36 |
return None |
37 | 37 | |
38 | ||
38 | 39 |
def sha256_of_upload(upload): |
39 | 40 |
return hashlib.sha256(upload.get_content()).hexdigest() |
40 | 41 | |
42 | ||
41 | 43 |
def get_document_types(): |
42 | 44 |
if not has_file_validation(): |
43 | 45 |
return {} |
44 | 46 |
response = fargo_get('/document-types/') |
45 |
publisher = get_publisher() |
|
46 | 47 |
if response.get('err') == 0: |
47 | 48 |
result = {} |
48 | 49 |
for schema in response['data']: |
... | ... | |
52 | 53 |
'fargo': True, |
53 | 54 |
} |
54 | 55 |
if 'mimetypes' in schema: |
55 |
d['mimetypes'] = shema['mimetypes'] |
|
56 |
d['mimetypes'] = schema['mimetypes']
|
|
56 | 57 |
result[d['id']] = d |
57 | ||
58 |
d['metadata'] = schema.get('metadata', []) |
|
58 | 59 |
return result |
59 | 60 |
return {} |
60 | 61 | |
61 |
def validation_path(filled, field, upload): |
|
62 |
user = filled.get_user() |
|
63 |
if not user: |
|
64 |
return None |
|
65 |
if not user.name_identifiers: |
|
66 |
return None |
|
67 |
if not field.document_type or not field.document_type.get('fargo'): |
|
68 |
return None |
|
69 |
name_id = user.name_identifiers[0] |
|
70 |
sha_256 = sha256_of_upload(upload) |
|
71 |
document_type = field.document_type['id'] |
|
72 |
path = '%s/%s/%s/' % ( |
|
73 |
urllib.quote(name_id), |
|
74 |
urllib.quote(sha_256), |
|
75 |
urllib.quote(document_type), |
|
76 |
) |
|
77 |
return path |
|
78 | ||
79 |
def validate_upload(filled, field, upload): |
|
62 | ||
63 |
def get_validation(url): |
|
64 |
response, status, data, auth_header = http_get_page(url) |
|
65 |
if status == 200: |
|
66 |
return json_loads(data)['data'] |
|
67 |
return None |
|
68 | ||
69 | ||
70 |
def get_validations(document_type): |
|
71 |
request = get_request() |
|
72 |
document_type_id = document_type['id'] |
|
73 |
qs = {} |
|
74 |
if not request.user: |
|
75 |
return [] |
|
76 |
if request.user.name_identifiers: |
|
77 |
qs['user_nameid'] = request.user.name_identifiers[0] |
|
78 |
elif request.user.email: |
|
79 |
qs['user_email'] = request.user.email |
|
80 |
else: |
|
81 |
return [] |
|
82 |
path = 'api/validation/%s/' % urllib.quote(document_type_id) |
|
83 |
validations = fargo_get(path) |
|
84 |
if validations and validations.get('data', {}).get('results', []): |
|
85 |
return validations['data']['results'] |
|
86 |
return [] |
|
87 | ||
88 | ||
89 |
def is_valid(filled, field, upload): |
|
80 | 90 |
'''Check validation of the uploaded file with Fargo''' |
81 |
path = validation_path(filled, field, upload) |
|
82 |
if not path: |
|
83 |
return None |
|
84 |
response = fargo_get('metadata/' + path) |
|
85 |
if response is None: |
|
86 |
return None |
|
87 |
if response['err'] == 1: |
|
88 |
return False |
|
89 |
return response['data'] |
|
90 | ||
91 |
def get_document_type_label(document_type): |
|
92 |
return get_document_types().get(document_type, {}).get('label') |
|
93 | ||
94 |
def validation_link(filled, field, upload): |
|
91 |
return 'url' in getattr(upload, 'metadata', {}) |
|
92 | ||
93 | ||
94 |
def validate(filled, field, upload): |
|
95 | 95 |
'''Compute link to Fargo to validate the given document''' |
96 |
path = validation_path(filled, field, upload) |
|
97 |
if not path: |
|
98 |
return '' |
|
99 |
get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css') |
|
100 |
get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js']) |
|
101 |
label = get_document_type_label(field.document_type['id']) |
|
96 |
document_type_id = field.document_type['id'] |
|
97 |
path = 'api/validation/%s/' % urllib.quote(document_type_id) |
|
102 | 98 |
fargo_url = get_publisher().get_site_option('fargo_url') |
103 |
url = urlparse.urljoin(fargo_url, 'validation/' + path) |
|
104 |
next_url = get_publisher().get_frontoffice_url() + '/reload-top' |
|
105 |
url += '?next=%s' % urllib.quote(next_url) |
|
106 |
title = _('Validate as a %s') % label |
|
107 |
return htmltext('<a data-title="%(title)s" data-width="800" ' |
|
108 |
'data-height="500" href="%(url)s">%(title)s</a>') % { |
|
109 |
'title': title, |
|
110 |
'url': url, |
|
111 |
} |
|
99 |
url = urlparse.urljoin(fargo_url, path) |
|
100 |
payload = {} |
|
101 |
if filled.user: |
|
102 |
if filled.user.name_identifiers: |
|
103 |
payload['user_nameid'] = filled.user.name_identifiers[0] |
|
104 |
else: |
|
105 |
payload['user_email'] = filled.user.email |
|
106 |
payload['origin'] = get_request().get_server() |
|
107 |
payload['creator'] = get_request().user.display_name |
|
108 |
payload['content_hash'] = sha256_of_upload(upload) |
|
109 |
for meta_field in field.metadata: |
|
110 |
payload[meta_field['name']] = upload.metadata.get(meta_field['name'], '') |
|
111 |
headers = {'Content-type': 'application/json'} |
|
112 |
response, status, response_payload, auth_header = http_post_request(url, json.dumps(payload), |
|
113 |
headers=headers) |
|
114 |
if status == 201: |
|
115 |
upload.metadata = json_loads(response_payload)['data'] |
|
116 |
filled.data['%s_structured' % field.id] = upload.metadata |
|
117 |
filled.store() |
wcs/forms/common.py | ||
---|---|---|
97 | 97 |
def html_top(self, title = None): |
98 | 98 |
template.html_top(title = title, default_org = _('Forms')) |
99 | 99 | |
100 |
def validate(self): |
|
101 |
if not file_validation.has_file_validation(): |
|
102 |
return redirect('.') |
|
103 |
field_id = get_request().form.get('field_id') |
|
104 |
if not field_id: |
|
105 |
return redirect('.') |
|
106 |
for field in self.formdef.fields: |
|
107 |
if field.id == field_id: |
|
108 |
break |
|
109 |
else: |
|
110 |
return redirect('.') |
|
111 |
if field.key != 'file': |
|
112 |
return redirect('.') |
|
113 |
if not field.document_type.get('fargo', False): |
|
114 |
return redirect('.') |
|
115 |
value = self.filled.data.get(field_id) |
|
116 |
file_validation.validate(self.filled, field, value) |
|
117 |
return redirect('.') |
|
118 | ||
100 | 119 |
def __init__(self, formdef, filled, register_workflow_subdirs=True): |
101 | 120 |
get_publisher().substitutions.feed(filled) |
102 | 121 |
self.formdef = formdef |
... | ... | |
612 | 631 | |
613 | 632 |
def display_file_field(self, form_url, field, value): |
614 | 633 |
r = TemplateIO(html=True) |
615 |
status = None |
|
616 |
if file_validation.has_file_validation(): |
|
617 |
status = file_validation.validate_upload(self.filled, field, value) |
|
618 |
if status is False: |
|
634 |
validated = None |
|
635 |
is_fargo_dt = field.document_type.get('fargo', False) |
|
636 |
if file_validation.has_file_validation() and is_fargo_dt: |
|
637 |
validated = file_validation.is_valid(self.filled, field, value) |
|
638 |
if validated is False: |
|
619 | 639 |
extra_class = ' invalid' |
620 |
elif status is None: |
|
621 |
extra_class = '' |
|
622 | 640 |
else: |
623 | 641 |
extra_class = ' valid' |
624 | 642 |
r += htmltext('<div class="value%s">' % extra_class) |
... | ... | |
627 | 645 |
s = field.get_view_value(value) |
628 | 646 |
s = s.replace(str('[download]'), str('%sdownload' % form_url)) |
629 | 647 |
r += s |
630 |
if status is not None and get_request().is_in_backoffice(): |
|
631 |
if status is False: |
|
632 |
r += htmltext(' <span class="validation-status">%s</span>') % _('not validated') |
|
648 |
if validated is not None and get_request().is_in_backoffice(): |
|
633 | 649 |
r += htmltext('<div class="file-validation">') |
634 |
if status: |
|
635 |
r += htmltext('<p class="validation-status">%s</p>') % _( |
|
636 |
'%(label)s validated by %(creator)s on %(created)s') % status |
|
637 |
r += htmltext('<ul>') |
|
638 |
for meta in status['metadata']: |
|
639 |
r += htmltext(_('<li>%(label)s: %(value)s</li>')) % { |
|
640 |
'label': meta['label'], |
|
641 |
'value': meta['value'] |
|
642 |
} |
|
643 |
r += htmltext('</ul>') |
|
650 |
if validated: |
|
651 |
r += htmltext('<p class="validation-validated">%s</p>') % _( |
|
652 |
'validated by %(creator)s on %(created)s') % value.metadata |
|
644 | 653 |
r += htmltext('<p>%s</p>') % (_('Valid from %(start)s to %(end)s') % { |
645 |
'start': status['start'],
|
|
646 |
'end': status['end'],
|
|
654 |
'start': value.metadata['start'],
|
|
655 |
'end': value.metadata['end'],
|
|
647 | 656 |
}) |
648 | 657 |
else: |
649 |
r += file_validation.validation_link(self.filled, field, value) |
|
658 |
r += htmltext('<form method="post" action="./validate?field_id=%s">' |
|
659 |
'<button>%s</button></form>') % ( |
|
660 |
field.id, _('Validate')) |
|
650 | 661 |
r += htmltext('</div>') |
651 | 662 |
r += htmltext('</div>') |
652 | 663 |
return str(r) |
wcs/qommon/form.py | ||
---|---|---|
59 | 59 |
import misc |
60 | 60 |
from strftime import strftime |
61 | 61 |
from publisher import get_cfg |
62 |
from wcs import file_validation |
|
62 | 63 | |
63 | 64 |
QuixoteForm = Form |
64 | 65 | |
... | ... | |
581 | 582 |
self.value = None |
582 | 583 | |
583 | 584 | |
585 |
class NoUpload(object): |
|
586 |
no_file = True |
|
587 |
metadata = None |
|
588 | ||
589 |
def __init__(self, validation_url): |
|
590 |
self.metadata = file_validation.get_validation(validation_url) |
|
591 | ||
592 | ||
584 | 593 |
class FileWithPreviewWidget(CompositeWidget): |
585 | 594 |
"""Widget that proposes a File Upload widget but that stores the file |
586 | 595 |
ondisk so it has a "readonly" mode where the filename is shown.""" |
... | ... | |
591 | 600 | |
592 | 601 |
max_file_size_bytes = None # will be filled automatically |
593 | 602 | |
603 |
PARTIAL_FILL_ERROR = _('This field is normally not required, but you must leave it completely ' |
|
604 |
'empty or fill it, You cannot fill it partially') |
|
605 | ||
594 | 606 |
def __init__(self, name, value=None, **kwargs): |
595 | 607 |
CompositeWidget.__init__(self, name, value, **kwargs) |
596 | 608 |
self.value = value |
609 |
self.document_type = kwargs.pop('document_type', None) or {} |
|
597 | 610 |
self.preview = kwargs.get('readonly') |
598 | 611 |
self.max_file_size = kwargs.pop('max_file_size', None) |
599 | 612 |
self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True) |
... | ... | |
609 | 622 |
# this could be used for client size validation of file size |
610 | 623 |
attrs['data-max-file-size'] = str(self.max_file_size_bytes) |
611 | 624 |
self.add(FileWidget, 'file', render_br=False, attrs=attrs) |
625 |
if self.document_type.get('metadata'): |
|
626 |
if self.preview: |
|
627 |
self.add(HiddenWidget, 'validation_url') |
|
628 |
else: |
|
629 |
validations = file_validation.get_validations(self.document_type) |
|
630 |
if validations: |
|
631 |
options = [('', _('Known documents'), '')] |
|
632 |
options += [(v['url'], v['display'], v['url']) for v in validations] |
|
633 |
self.add(SingleSelectWidget, 'validation_url', options=options, |
|
634 |
attrs={'data-validations': json.dumps(validations)}) |
|
635 |
for meta_field in self.metadata: |
|
636 |
subvalue = getattr(value, 'metadata', {}).get(meta_field['name']) |
|
637 |
self.add(StringWidget, meta_field['name'], |
|
638 |
title=meta_field['label'], |
|
639 |
required=meta_field.get('required', True) and self.required, |
|
640 |
readonly=self.preview, |
|
641 |
value=subvalue) |
|
642 |
self.get_widget(meta_field['name']).extra_css_class = 'subwidget' |
|
612 | 643 |
if value: |
613 | 644 |
self.set_value(value) |
614 | 645 | |
... | ... | |
651 | 682 |
r += htmltext('</div>') |
652 | 683 |
return r.getvalue() |
653 | 684 | |
685 |
@property |
|
686 |
def metadata(self): |
|
687 |
return self.document_type.get('metadata', []) |
|
688 | ||
654 | 689 |
def _parse(self, request): |
655 | 690 |
self.value = None |
656 | 691 |
if self.get('token'): |
657 | 692 |
token = self.get('token') |
693 |
elif self.get('validation_url'): |
|
694 |
self.value = NoUpload(self.get('validation_url')) |
|
695 |
return |
|
658 | 696 |
elif self.get('file'): |
659 | 697 |
token = get_session().add_tempfile(self.get('file')) |
660 | 698 |
request.form[self.get_widget('token').get_name()] = token |
... | ... | |
663 | 701 | |
664 | 702 |
session = get_session() |
665 | 703 |
if token and session.tempfiles and session.tempfiles.has_key(token): |
666 |
temp = session.tempfiles[token] |
|
667 | 704 |
self.value = session.get_tempfile_content(token) |
668 | 705 | |
669 | 706 |
if self.value is None: |
670 |
# there's no file, the other checks are irrelevant. |
|
707 |
# there's no file, check all metadata field are empty too |
|
708 |
# if not file and required metadata field become required |
|
709 |
if any([self.get(meta_field['name']) for meta_field in self.metadata]): |
|
710 |
self.set_error(self.PARTIAL_FILL_ERROR) |
|
711 |
self.get_widget('file').set_error(self.REQUIRED_ERROR) |
|
712 |
for meta_field in self.metadata: |
|
713 |
required = meta_field.get('required', True) |
|
714 |
if required and not self.get(meta_field['name']): |
|
715 |
self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR) |
|
671 | 716 |
return |
672 | 717 | |
718 |
# There is some file, check all required metadata files have been filled |
|
719 |
for meta_field in self.metadata: |
|
720 |
required = meta_field.get('required', True) |
|
721 |
if required and not self.get(meta_field['name']): |
|
722 |
self.set_error(self.PARTIAL_FILL_ERROR) |
|
723 |
self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR) |
|
724 | ||
725 | ||
726 |
if self.metadata: |
|
727 |
self.value.metadata = {} |
|
728 |
for meta_field in self.metadata: |
|
729 |
self.value.metadata[meta_field['name']] = self.get(meta_field['name']) |
|
730 | ||
673 | 731 |
# Don't trust the browser supplied MIME type, update the Upload object |
674 | 732 |
# with a MIME type created with magic (or based on the extension if the |
675 | 733 |
# module is missing). |
wcs/qommon/static/css/qommon.css | ||
---|---|---|
79 | 79 |
vertical-align: middle; |
80 | 80 |
} |
81 | 81 | |
82 |
/* nested widgets */ |
|
83 |
.widget .subwidget { |
|
84 |
padding-left: 2em; |
|
85 |
} |
|
86 |
div.FileWithPreviewWidget .validation { |
|
87 |
display: inline-block; |
|
88 |
font-size: xx-small; |
|
89 |
margin-right: 2em; |
|
90 |
} |
|
91 | ||
82 | 92 |
div.form .title, form.quixote .title { |
83 | 93 |
font-weight: bold; |
84 | 94 |
} |
wcs/qommon/static/js/qommon.fileupload.js | ||
---|---|---|
6 | 6 |
} else { |
7 | 7 |
$(base_widget).find('.fileinfo').hide(); |
8 | 8 |
} |
9 |
function disable() { |
|
10 |
base_widget.find('.subwidget input').prop('disabled', true); |
|
11 |
$('form').on('submit', function () { |
|
12 |
$('input').prop('disabled', false); |
|
13 |
}); |
|
14 |
} |
|
15 |
function enable() { |
|
16 |
base_widget.find('.subwidget input').val(''); |
|
17 |
base_widget.find('.subwidget input').prop('disabled', false); |
|
18 |
} |
|
9 | 19 |
$(this).find('input[type=file]').fileupload({ |
10 | 20 |
dataType: 'json', |
11 | 21 |
add: function (e, data) { |
... | ... | |
19 | 29 |
$(base_widget).find('.fileprogress').hide(); |
20 | 30 |
$(base_widget).find('.filename').text(data.result[0].name); |
21 | 31 |
$(base_widget).find('.fileinfo').show(); |
22 |
$(base_widget).find('input[type=hidden]').val(data.result[0].token); |
|
32 |
$(base_widget).find('input[name$="$token"]').val(data.result[0].token); |
|
33 |
$(base_widget).find('input[name$="$validation_url"]').val(''); |
|
34 |
enable(); |
|
23 | 35 |
$(base_widget).parents('form').find('input[name=submit]').prop('disabled', false); |
24 | 36 |
$(this).hide(); |
25 | 37 |
}, |
... | ... | |
29 | 41 |
} |
30 | 42 |
}); |
31 | 43 |
$(this).find('a.remove').click(function() { |
32 |
$(base_widget).find('input[type=hidden]').val('');
|
|
44 |
$(base_widget).find('input[name$="$token"]').val('');
|
|
33 | 45 |
$(base_widget).find('.fileinfo').hide(); |
34 | 46 |
$(base_widget).find('input[type=file]').show(); |
35 | 47 |
return false; |
... | ... | |
38 | 50 |
$(base_widget).find('input[type=file]').click(); |
39 | 51 |
return false; |
40 | 52 |
}); |
53 |
if ($(this).find('select[name$="$validation_url"] option:selected').val()) { |
|
54 |
disable(); |
|
55 |
} |
|
56 |
$(this).find('select[name$="$validation_url"]').on('change', function() { |
|
57 |
var url = $(this).find(':selected').val(); |
|
58 |
if (url) { |
|
59 |
var validations = $(this).data('validations'); |
|
60 | ||
61 |
for (var i = 0; i < validations.length; i++) { |
|
62 |
if (validations[i].url == url) { |
|
63 |
base_widget.find('a.remove').trigger('click'); |
|
64 |
for (var item in validations[i]) { |
|
65 |
if (! item) { |
|
66 |
continue; |
|
67 |
} |
|
68 |
var $input = base_widget.find('input[name$="$' + item + '"]'); |
|
69 |
if ($input.length) { |
|
70 |
$input.val(validations[i][item]); |
|
71 |
} |
|
72 |
} |
|
73 |
disable(); |
|
74 |
} |
|
75 |
} |
|
76 |
} else { |
|
77 |
enable(); |
|
78 |
} |
|
79 |
}); |
|
41 | 80 |
}); |
42 | 81 |
}); |
43 |
- |