Projet

Général

Profil

0002-rewrite-file-validation-fixes-10444.patch

Benjamin Dauvergne, 28 mars 2016 03:43

Télécharger (21,1 ko)

Voir les différences:

Subject: [PATCH 2/2] rewrite file validation (fixes #10444)

- metadata fields are now directly added to the form, as part of the
  FileWithPreviewWidget sub-widgets (it's a CompositeWidget)
- existing validated metadatas are proposed for prefilling
- if prefilled no file is uploadee, and a special upload class NoUpload is used
  instead of a PicklableUpload, it only contains the metadatas.
- prefilled fields are disabled,
- on upload of a new file all prefilled fields are emptied and re-enabled.
 wcs/fields.py                             |  17 ++++-
 wcs/file_validation.py                    | 114 ++++++++++++++++--------------
 wcs/forms/common.py                       |  55 ++++++++------
 wcs/qommon/form.py                        |  62 +++++++++++++++-
 wcs/qommon/static/css/qommon.css          |  10 +++
 wcs/qommon/static/js/qommon.fileupload.js |  43 ++++++++++-
 6 files changed, 220 insertions(+), 81 deletions(-)
wcs/fields.py
720 720
        self.document_type = self.document_type or {}
721 721

  
722 722
    @property
723
    def metadata(self):
724
        return self.document_type.get('metadata', [])
725

  
726
    @property
723 727
    def file_type(self):
724 728
        return (self.document_type or {}).get('mimetypes', [])
725 729

  
......
752 756
                'document_type', 'max_file_size', 'allow_portfolio_picking']
753 757

  
754 758
    def get_view_value(self, value):
755
        return htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % (
759
        r = TemplateIO(html=True)
760
        if not getattr(value, 'no_file', False):
761
            r += htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % (
756 762
                value.base_filename, self.id, value)
763
        for meta_field in self.metadata:
764
            metadata_value = getattr(value, 'metadata', {}).get(meta_field['name'], '')
765
            r += htmltext('<p>%s&nbsp;: %s</p>') % (meta_field['label'], metadata_value)
766
        return r.getvalue()
757 767

  
758 768
    def get_csv_value(self, value, hint=None):
759 769
        if not value:
......
788 798
            value = get_request().get_field(self.field_key)
789 799
            if value and hasattr(value, 'token'):
790 800
                get_request().form[self.field_key + '$token'] = value.token
801
        kwargs['document_type'] = self.document_type
791 802

  
792 803
    def get_document_types(self):
793 804
        document_types = {
......
867 878
        if self.document_type and self.document_type.get('fargo'):
868 879
            self.document_type['fargo'] = self.document_type['fargo'] == 'True'
869 880

  
881
    def store_structured_value(self, data, field_id):
882
        value = data.get(field_id)
883
        return getattr(value, 'metadata', {})
884

  
870 885

  
871 886
register_field_class(FileField)
872 887

  
wcs/file_validation.py
19 19
import hashlib
20 20
import urllib
21 21

  
22
from qommon.misc import http_get_page, json_loads
23
from quixote import get_publisher, get_response
24
from quixote.html import htmltext
22
from qommon.misc import http_get_page, json_loads, http_post_request
23
from quixote import get_publisher, get_request
25 24

  
26 25

  
27 26
def has_file_validation():
28 27
    return get_publisher().get_site_option('fargo_url') is not None
29 28

  
29

  
30 30
def fargo_get(path):
31 31
    fargo_url = get_publisher().get_site_option('fargo_url')
32 32
    url = urlparse.urljoin(fargo_url, path)
......
35 35
        return json_loads(data)
36 36
    return None
37 37

  
38

  
38 39
def sha256_of_upload(upload):
39 40
    return hashlib.sha256(upload.get_content()).hexdigest()
40 41

  
42

  
41 43
def get_document_types():
42 44
    if not has_file_validation():
43 45
        return {}
44 46
    response = fargo_get('/document-types/')
45
    publisher = get_publisher()
46 47
    if response.get('err') == 0:
47 48
        result = {}
48 49
        for schema in response['data']:
......
52 53
                'fargo': True,
53 54
            }
54 55
            if 'mimetypes' in schema:
55
                d['mimetypes'] = shema['mimetypes']
56
                d['mimetypes'] = schema['mimetypes']
56 57
            result[d['id']] = d
57

  
58
            d['metadata'] = schema.get('metadata', [])
58 59
        return result
59 60
    return {}
60 61

  
61
def validation_path(filled, field, upload):
62
    user = filled.get_user()
63
    if not user:
64
        return None
65
    if not user.name_identifiers:
66
        return None
67
    if not field.document_type or not field.document_type.get('fargo'):
68
        return None
69
    name_id = user.name_identifiers[0]
70
    sha_256 = sha256_of_upload(upload)
71
    document_type = field.document_type['id']
72
    path = '%s/%s/%s/' % (
73
        urllib.quote(name_id),
74
        urllib.quote(sha_256),
75
        urllib.quote(document_type),
76
    )
77
    return path
78

  
79
def validate_upload(filled, field, upload):
62

  
63
def get_validation(url):
64
    response, status, data, auth_header = http_get_page(url)
65
    if status == 200:
66
        return json_loads(data)['data']
67
    return None
68

  
69

  
70
def get_validations(document_type):
71
    request = get_request()
72
    document_type_id = document_type['id']
73
    qs = {}
74
    if not request.user:
75
        return []
76
    if request.user.name_identifiers:
77
        qs['user_nameid'] = request.user.name_identifiers[0]
78
    elif request.user.email:
79
        qs['user_email'] = request.user.email
80
    else:
81
        return []
82
    path = 'api/validation/%s/' % urllib.quote(document_type_id)
83
    validations = fargo_get(path)
84
    if validations and validations.get('data', {}).get('results', []):
85
        return validations['data']['results']
86
    return []
87

  
88

  
89
def is_valid(filled, field, upload):
80 90
    '''Check validation of the uploaded file with Fargo'''
81
    path = validation_path(filled, field, upload)
82
    if not path:
83
        return None
84
    response = fargo_get('metadata/' + path)
85
    if response is None:
86
        return None
87
    if response['err'] == 1:
88
        return False
89
    return response['data']
90

  
91
def get_document_type_label(document_type):
92
    return get_document_types().get(document_type, {}).get('label')
93

  
94
def validation_link(filled, field, upload):
91
    return 'url' in getattr(upload, 'metadata', {})
92

  
93

  
94
def validate(filled, field, upload):
95 95
    '''Compute link to Fargo to validate the given document'''
96
    path = validation_path(filled, field, upload)
97
    if not path:
98
        return ''
99
    get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css')
100
    get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js'])
101
    label = get_document_type_label(field.document_type['id'])
96
    document_type_id = field.document_type['id']
97
    path = 'api/validation/%s/' % urllib.quote(document_type_id)
102 98
    fargo_url = get_publisher().get_site_option('fargo_url')
103
    url = urlparse.urljoin(fargo_url, 'validation/' + path)
104
    next_url = get_publisher().get_frontoffice_url() + '/reload-top'
105
    url += '?next=%s' % urllib.quote(next_url)
106
    title = _('Validate as a %s') % label
107
    return htmltext('<a data-title="%(title)s" data-width="800" '
108
                    'data-height="500" href="%(url)s">%(title)s</a>') % {
109
        'title': title,
110
        'url': url,
111
    }
99
    url = urlparse.urljoin(fargo_url, path)
100
    payload = {}
101
    if filled.user:
102
        if filled.user.name_identifiers:
103
            payload['user_nameid'] = filled.user.name_identifiers[0]
104
        else:
105
            payload['user_email'] = filled.user.email
106
    payload['origin'] = get_request().get_server()
107
    payload['creator'] = get_request().user.display_name
108
    payload['content_hash'] = sha256_of_upload(upload)
109
    for meta_field in field.metadata:
110
        payload[meta_field['name']] = upload.metadata.get(meta_field['name'], '')
111
    headers = {'Content-type': 'application/json'}
112
    response, status, response_payload, auth_header = http_post_request(url, json.dumps(payload),
113
                                                                        headers=headers)
114
    if status == 201:
115
        upload.metadata = json_loads(response_payload)['data']
116
        filled.data['%s_structured' % field.id] = upload.metadata
117
        filled.store()
wcs/forms/common.py
97 97
    def html_top(self, title = None):
98 98
        template.html_top(title = title, default_org = _('Forms'))
99 99

  
100
    def validate(self):
101
        if not file_validation.has_file_validation():
102
            return redirect('.')
103
        field_id = get_request().form.get('field_id')
104
        if not field_id:
105
            return redirect('.')
106
        for field in self.formdef.fields:
107
            if field.id == field_id:
108
                break
109
        else:
110
            return redirect('.')
111
        if field.key != 'file':
112
            return redirect('.')
113
        if not field.document_type.get('fargo', False):
114
            return redirect('.')
115
        value = self.filled.data.get(field_id)
116
        file_validation.validate(self.filled, field, value)
117
        return redirect('.')
118

  
100 119
    def __init__(self, formdef, filled, register_workflow_subdirs=True):
101 120
        get_publisher().substitutions.feed(filled)
102 121
        self.formdef = formdef
......
612 631

  
613 632
    def display_file_field(self, form_url, field, value):
614 633
        r = TemplateIO(html=True)
615
        status = None
616
        if file_validation.has_file_validation():
617
            status = file_validation.validate_upload(self.filled, field, value)
618
            if status is False:
634
        validated = None
635
        is_fargo_dt = field.document_type.get('fargo', False)
636
        if file_validation.has_file_validation() and is_fargo_dt:
637
            validated = file_validation.is_valid(self.filled, field, value)
638
            if validated is False:
619 639
                extra_class = ' invalid'
620
            elif status is None:
621
                extra_class = ''
622 640
            else:
623 641
                extra_class = ' valid'
624 642
            r += htmltext('<div class="value%s">' % extra_class)
......
627 645
        s = field.get_view_value(value)
628 646
        s = s.replace(str('[download]'), str('%sdownload' % form_url))
629 647
        r += s
630
        if status is not None and get_request().is_in_backoffice():
631
            if status is False:
632
                r += htmltext(' <span class="validation-status">%s</span>') % _('not validated')
648
        if validated is not None and get_request().is_in_backoffice():
633 649
            r += htmltext('<div class="file-validation">')
634
            if status:
635
                r += htmltext('<p class="validation-status">%s</p>') % _(
636
                        '%(label)s validated by %(creator)s on %(created)s') % status
637
                r += htmltext('<ul>')
638
                for meta in status['metadata']:
639
                    r += htmltext(_('<li>%(label)s: %(value)s</li>')) % {
640
                        'label': meta['label'],
641
                        'value': meta['value']
642
                    }
643
                r += htmltext('</ul>')
650
            if validated:
651
                r += htmltext('<p class="validation-validated">%s</p>') % _(
652
                    'validated by %(creator)s on %(created)s') % value.metadata
644 653
                r += htmltext('<p>%s</p>') % (_('Valid from %(start)s to %(end)s') % {
645
                    'start': status['start'],
646
                    'end': status['end'],
654
                    'start': value.metadata['start'],
655
                    'end': value.metadata['end'],
647 656
                })
648 657
            else:
649
                r += file_validation.validation_link(self.filled, field, value)
658
                r += htmltext('<form method="post" action="./validate?field_id=%s">'
659
                              '<button>%s</button></form>') % (
660
                    field.id, _('Validate'))
650 661
            r += htmltext('</div>')
651 662
        r += htmltext('</div>')
652 663
        return str(r)
wcs/qommon/form.py
59 59
import misc
60 60
from strftime import strftime
61 61
from publisher import get_cfg
62
from wcs import file_validation
62 63

  
63 64
QuixoteForm = Form
64 65

  
......
581 582
            self.value = None
582 583

  
583 584

  
585
class NoUpload(object):
586
    no_file = True
587
    metadata = None
588

  
589
    def __init__(self, validation_url):
590
        self.metadata = file_validation.get_validation(validation_url)
591

  
592

  
584 593
class FileWithPreviewWidget(CompositeWidget):
585 594
    """Widget that proposes a File Upload widget but that stores the file
586 595
    ondisk so it has a "readonly" mode where the filename is shown."""
......
591 600

  
592 601
    max_file_size_bytes = None # will be filled automatically
593 602

  
603
    PARTIAL_FILL_ERROR = _('This field is normally not required, but you must leave it completely '
604
                           'empty or fill it, You cannot fill it partially')
605

  
594 606
    def __init__(self, name, value=None, **kwargs):
595 607
        CompositeWidget.__init__(self, name, value, **kwargs)
596 608
        self.value = value
609
        self.document_type = kwargs.pop('document_type', None) or {}
597 610
        self.preview = kwargs.get('readonly')
598 611
        self.max_file_size = kwargs.pop('max_file_size', None)
599 612
        self.allow_portfolio_picking = kwargs.pop('allow_portfolio_picking', True)
......
609 622
                # this could be used for client size validation of file size
610 623
                attrs['data-max-file-size'] = str(self.max_file_size_bytes)
611 624
            self.add(FileWidget, 'file', render_br=False, attrs=attrs)
625
        if self.document_type.get('metadata'):
626
            if self.preview:
627
                self.add(HiddenWidget, 'validation_url')
628
            else:
629
                validations = file_validation.get_validations(self.document_type)
630
                if validations:
631
                    options = [('', _('Known documents'), '')]
632
                    options += [(v['url'], v['display'], v['url']) for v in validations]
633
                    self.add(SingleSelectWidget, 'validation_url', options=options,
634
                             attrs={'data-validations': json.dumps(validations)})
635
            for meta_field in self.metadata:
636
                subvalue = getattr(value, 'metadata', {}).get(meta_field['name'])
637
                self.add(StringWidget, meta_field['name'],
638
                         title=meta_field['label'],
639
                         required=meta_field.get('required', True) and self.required,
640
                         readonly=self.preview,
641
                         value=subvalue)
642
                self.get_widget(meta_field['name']).extra_css_class = 'subwidget'
612 643
        if value:
613 644
            self.set_value(value)
614 645

  
......
651 682
        r += htmltext('</div>')
652 683
        return r.getvalue()
653 684

  
685
    @property
686
    def metadata(self):
687
        return self.document_type.get('metadata', [])
688

  
654 689
    def _parse(self, request):
655 690
        self.value = None
656 691
        if self.get('token'):
657 692
            token = self.get('token')
693
        elif self.get('validation_url'):
694
            self.value = NoUpload(self.get('validation_url'))
695
            return
658 696
        elif self.get('file'):
659 697
            token = get_session().add_tempfile(self.get('file'))
660 698
            request.form[self.get_widget('token').get_name()] = token
......
663 701

  
664 702
        session = get_session()
665 703
        if token and session.tempfiles and session.tempfiles.has_key(token):
666
            temp = session.tempfiles[token]
667 704
            self.value = session.get_tempfile_content(token)
668 705

  
669 706
        if self.value is None:
670
            # there's no file, the other checks are irrelevant.
707
            # there's no file, check all metadata field are empty too
708
            # if not file and required metadata field become required
709
            if any([self.get(meta_field['name']) for meta_field in self.metadata]):
710
                self.set_error(self.PARTIAL_FILL_ERROR)
711
                self.get_widget('file').set_error(self.REQUIRED_ERROR)
712
                for meta_field in self.metadata:
713
                    required = meta_field.get('required', True)
714
                    if required and not self.get(meta_field['name']):
715
                        self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR)
671 716
            return
672 717

  
718
        # There is some file, check all required metadata files have been filled
719
        for meta_field in self.metadata:
720
            required = meta_field.get('required', True)
721
            if required and not self.get(meta_field['name']):
722
                self.set_error(self.PARTIAL_FILL_ERROR)
723
                self.get_widget(meta_field['name']).set_error(self.REQUIRED_ERROR)
724

  
725

  
726
        if self.metadata:
727
            self.value.metadata = {}
728
            for meta_field in self.metadata:
729
                self.value.metadata[meta_field['name']] = self.get(meta_field['name'])
730

  
673 731
        # Don't trust the browser supplied MIME type, update the Upload object
674 732
        # with a MIME type created with magic (or based on the extension if the
675 733
        # module is missing).
wcs/qommon/static/css/qommon.css
79 79
	vertical-align: middle;
80 80
}
81 81

  
82
/* nested widgets */
83
.widget .subwidget {
84
	padding-left: 2em;
85
}
86
div.FileWithPreviewWidget .validation {
87
	display: inline-block;
88
	font-size: xx-small;
89
	margin-right: 2em;
90
}
91

  
82 92
div.form .title, form.quixote .title {
83 93
	font-weight: bold;
84 94
}
wcs/qommon/static/js/qommon.fileupload.js
6 6
        } else {
7 7
            $(base_widget).find('.fileinfo').hide();
8 8
        }
9
        function disable() {
10
            base_widget.find('.subwidget input').prop('disabled', true);
11
            $('form').on('submit', function () {
12
                $('input').prop('disabled', false);
13
            });
14
        }
15
        function enable() {
16
            base_widget.find('.subwidget input').val('');
17
            base_widget.find('.subwidget input').prop('disabled', false);
18
        }
9 19
        $(this).find('input[type=file]').fileupload({
10 20
            dataType: 'json',
11 21
            add: function (e, data) {
......
19 29
                $(base_widget).find('.fileprogress').hide();
20 30
                $(base_widget).find('.filename').text(data.result[0].name);
21 31
                $(base_widget).find('.fileinfo').show();
22
                $(base_widget).find('input[type=hidden]').val(data.result[0].token);
32
                $(base_widget).find('input[name$="$token"]').val(data.result[0].token);
33
                $(base_widget).find('input[name$="$validation_url"]').val('');
34
                enable();
23 35
                $(base_widget).parents('form').find('input[name=submit]').prop('disabled', false);
24 36
                $(this).hide();
25 37
            },
......
29 41
            }
30 42
        });
31 43
        $(this).find('a.remove').click(function() {
32
            $(base_widget).find('input[type=hidden]').val('');
44
            $(base_widget).find('input[name$="$token"]').val('');
33 45
            $(base_widget).find('.fileinfo').hide();
34 46
            $(base_widget).find('input[type=file]').show();
35 47
            return false;
......
38 50
            $(base_widget).find('input[type=file]').click();
39 51
            return false;
40 52
        });
53
        if ($(this).find('select[name$="$validation_url"] option:selected').val()) {
54
            disable();
55
        }
56
        $(this).find('select[name$="$validation_url"]').on('change', function() {
57
            var url = $(this).find(':selected').val();
58
            if (url) {
59
                var validations = $(this).data('validations');
60

  
61
                for (var i = 0; i < validations.length; i++) {
62
                    if (validations[i].url == url) {
63
                        base_widget.find('a.remove').trigger('click');
64
                        for (var item in validations[i]) {
65
                            if (! item) {
66
                                continue;
67
                            }
68
                            var $input = base_widget.find('input[name$="$' + item + '"]');
69
                            if ($input.length) {
70
                                $input.val(validations[i][item]);
71
                            }
72
                        }
73
                        disable();
74
                    }
75
                }
76
            } else {
77
                enable();
78
            }
79
        });
41 80
    });
42 81
});
43
-