Projet

Général

Profil

0003-add-support-for-file-validation-8402.patch

Benjamin Dauvergne, 05 octobre 2015 17:12

Télécharger (19,6 ko)

Voir les différences:

Subject: [PATCH 3/3] add support for file validation (#8402)

 tests/test_formdef.py         |  26 +++++++++-
 tests/test_formdef_import.py  |  13 +++++
 wcs/fields.py                 | 115 ++++++++++++++++++++++++++++++++++--------
 wcs/file_validation.py        | 111 ++++++++++++++++++++++++++++++++++++++++
 wcs/forms/common.py           |  32 +++++++++++-
 wcs/qommon/static/js/fargo.js |  56 ++++++++++++++++++++
 wcs/root.py                   |   7 ++-
 7 files changed, 336 insertions(+), 24 deletions(-)
 create mode 100644 wcs/file_validation.py
 create mode 100644 wcs/qommon/static/js/fargo.js
tests/test_formdef.py
4 4

  
5 5
import pytest
6 6

  
7
from mock import patch
8

  
7 9
from quixote import cleanup
8 10
from wcs import formdef
9 11
from wcs.formdef import FormDef
10 12
from wcs.workflows import Workflow
11
from wcs.fields import StringField
13
from wcs.fields import StringField, FileField
12 14

  
13 15
from utilities import create_temporary_pub
14 16

  
......
132 134

  
133 135
    with pytest.raises(AttributeError):
134 136
        assert substs.foobar
137

  
138
def test_file_field_migration():
139
    with patch('wcs.file_validation.get_document_types') as get_document_types:
140
        get_document_types.return_value = {
141
                'justificatif-de-domicile': {
142
                    'id': 'justificatif-de-domicile',
143
                    'label': 'Justificatif de domicile',
144
                    'fargo': True,
145
                },
146
        }
147
        formdef = FormDef()
148
        formdef.name = 'foo'
149
        file_type = ['image/*', 'application/pdf,application/vnd.oasis.opendocument.text,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.oasis.opendocument.spreadsheet,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']
150
        formdef.fields = [FileField(type='file', id='1', label='file')]
151
        formdef.fields[0].__dict__['file_type'] = file_type
152
        formdef.store()
153
        formdef = FormDef.get(1)
154
        assert 'file_type' not in formdef.fields[0].__dict__
155
        assert formdef.fields[0].document_type
156
        assert formdef.fields[0].document_type['id'] == '_legacy'
157
        assert formdef.fields[0].document_type['mimetypes'] == ['image/*', 'application/pdf,application/vnd.oasis.opendocument.text,application/msword,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.oasis.opendocument.spreadsheet,application/vnd.ms-excel,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet']
158
        assert formdef.fields[0].document_type['label'] == ','.join(file_type)
tests/test_formdef_import.py
227 227
    cat.store()
228 228
    assert FormDef.import_from_xml_tree(formdef_xml_with_id, include_id=False).category_id == '2'
229 229
    assert FormDef.import_from_xml_tree(formdef_xml_with_id, include_id=True).category_id is None
230

  
231
def test_file_field():
232
    formdef = FormDef()
233
    formdef.name = 'foo'
234
    formdef.fields = [fields.FileField(type='file', id='1', document_type={
235
        'id': 'justificatif-de-domicile',
236
        'fargo': True,
237
        'mimetypes': ['application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'image/*'],
238
    })]
239
    assert_xml_import_export_works(formdef, include_id=True)
240
    assert_xml_import_export_works(formdef)
241
    assert_json_import_export_works(formdef, include_id=True)
242
    assert_json_import_export_works(formdef)
wcs/fields.py
30 30
from qommon.strftime import strftime
31 31

  
32 32
import data_sources
33
import file_validation
33 34

  
34 35

  
35 36
class PrefillSelectionWidget(CompositeWidget):
......
676 677
class FileField(WidgetField):
677 678
    key = 'file'
678 679
    description = N_('File Upload')
679
    file_type = []
680
    document_type = None
680 681
    max_file_size = None
681 682

  
682 683
    widget_class = FileWithPreviewWidget
683 684
    extra_attributes = ['file_type', 'max_file_size']
684 685

  
686
    def __init__(self, *args, **kwargs):
687
        super(FileField, self).__init__(*args, **kwargs)
688
        self.document_type = self.document_type or {}
689

  
690
    @property
691
    def file_type(self):
692
        return (self.document_type or {}).get('mimetypes', [])
693

  
685 694
    def fill_admin_form(self, form):
686 695
        WidgetField.fill_admin_form(self, form)
687
        file_types = [
688
                ('audio/*', _('Sound files')),
689
                ('video/*', _('Video files')),
690
                ('image/*', _('Image files'))]
691
        filetypes_cfg = get_cfg('filetypes', {})
692
        if filetypes_cfg:
693
            for file_type in filetypes_cfg.values():
694
                file_types.append((
695
                    ','.join(file_type['mimetypes']), file_type['label']))
696
        if self.file_type:
697
            known_file_types = [x[0] for x in file_types]
698
            for file_type in self.file_type:
699
                if not file_type in known_file_types:
700
                    file_types.append((file_type, file_type))
701
        form.add(CheckboxesWidget, 'file_type', title=_('File type suggestion'),
702
                value=self.file_type, elements=file_types, inline=True,
703
                advanced=not(self.file_type))
696
        document_types = self.get_document_types()
697
        cur_dt = self.document_type
698
        # SingleSelectWidget compare the value and not the keys, so if we want
699
        # the current value not to be hidden, we must reset it with the corresponding
700
        # value from settings based on the 'id'
701
        document_type_id = self.document_type.get('id')
702
        if document_type_id in document_types \
703
               and self.document_type != document_types[document_type_id]:
704
            self.document_type = document_types[document_type_id]
705
        options = [(None, '---', {})]
706
        options += [(doc_type, doc_type['label'], key) for key, doc_type in document_types.iteritems()]
707
        form.add(SingleSelectWidget, 'document_type', title=_('File type suggestion'),
708
                value=self.document_type, options=options,
709
                advanced=not(self.document_type))
704 710
        form.add(FileSizeWidget, 'max_file_size', title=('Max file size'),
705 711
                value=self.max_file_size,
706 712
                advanced=not(self.max_file_size))
707 713

  
708 714
    def get_admin_attributes(self):
709
        return WidgetField.get_admin_attributes(self) + ['file_type',
710
                'max_file_size']
715
        return WidgetField.get_admin_attributes(self) + ['document_type',
716
                                                         'max_file_size']
711 717

  
712 718
    def get_view_value(self, value):
713 719
        return htmltext('<a download="%s" href="[download]?f=%s">%s</a>') % (
......
734 740
            if value and hasattr(value, 'token'):
735 741
                get_request().form[self.field_key + '$token'] = value.token
736 742

  
743
    def get_document_types(self):
744
        document_types = {
745
            '_audio': {
746
                'label': _('Sound files'),
747
                'mimetypes': ['audio/*'],
748
            },
749
            '_video': {
750
                'label': _('Video files'),
751
                'mimetypes': ['video/*'],
752
            },
753
            '_image': {
754
                'label': _('Image files'),
755
                'mimetypes': ['image/*'],
756
            }
757
        }
758
        # Local document types
759
        document_types.update(get_cfg('filetypes', {}))
760
        # Remote documents types
761
        document_types.update(file_validation.get_document_types())
762
        for key, document_type in document_types.iteritems():
763
            document_type['id'] = key
764
        # add current file type if it does not exist anymore in the settings
765
        cur_dt = self.document_type
766
        if cur_dt and cur_dt['id'] not in document_types:
767
            document_types[cur_dt['id']] = cur_dt
768
        return document_types
769

  
770
    def migrate(self):
771
        if 'file_type' in self.__dict__:
772
            self.document_type = {}
773
            if self.__dict__['file_type']:
774
                file_type = self.__dict__['file_type']
775
                document_types = self.get_document_types()
776
                for key, value in document_types.iteritems():
777
                    if self.file_type == value.get('mimetypes'):
778
                        self.document_type = value.copy()
779
                        self.document_type['id'] = key
780
                else:
781
                    # self.file_type is a combination of file type, we create a
782
                    # virtual one from them
783
                    self.document_type = {
784
                        'id': '_legacy',
785
                        'label': ','.join(file_type),
786
                        'mimetypes': file_type,
787
                    }
788
            del self.__dict__['file_type']
789
            return True
790
        return False
791

  
792
    def export_to_xml(self, charset, include_id=False):
793
        # convert some sub-fields to strings as export_to_xml() only supports
794
        # dictionnaries with strings values
795
        if self.document_type and self.document_type.get('mimetypes'):
796
            old_value = self.document_type['mimetypes']
797
            self.document_type['mimetypes'] = '|'.join(self.document_type['mimetypes'])
798
        result = super(FileField, self).export_to_xml(charset, include_id=include_id)
799
        if self.document_type and self.document_type.get('mimetypes'):
800
            self.document_type['mimetypes'] = old_value
801
        return result
802

  
803
    def init_with_xml(self, element, charset, include_id=False):
804
        super(FileField, self).init_with_xml(element, charset, include_id=include_id)
805
        # translate fields flattened to strings
806
        if self.document_type and self.document_type.get('mimetypes'):
807
            self.document_type['mimetypes'] = self.document_type['mimetypes'].split('|')
808
        if self.document_type and self.document_type.get('fargo'):
809
            self.document_type['fargo'] = self.document_type['fargo'] == 'True'
810

  
811

  
737 812
register_field_class(FileField)
738 813

  
739 814

  
wcs/file_validation.py
1
# w.c.s. - web application for online forms
2
# Copyright (C) 2005-2010  Entr'ouvert
3
#
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
8
#
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU General Public License for more details.
13
#
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16

  
17
import json
18
import urlparse
19
import hashlib
20
import urllib
21

  
22
from qommon.misc import http_get_page, json_loads
23
from quixote import get_publisher, get_response
24
from quixote.html import htmltext
25

  
26

  
27
def has_file_validation():
28
    return get_publisher().get_site_option('fargo_url') is not None
29

  
30
def fargo_get(path):
31
    fargo_url = get_publisher().get_site_option('fargo_url')
32
    url = urlparse.urljoin(fargo_url, path)
33
    response, status, data, auth_header = http_get_page(url)
34
    if status == 200:
35
        return json_loads(data)
36
    return None
37

  
38
def sha256_of_upload(upload):
39
    return hashlib.sha256(upload.get_content()).hexdigest()
40

  
41
def get_document_types():
42
    if not has_file_validation():
43
        return {}
44
    response = fargo_get('/document-types/')
45
    publisher = get_publisher()
46
    if response.get('err') == 0:
47
        result = {}
48
        for schema in response['data']:
49
            d = {
50
                'id': schema['name'],
51
                'label': schema['label'],
52
                'fargo': True,
53
            }
54
            if 'mimetypes' in schema:
55
                d['mimetypes'] = shema['mimetypes']
56
            result[d['id']] = d
57

  
58
        return result
59
    return {}
60

  
61
def validation_path(filled, field, upload):
62
    user = filled.get_user()
63
    if not user:
64
        return None
65
    if not user.name_identifiers:
66
        return None
67
    if not field.document_type or not field.document_type.get('fargo'):
68
        return None
69
    name_id = user.name_identifiers[0]
70
    sha_256 = sha256_of_upload(upload)
71
    document_type = field.document_type['id']
72
    path = '%s/%s/%s/' % (
73
        urllib.quote(name_id),
74
        urllib.quote(sha_256),
75
        urllib.quote(document_type),
76
    )
77
    return path
78

  
79
def validate_upload(filled, field, upload):
80
    '''Check validation of the uploaded file with Fargo'''
81
    path = validation_path(filled, field, upload)
82
    if not path:
83
        return None
84
    response = fargo_get('metadata/' + path)
85
    if response is None:
86
        return None
87
    if response['err'] == 1:
88
        return False
89
    return response['data']
90

  
91
def get_document_type_label(document_type):
92
    return get_document_types().get(document_type, {}).get('label')
93

  
94
def validation_link(filled, field, upload):
95
    '''Compute link to Fargo to validate the given document'''
96
    path = validation_path(filled, field, upload)
97
    if not path:
98
        return ''
99
    get_response().add_css_include('../js/smoothness/jquery-ui-1.10.0.custom.min.css')
100
    get_response().add_javascript(['jquery-ui.js', 'jquery.js', 'fargo.js'])
101
    label = get_document_type_label(field.document_type['id'])
102
    fargo_url = get_publisher().get_site_option('fargo_url')
103
    url = urlparse.urljoin(fargo_url, 'validation/' + path)
104
    next_url = get_publisher().get_frontoffice_url() + '/reload-top'
105
    url += '?next=%s' % urllib.quote(next_url)
106
    title = _('Validate as a %s') % label
107
    return htmltext('<a data-title="%(title)s" data-width="800" '
108
                    'data-height="500" href="%(url)s">%(title)s</a>') % {
109
        'title': title,
110
        'url': url,
111
    }
wcs/forms/common.py
15 15
# along with this program; if not, see <http://www.gnu.org/licenses/>.
16 16

  
17 17
import sys
18
import hashlib
19
import urlparse
18 20

  
19 21
from quixote import get_publisher, get_request, get_response, get_session, redirect
20 22
from quixote.directory import Directory
21 23
from quixote.html import TemplateIO, htmltext
22 24

  
23
from wcs.fields import WidgetField
25
from wcs.fields import WidgetField, FileField
26
from wcs import file_validation
24 27

  
25 28
from qommon import template
26 29
from qommon import get_logger
......
399 402
            s = f.get_view_value(value)
400 403
            s = s.replace(str('[download]'), str('%sdownload' % form_url))
401 404
            r += s
405
            if isinstance(f, FileField) and get_request().is_in_backoffice():
406
                r += htmltext(self.file_validation_status(f, value))
402 407
            r += htmltext('</div></div>')
403 408

  
404

  
405 409
        if on_page:
406 410
            r += htmltext('</div>')
407 411

  
......
529 533
        else:
530 534
            return redirect('files/%s/' % fn)
531 535

  
536
    def file_validation_status(self, field, value):
537
        status = file_validation.validate_upload(self.filled, field, value)
538
        if status is None:
539
            return ''
540
        r = TemplateIO(html=True)
541
        r += htmltext('<div class="file-validation">')
542
        r += file_validation.validation_link(self.filled, field, value)
543
        if status:
544
            r += htmltext(_('<p>%s validated by %s on %s</p>')) % (
545
                status['label'], status['creator'], status['created'])
546
            r += htmltext('<ul>')
547
            for meta in status['metadata']:
548
                r += htmltext(_('<li>%(label)s: %(value)s</li>')) % {
549
                    'label': meta['label'],
550
                    'value': meta['value']
551
                }
552
            r += htmltext('</ul>')
553
            r += htmltext('<p>%s</p>') % (_('Valid from %(start)s to %(end)s') % {
554
                'start': status['start'],
555
                'end': status['end'],
556
            })
557
        r += htmltext('</div>')
558
        return str(r)
559

  
532 560
    def _q_lookup(self, component):
533 561
        if component == 'files':
534 562
            self.check_receiver()
wcs/qommon/static/js/fargo.js
1

  
2
$(function() {
3
  var iframe = $('<iframe frameborder="0" marginwidth="0" marginheight="0" allowfullscreen></iframe>');
4
  var dialog = $("<div></div>").append(iframe).appendTo("body").dialog({
5
        autoOpen: false,
6
        modal: true,
7
        resizable: false,
8
        width: "auto",
9
        height: "auto",
10
        close: function () {
11
            iframe.attr("src", "");
12
        }
13
  });
14
  $('.file-validation a').click(function (e) {
15
    e.preventDefault();
16
    var src = $(e.target).attr('href');
17
    var title = $(e.target).data("title");
18
    var width = $(e.target).data("width");
19
    var height = $(e.target).data("height");
20
    iframe.attr({
21
        width: parseInt(width),
22
        height: parseInt(height),
23
        src: src
24
    });
25
    dialog.dialog("option", "title", title);
26
    dialog.dialog("open");
27
  });
28
  $('p.use-file-from-fargo span').click(function(e) {
29
    e.preventDefault();
30
    var base_widget = $(this).parents('.file-upload-widget');
31
    document.fargo_set_token = function (token, title) {
32
       if (token) {
33
         $(base_widget).find('.filename').text(title);
34
         $(base_widget).find('.fileinfo').show();
35
         $(base_widget).find('input[type=hidden]').val(token);
36
         $(base_widget).find('input[type=file]').hide();
37
       }
38
       document.fargo_close_dialog();
39
    }
40
    document.fargo_close_dialog = function () {
41
       document.fargo_set_token = undefined;
42
       dialog.dialog('close');
43
    }
44
    var src = $(this).data('src');
45
    var title = $(this).data("title");
46
    var width = $(this).data("width");
47
    var height = $(this).data("height");
48
    iframe.attr({
49
        width: parseInt(width),
50
        height: parseInt(height),
51
        src: src
52
    });
53
    dialog.dialog("option", "title", title);
54
    dialog.dialog("open");
55
  });
56
});
wcs/root.py
192 192
    _q_exports = ['admin', 'backoffice', 'forms', 'login', 'logout', 'saml',
193 193
            'ident', 'register', 'afterjobs', 'themes', 'myspace', 'user', 'roles',
194 194
            'pages', ('tmp-upload', 'tmp_upload'), 'api', '__version__',
195
            'tryauth', 'auth', 'preview']
195
            'tryauth', 'auth', 'preview', ('reload-top', 'reload_top')]
196 196

  
197 197
    api = ApiDirectory()
198 198
    themes = template.ThemesDirectory()
......
308 308
        # or a form ?
309 309
        return forms.root.RootDirectory()._q_lookup(component)
310 310

  
311
    def reload_top(self):
312
        r = TemplateIO(html=True)
313
        r += htmltext('<script>window.top.document.location.reload();</script>')
314
        return r.getvalue()
315

  
311 316
    admin = None
312 317
    backoffice = None
313 318

  
314
-