Projet

Général

Profil

0001-backoffice-use-actual-data-for-item-s-field-filters-.patch

Frédéric Péters, 17 décembre 2019 13:37

Télécharger (24,4 ko)

Voir les différences:

Subject: [PATCH] backoffice: use actual data for item(s) field filters
 (#35703)

 tests/test_backoffice_pages.py      | 154 ++++++++++++++++++++++++++--
 wcs/backoffice/management.py        | 116 ++++++++++++++++++---
 wcs/fields.py                       |  19 ++--
 wcs/formdata.py                     |  10 +-
 wcs/forms/backoffice.py             |   1 -
 wcs/qommon/static/js/wcs.listing.js |  57 ++++++++++
 wcs/sql.py                          |  16 ++-
 7 files changed, 341 insertions(+), 32 deletions(-)
tests/test_backoffice_pages.py
694 694
    assert resp.text.count('<td>Yes</td>') == 0
695 695
    assert resp.text.count('<td>No</td>') > 0
696 696

  
697
def test_backoffice_item_filter(pub):
698
    create_superuser(pub)
699
    create_environment(pub)
700
    formdef = FormDef.get_by_urlname('form-title')
701
    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
702
        items=['a', 'b', 'c', 'd'],
703
        display_locations=['validation', 'summary', 'listings']))
704
    formdef.store()
705

  
706
    for i, formdata in enumerate(formdef.data_class().select()):
707
        if i%4 == 0:
708
            formdata.data['4'] = 'a'
709
            formdata.data['4_display'] = 'a'
710
        elif i%4 == 1:
711
            formdata.data['4'] = 'b'
712
            formdata.data['4_display'] = 'b'
713
        elif i%4 == 2:
714
            formdata.data['4'] = 'd'
715
            formdata.data['4_display'] = 'd'
716
        formdata.store()
717

  
718
    app = login(get_app(pub))
719
    resp = app.get('/backoffice/management/form-title/')
720
    resp.form['filter-4'].checked = True
721
    resp = resp.form.submit()
722

  
723
    assert resp.form['filter-4-value'].value == ''
724

  
725
    resp.form['filter-4-value'].value = 'a'
726
    resp = resp.form.submit()
727
    assert resp.text.count('<td>a</td>') > 0
728
    assert resp.text.count('<td>b</td>') == 0
729
    assert resp.text.count('<td>d</td>') == 0
730

  
731
    resp.form['filter-4-value'].value = 'b'
732
    resp = resp.form.submit()
733
    assert resp.text.count('<td>a</td>') == 0
734
    assert resp.text.count('<td>b</td>') > 0
735
    assert resp.text.count('<td>d</td>') == 0
736

  
737
    if not pub.is_using_postgresql():
738
        # in pickle all options are always displayed
739
        resp.form['filter-4-value'].value = 'c'
740
        resp = resp.form.submit()
741
        assert resp.text.count('<td>a</td>') == 0
742
        assert resp.text.count('<td>b</td>') == 0
743
        assert resp.text.count('<td>c</td>') == 0
744

  
745
    else:
746
        # in postgresql, option 'c' is never used so not even listed
747
        with pytest.raises(ValueError):
748
            resp.form['filter-4-value'].value = 'c'
749

  
750
        # check json view used to fill select filters from javascript
751
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&' + resp.request.query_string)
752
        assert [x['id'] for x in resp2.json['data']] == ['a', 'b', 'd']
753
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&_search=d&' + resp.request.query_string)
754
        assert [x['id'] for x in resp2.json['data']] == ['d']
755
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=7&' + resp.request.query_string, status=404)
756

  
757
        for status in ('all', 'waiting', 'pending', 'done'):
758
            resp.form['status'] = status
759
            resp = resp.form.submit()
760
            resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&' + resp.request.query_string)
761
            assert [x['id'] for x in resp2.json['data']] == ['a', 'b', 'd']
762

  
763
def test_backoffice_item_double_filter(pub):
764
    if not pub.is_using_postgresql():
765
        pytest.skip('this requires SQL')
766
        return
767
    create_superuser(pub)
768
    create_environment(pub)
769
    formdef = FormDef.get_by_urlname('form-title')
770
    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
771
        items=['a', 'b', 'c', 'd'],
772
        display_locations=['validation', 'summary', 'listings']))
773
    formdef.fields.append(fields.ItemField(id='5', label='5th field', type='item',
774
        items=['A', 'B', 'C', 'D'],
775
        display_locations=['validation', 'summary', 'listings']))
776
    formdef.store()
777

  
778
    for i, formdata in enumerate(formdef.data_class().select()):
779
        if i%4 == 0:
780
            formdata.data['4'] = 'a'
781
            formdata.data['4_display'] = 'a'
782
            formdata.data['5'] = 'A'
783
            formdata.data['5_display'] = 'A'
784
        elif i%4 == 1:
785
            formdata.data['4'] = 'a'
786
            formdata.data['4_display'] = 'a'
787
            formdata.data['5'] = 'B'
788
            formdata.data['5_display'] = 'B'
789
        elif i%4 == 2:
790
            formdata.data['4'] = 'a'
791
            formdata.data['4_display'] = 'a'
792
            formdata.data['5'] = 'C'
793
            formdata.data['5_display'] = 'C'
794
        elif i%4 == 3:
795
            formdata.data['4'] = 'b'
796
            formdata.data['4_display'] = 'b'
797
            formdata.data['5'] = 'B'
798
            formdata.data['5_display'] = 'B'
799
        formdata.store()
800

  
801
    app = login(get_app(pub))
802
    resp = app.get('/backoffice/management/form-title/')
803
    resp.form['filter-4'].checked = True
804
    resp.form['filter-5'].checked = True
805
    resp = resp.form.submit()
806

  
807
    assert resp.form['filter-4-value'].value == ''
808
    assert resp.form['filter-5-value'].value == ''
809
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
810
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
811

  
812
    resp.form['filter-4-value'].value = 'a'
813
    resp = resp.form.submit()
814
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
815
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
816

  
817
    resp.form['filter-4-value'].value = 'b'
818
    resp = resp.form.submit()
819
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
820
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
821

  
822
    resp.form['filter-5-value'].value = 'B'
823
    resp = resp.form.submit()
824
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
825
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
826

  
827
    resp.form['filter-4-value'].value = ''
828
    resp = resp.form.submit()
829
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
830
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
831

  
697 832
def test_backoffice_items_filter(pub):
698 833
    create_superuser(pub)
699 834
    create_environment(pub)
......
734 869
    assert resp.text.count('<td>a</td>') == 0
735 870
    assert resp.text.count('<td>b, d</td>') > 0
736 871

  
737
    resp.form['filter-4-value'].value = 'c'
738
    resp = resp.form.submit()
739
    assert resp.text.count('<td>a, b</td>') == 0
740
    assert resp.text.count('<td>a</td>') == 0
741
    assert resp.text.count('<td>b, d</td>') == 0
742
    assert resp.text.count('data-link') == 0 # no rows
872
    if pub.is_using_postgresql():
873
        # option 'c' is never used so not even listed
874
        with pytest.raises(ValueError):
875
            resp.form['filter-4-value'].value = 'c'
876
    else:
877
        resp.form['filter-4-value'].value = 'c'
878
        resp = resp.form.submit()
879
        assert resp.text.count('<td>a, b</td>') == 0
880
        assert resp.text.count('<td>a</td>') == 0
881
        assert resp.text.count('<td>b, d</td>') == 0
882
        assert resp.text.count('data-link') == 0 # no rows
743 883

  
744 884
def test_backoffice_csv(pub):
745 885
    create_superuser(pub)
......
789 929

  
790 930
    resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
791 931
    resp = resp.forms[0].submit()
932
    resp.forms[0]['filter-2-value'] = 'baz'
933
    resp = resp.forms[0].submit()
792 934
    resp_csv = resp.click('Export as CSV File')
793 935
    assert len(resp_csv.text.splitlines()) == 9
794 936
    assert 'Created' in resp_csv.text.splitlines()[0]
wcs/backoffice/management.py
50 50
from ..qommon import ods
51 51
from ..qommon.form import *
52 52
from ..qommon.storage import (Equal, NotEqual, LessOrEqual, GreaterOrEqual, Or,
53
        Intersects, ILike, FtsMatch, Contains, Null)
53
        Intersects, ILike, FtsMatch, Contains, Null, NotNull)
54 54

  
55 55
from wcs.api_utils import get_user_from_api_query_string
56 56
from wcs.conditions import Condition
......
998 998

  
999 999
class FormPage(Directory):
1000 1000
    _q_exports = ['', 'csv', 'stats', 'xls', 'ods', 'json', 'export', 'map',
1001
                  'geojson']
1001
                  'geojson', ('filter-options', 'filter_options')]
1002 1002

  
1003 1003
    def __init__(self, component):
1004 1004
        try:
......
1049 1049
            return ('start', 'end')
1050 1050
        return ()
1051 1051

  
1052
    def get_filter_sidebar(self, selected_filter=None, mode='listing'):
1052
    def get_item_filter_options(self, filter_field, selected_filter, criterias):
1053
        criterias = (criterias or [])[:]
1054
        # remove potential filter on self (Equal for item, Intersects for items)
1055
        criterias = [x for x in criterias if not (isinstance(x, (Equal, Intersects)) and
1056
                                                  x.attribute == 'f%s' % filter_field.id)]
1057
        # apply other filters
1058
        criterias.append(Null('anonymised'))
1059
        if selected_filter == 'all':
1060
            criterias.append(NotEqual('status', 'draft'))
1061
        elif selected_filter in ('waiting', 'pending'):
1062
            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_not_endpoint_status()]
1063
            criterias.append(Contains('status', statuses))
1064
            if selected_filter == 'waiting':
1065
                user = get_request().user
1066
                user_roles = [logged_users_role().id] + user.get_roles()
1067
                criterias.append(Intersects('actions_roles_array', user_roles))
1068
        elif selected_filter == 'done':
1069
            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_endpoint_status()]
1070
            criterias.append(Contains('status', statuses))
1071
        criterias.append(NotNull('f%s' % filter_field.id))
1072
        options = self.formdef.data_class().select_distinct(
1073
                ['f%s' % filter_field.id, 'f%s_display' % filter_field.id],
1074
                clause=criterias)
1075

  
1076
        if filter_field.type == 'items':
1077
            # unnest key/values
1078
            exploded_options = {}
1079
            for option_keys, option_label in options:
1080
                for option_key, option_label in zip(option_keys, option_label.split(', ')):
1081
                    exploded_options[option_key] = option_label
1082
            options = list(sorted(exploded_options.items(), key=lambda x: x[1]))
1083

  
1084
        return options
1085

  
1086
    def filter_options(self):
1087
        get_request().is_json_marker = True
1088
        field_id = get_request().form.get('filter_field_id')
1089
        for filter_field in self.get_formdef_fields():
1090
            if filter_field.id == field_id:
1091
                break
1092
        else:
1093
            raise errors.TraversalError()
1094

  
1095
        selected_filter = self.get_filter_from_query()
1096
        criterias = self.get_criterias_from_query()
1097
        options = self.get_item_filter_options(filter_field, selected_filter, criterias)
1098
        if get_request().form.get('_search'):  # select2
1099
            term = get_request().form.get('_search')
1100
            if term:
1101
                options = [x for x in options if term.lower() in x[1].lower()]
1102
            options = options[:15]
1103
        get_response().set_content_type('application/json')
1104
        return json.dumps({'err': 0, 'data': [{'id': x[0], 'text': x[1]} for x in options]})
1105

  
1106
    def get_filter_sidebar(self, selected_filter=None, mode='listing', query=None, criterias=None):
1053 1107
        r = TemplateIO(html=True)
1054 1108

  
1055 1109
        waitpoint_status = self.formdef.workflow.get_waitpoint_status()
......
1119 1173

  
1120 1174
            elif filter_field.type in ('item', 'items'):
1121 1175
                filter_field.required = False
1122
                options = filter_field.get_options()
1123
                if options:
1124
                    if len(options[0]) == 2:
1176

  
1177
                if get_publisher().is_using_postgresql():
1178
                    # Get options from existing formdatas.
1179
                    # This allows for options that don't appear anymore in the
1180
                    # data source to be listed (for example because the field
1181
                    # is using a parametrized URL depending on unavailable
1182
                    # variables, or simply returning different results now).
1183
                    display_mode = 'select'
1184
                    if filter_field.type == 'item' and filter_field.get_display_mode() == 'autocomplete':
1185
                        display_mode = 'select2'
1186

  
1187
                    if display_mode == 'select':
1188
                        options = self.get_item_filter_options(
1189
                                filter_field, selected_filter, criterias)
1125 1190
                        options = [(x[0], x[1], x[0]) for x in options]
1126
                    options.insert(0, (None, '', ''))
1191
                        options.insert(0, (None, '', ''))
1192
                        attrs = {'data-refresh-options': str(filter_field.id)}
1193
                    else:
1194
                        current_filter = get_request().form.get('filter-%s-value' % filter_field.id)
1195
                        options = [(current_filter, '', current_filter or '')]
1196
                        attrs = {'data-remote-options': str(filter_field.id)}
1197
                        get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
1198
                        get_response().add_css_include('../js/select2/select2.css')
1199

  
1127 1200
                    r += SingleSelectWidget(filter_field_key, title=filter_field.label,
1128 1201
                            options=options, value=filter_field_value,
1129
                            render_br=False).render()
1202
                            render_br=False, attrs=attrs).render()
1203

  
1130 1204
                else:
1131
                    # There may be no options because the field is using
1132
                    # a jsonp data source, or a json source using a
1133
                    # parametrized URL depending on unavailable variables.
1134
                    #
1135
                    # In that case fall back on a string widget.
1136
                    r += StringWidget(filter_field_key, title=filter_field.label,
1137
                            value=filter_field_value, render_br=False).render()
1205
                    # In pickle environments, get options from data source
1206
                    options = filter_field.get_options()
1207
                    if options:
1208
                        if len(options[0]) == 2:
1209
                            options = [(x[0], x[1], x[0]) for x in options]
1210
                        options.insert(0, (None, '', ''))
1211
                        r += SingleSelectWidget(filter_field_key, title=filter_field.label,
1212
                                options=options, value=filter_field_value,
1213
                                render_br=False).render()
1214
                    else:
1215
                        # and fall back on a string widget if there are none.
1216
                        r += StringWidget(filter_field_key, title=filter_field.label,
1217
                                value=filter_field_value, render_br=False).render()
1138 1218

  
1139 1219
            elif filter_field.type == 'bool':
1140 1220
                options = [(None, '', ''), (True, _('Yes'), 'true'), (False, _('No'), 'false')]
......
1160 1240
        return r.getvalue()
1161 1241

  
1162 1242
    def get_fields_sidebar(self, selected_filter, fields, offset=None,
1163
            limit=None, order_by=None, columns_settings_label=None):
1243
            limit=None, order_by=None, columns_settings_label=None,
1244
            query=None, criterias=None):
1164 1245
        get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'wcs.listing.js'])
1165 1246

  
1166 1247
        r = TemplateIO(html=True)
......
1186 1267
                r += htmltext('<input class="inline-input" name="q">')
1187 1268
            r += htmltext('<input type="submit" class="side-button" value="%s"/>') % _('Search')
1188 1269

  
1189
        r += self.get_filter_sidebar(selected_filter=selected_filter)
1270
        r += self.get_filter_sidebar(selected_filter=selected_filter, query=query, criterias=criterias)
1190 1271

  
1191 1272
        r += htmltext('<button class="refresh">%s</button>') % _('Refresh')
1192 1273

  
......
1421 1502

  
1422 1503
        get_response().filter['sidebar'] = self.get_formdata_sidebar(qs) + \
1423 1504
                self.get_fields_sidebar(selected_filter, fields, limit=limit,
1505
                        query=query, criterias=criterias,
1424 1506
                        offset=offset, order_by=order_by,
1425 1507
                        columns_settings_label=_('Columns Settings'))
1426 1508

  
wcs/fields.py
1307 1307
            return [(x, x) for x in self.items]
1308 1308
        return []
1309 1309

  
1310
    def perform_more_widget_changes(self, form, kwargs, edit=True):
1311
        data_source = data_sources.get_object(self.data_source)
1310
    def get_display_mode(self, data_source=None):
1311
        if not data_source:
1312
            data_source = data_sources.get_object(self.data_source)
1312 1313

  
1313 1314
        if data_source and data_source.type == 'jsonp':
1314 1315
            # a source defined as JSONP can only be used in autocomplete mode
1315
            self.display_mode = 'autocomplete'
1316
            return 'autocomplete'
1317

  
1318
        return self.display_mode
1319

  
1320
    def perform_more_widget_changes(self, form, kwargs, edit=True):
1321
        data_source = data_sources.get_object(self.data_source)
1322
        display_mode = self.get_display_mode(data_source)
1316 1323

  
1317
        if self.display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
1324
        if display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
1318 1325
            self.url = kwargs['url'] = data_source.get_jsonp_url()
1319 1326
            self.widget_class = JsonpSingleSelectWidget
1320 1327
            return
......
1330 1337
            kwargs['options'] = self.get_options()
1331 1338
        if not kwargs.get('options'):
1332 1339
            kwargs['options'] = [(None, '---')]
1333
        if self.display_mode == 'radio':
1340
        if display_mode == 'radio':
1334 1341
            self.widget_class = RadiobuttonsWidget
1335 1342
            if type(kwargs['options'][0]) is str:
1336 1343
                first_items = [x for x in kwargs['options'][:3]]
......
1340 1347
            if len(kwargs['options']) > 3 or length_first_items > 40:
1341 1348
                # TODO: absence/presence of delimitor should be an option
1342 1349
                kwargs['delim'] = htmltext('<br />')
1343
        elif self.display_mode == 'autocomplete':
1350
        elif display_mode == 'autocomplete':
1344 1351
            kwargs['select2'] = True
1345 1352

  
1346 1353
    def get_display_value(self, value):
wcs/formdata.py
1101 1101
        except KeyError:
1102 1102
            # give direct access to values from the data dictionary
1103 1103
            if attr[0] == 'f':
1104
                return self.__dict__['data'][attr[1:]]
1104
                field_id = attr[1:]
1105
                if field_id in self.__dict__['data']:
1106
                    return self.__dict__['data'][field_id]
1107
                # if field id is not in data dictionary it may still be a valid
1108
                # field, never initialized, check requested field id against
1109
                # existing fields ids.
1110
                formdef_fields = self.formdef.get_all_fields()
1111
                if field_id in [x.id for x in formdef_fields]:
1112
                    return None
1105 1113
            raise AttributeError(attr)
1106 1114

  
1107 1115
    # don't pickle _formdef cache
wcs/forms/backoffice.py
38 38
        partial_display = False
39 39
        using_postgresql = get_publisher().is_using_postgresql()
40 40

  
41

  
42 41
        if not items:
43 42
            if offset and not limit:
44 43
                limit = int(get_publisher().get_site_option('default-page-size') or 20)
wcs/qommon/static/js/wcs.listing.js
166 166
      if (window.history) {
167 167
        window.history.replaceState(null, null, pathname + '?' + listing_settings.qs);
168 168
      }
169
      /* refresh dynamic filters */
170
      $('[data-refresh-options]').each(function(idx, elem) {
171
        var $select = $(elem);
172
        var current_value = $select.val();
173
        var filter_path = pathname + 'filter-options?filter_field_id=' + $(elem).data('refresh-options') + '&' + listing_settings.qs;
174
        $.ajax({
175
          url: filter_path,
176
          success: function(data) {
177
            $select.empty();
178
            var $option = $('<option></option>', {value: ''});
179
            $option.appendTo($select);
180
            for (var i=0; i<data.data.length; i++) {
181
              var $option = $('<option></option>', {value: data.data[i].id, text: data.data[i].text});
182
              if (data.data[i].id == current_value) {
183
                $option.attr('selected', 'selected');
184
              }
185
              $option.appendTo($select);
186
            }
187
          }
188
        });
189
      });
190

  
169 191
      /* makes sure activity and disabled-during-submit are removed */
170 192
      $('#more-user-links, #listing, #statistics').removeClass('activity');
171 193
      $('form').removeClass('disabled-during-submit');
......
236 258
    return false;
237 259
  });
238 260

  
261
  /* set filter options from server (select2) */
262
  $('[data-remote-options]').each(function(idx, elem) {
263
    var filter_field_id = $(elem).data('remote-options');
264
    var options = {
265
      language: {
266
        errorLoading: function() { return WCS_I18N.s2_errorloading; },
267
        noResults: function () { return WCS_I18N.s2_nomatches; },
268
        inputTooShort: function (input, min) { return WCS_I18N.s2_tooshort; },
269
        loadingMore: function () { return WCS_I18N.s2_loadmore; },
270
        searching: function () { return WCS_I18N.s2_searching; },
271
      },
272
      placeholder: '',
273
      allowClear: true,
274
      minimumInputLength: 1,
275
      ajax: {
276
        url: function() {
277
          var pathname = window.location.pathname.replace(/^\/+/, '/');
278
          var filter_settings = $('form#listing-settings').serialize();
279
          return pathname + 'filter-options?filter_field_id=' + filter_field_id + '&' + filter_settings;
280
        },
281
        dataType: 'json',
282
        data: function(params) {
283
          var query = {
284
            _search: params.term,
285
          }
286
          return query;
287
        },
288
        processResults: function (data, params) {
289
          return {results: data.data};
290
        },
291
      },
292
    };
293
    $(elem).select2(options);
294
  });
295

  
239 296
  $('button.pdf').click(function() {
240 297
    if (window.location.href.indexOf('?') == -1) {
241 298
      window.location = window.location + '?pdf=on';
wcs/sql.py
1142 1142
            return objects
1143 1143
        return list(objects)
1144 1144

  
1145

  
1145
    @classmethod
1146
    @guard_postgres
1147
    def select_distinct(cls, columns, clause=None):
1148
        conn, cur = get_connection_and_cursor()
1149
        sql_statement = 'SELECT DISTINCT ON (%s) %s FROM %s' % (columns[0], ', '.join(columns), cls._table_name)
1150
        where_clauses, parameters, func_clause = parse_clause(clause)
1151
        assert not func_clause
1152
        if where_clauses:
1153
            sql_statement += ' WHERE ' + ' AND '.join(where_clauses)
1154
        sql_statement += ' ORDER BY %s' % columns[0]
1155
        cur.execute(sql_statement, parameters)
1156
        values = [x for x in cur.fetchall()]
1157
        conn.commit()
1158
        cur.close()
1159
        return values
1146 1160

  
1147 1161
    def get_sql_dict_from_data(self, data, formdef):
1148 1162
        sql_dict = {}
1149
-