Projet

Général

Profil

0001-backoffice-use-actual-data-for-item-s-field-filters-.patch

Frédéric Péters, 16 décembre 2019 16:29

Télécharger (24,1 ko)

Voir les différences:

Subject: [PATCH] backoffice: use actual data for item(s) field filters
 (#35703)

 tests/test_backoffice_pages.py      | 148 ++++++++++++++++++++++++++--
 wcs/backoffice/management.py        | 115 +++++++++++++++++----
 wcs/fields.py                       |  19 ++--
 wcs/formdata.py                     |  10 +-
 wcs/forms/backoffice.py             |   1 -
 wcs/qommon/static/js/wcs.listing.js |  57 +++++++++++
 wcs/sql.py                          |  16 +++
 7 files changed, 335 insertions(+), 31 deletions(-)
tests/test_backoffice_pages.py
694 694
    assert resp.text.count('<td>Yes</td>') == 0
695 695
    assert resp.text.count('<td>No</td>') > 0
696 696

  
697
def test_backoffice_item_filter(pub):
698
    create_superuser(pub)
699
    create_environment(pub)
700
    formdef = FormDef.get_by_urlname('form-title')
701
    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
702
        items=['a', 'b', 'c', 'd'],
703
        display_locations=['validation', 'summary', 'listings']))
704
    formdef.store()
705

  
706
    for i, formdata in enumerate(formdef.data_class().select()):
707
        if i%4 == 0:
708
            formdata.data['4'] = 'a'
709
            formdata.data['4_display'] = 'a'
710
        elif i%4 == 1:
711
            formdata.data['4'] = 'b'
712
            formdata.data['4_display'] = 'b'
713
        elif i%4 == 2:
714
            formdata.data['4'] = 'd'
715
            formdata.data['4_display'] = 'd'
716
        formdata.store()
717

  
718
    app = login(get_app(pub))
719
    resp = app.get('/backoffice/management/form-title/')
720
    resp.form['filter-4'].checked = True
721
    resp = resp.form.submit()
722

  
723
    assert resp.form['filter-4-value'].value == ''
724

  
725
    resp.form['filter-4-value'].value = 'a'
726
    resp = resp.form.submit()
727
    assert resp.text.count('<td>a</td>') > 0
728
    assert resp.text.count('<td>b</td>') == 0
729
    assert resp.text.count('<td>d</td>') == 0
730

  
731
    resp.form['filter-4-value'].value = 'b'
732
    resp = resp.form.submit()
733
    assert resp.text.count('<td>a</td>') == 0
734
    assert resp.text.count('<td>b</td>') > 0
735
    assert resp.text.count('<td>d</td>') == 0
736

  
737
    if pub.is_using_postgresql():
738
        # option 'c' is never used so not even listed
739
        with pytest.raises(ValueError):
740
            resp.form['filter-4-value'].value = 'c'
741

  
742
        # check json view used to fill select filters from javascript
743
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&' + resp.request.query_string)
744
        assert [x['id'] for x in resp2.json['data']] == ['a', 'b', 'd']
745
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&_search=d&' + resp.request.query_string)
746
        assert [x['id'] for x in resp2.json['data']] == ['d']
747
        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=7&' + resp.request.query_string, status=404)
748

  
749
    else:
750
        # in pickle all options are always displayed
751
        resp.form['filter-4-value'].value = 'c'
752
        resp = resp.form.submit()
753
        assert resp.text.count('<td>a</td>') == 0
754
        assert resp.text.count('<td>b</td>') == 0
755
        assert resp.text.count('<td>c</td>') == 0
756

  
757
def test_backoffice_item_double_filter(pub):
758
    if not pub.is_using_postgresql():
759
        pytest.skip('this requires SQL')
760
        return
761
    create_superuser(pub)
762
    create_environment(pub)
763
    formdef = FormDef.get_by_urlname('form-title')
764
    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
765
        items=['a', 'b', 'c', 'd'],
766
        display_locations=['validation', 'summary', 'listings']))
767
    formdef.fields.append(fields.ItemField(id='5', label='5th field', type='item',
768
        items=['A', 'B', 'C', 'D'],
769
        display_locations=['validation', 'summary', 'listings']))
770
    formdef.store()
771

  
772
    for i, formdata in enumerate(formdef.data_class().select()):
773
        if i%4 == 0:
774
            formdata.data['4'] = 'a'
775
            formdata.data['4_display'] = 'a'
776
            formdata.data['5'] = 'A'
777
            formdata.data['5_display'] = 'A'
778
        elif i%4 == 1:
779
            formdata.data['4'] = 'a'
780
            formdata.data['4_display'] = 'a'
781
            formdata.data['5'] = 'B'
782
            formdata.data['5_display'] = 'B'
783
        elif i%4 == 2:
784
            formdata.data['4'] = 'a'
785
            formdata.data['4_display'] = 'a'
786
            formdata.data['5'] = 'C'
787
            formdata.data['5_display'] = 'C'
788
        elif i%4 == 3:
789
            formdata.data['4'] = 'b'
790
            formdata.data['4_display'] = 'b'
791
            formdata.data['5'] = 'B'
792
            formdata.data['5_display'] = 'B'
793
        formdata.store()
794

  
795
    app = login(get_app(pub))
796
    resp = app.get('/backoffice/management/form-title/')
797
    resp.form['filter-4'].checked = True
798
    resp.form['filter-5'].checked = True
799
    resp = resp.form.submit()
800

  
801
    assert resp.form['filter-4-value'].value == ''
802
    assert resp.form['filter-5-value'].value == ''
803
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
804
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
805

  
806
    resp.form['filter-4-value'].value = 'a'
807
    resp = resp.form.submit()
808
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
809
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
810

  
811
    resp.form['filter-4-value'].value = 'b'
812
    resp = resp.form.submit()
813
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
814
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
815

  
816
    resp.form['filter-5-value'].value = 'B'
817
    resp = resp.form.submit()
818
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
819
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
820

  
821
    resp.form['filter-4-value'].value = ''
822
    resp = resp.form.submit()
823
    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
824
    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
825

  
697 826
def test_backoffice_items_filter(pub):
698 827
    create_superuser(pub)
699 828
    create_environment(pub)
......
734 863
    assert resp.text.count('<td>a</td>') == 0
735 864
    assert resp.text.count('<td>b, d</td>') > 0
736 865

  
737
    resp.form['filter-4-value'].value = 'c'
738
    resp = resp.form.submit()
739
    assert resp.text.count('<td>a, b</td>') == 0
740
    assert resp.text.count('<td>a</td>') == 0
741
    assert resp.text.count('<td>b, d</td>') == 0
742
    assert resp.text.count('data-link') == 0 # no rows
866
    if pub.is_using_postgresql():
867
        # option 'c' is never used so not even listed
868
        with pytest.raises(ValueError):
869
            resp.form['filter-4-value'].value = 'c'
870
    else:
871
        resp.form['filter-4-value'].value = 'c'
872
        resp = resp.form.submit()
873
        assert resp.text.count('<td>a, b</td>') == 0
874
        assert resp.text.count('<td>a</td>') == 0
875
        assert resp.text.count('<td>b, d</td>') == 0
876
        assert resp.text.count('data-link') == 0 # no rows
743 877

  
744 878
def test_backoffice_csv(pub):
745 879
    create_superuser(pub)
......
789 923

  
790 924
    resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
791 925
    resp = resp.forms[0].submit()
926
    resp.forms[0]['filter-2-value'] = 'baz'
927
    resp = resp.forms[0].submit()
792 928
    resp_csv = resp.click('Export as CSV File')
793 929
    assert len(resp_csv.text.splitlines()) == 9
794 930
    assert 'Created' in resp_csv.text.splitlines()[0]
wcs/backoffice/management.py
50 50
from ..qommon import ods
51 51
from ..qommon.form import *
52 52
from ..qommon.storage import (Equal, NotEqual, LessOrEqual, GreaterOrEqual, Or,
53
        Intersects, ILike, FtsMatch, Contains, Null)
53
        Intersects, ILike, FtsMatch, Contains, Null, NotNull)
54 54

  
55 55
from wcs.api_utils import get_user_from_api_query_string
56 56
from wcs.conditions import Condition
......
998 998

  
999 999
class FormPage(Directory):
1000 1000
    _q_exports = ['', 'csv', 'stats', 'xls', 'ods', 'json', 'export', 'map',
1001
                  'geojson']
1001
                  'geojson', ('filter-options', 'filter_options')]
1002 1002

  
1003 1003
    def __init__(self, component):
1004 1004
        try:
......
1049 1049
            return ('start', 'end')
1050 1050
        return ()
1051 1051

  
1052
    def get_filter_sidebar(self, selected_filter=None, mode='listing'):
1052
    def get_item_filter_options(self, filter_field, selected_filter, criterias):
1053
        criterias = (criterias or [])[:]
1054
        # remove potential filter on self (Equal for item, Intersects for items)
1055
        criterias = [x for x in criterias if not (isinstance(x, (Equal, Intersects)) and
1056
                                                  x.attribute == 'f%s' % filter_field.id)]
1057
        # apply other filters
1058
        criterias.append(Null('anonymised'))
1059
        if selected_filter == 'all':
1060
            criterias.append(NotEqual('status', 'draft'))
1061
        elif selected_filter in ('waiting', 'pending'):
1062
            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_not_endpoint_status()]
1063
            criterias.append(Contains('status', statuses))
1064
            if selected_filter == 'waniting':
1065
                user_roles = [logged_users_role().id] + user.get_roles()
1066
                criterias.append(Intersects('actions_roles_array', user_roles))
1067
        elif selected_filter == 'done':
1068
            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_endpoint_status()]
1069
            criterias.append(Contains('status', statuses))
1070
        criterias.append(NotNull('f%s' % filter_field.id))
1071
        options = self.formdef.data_class().select_distinct(
1072
                ['f%s' % filter_field.id, 'f%s_display' % filter_field.id],
1073
                clause=criterias)
1074

  
1075
        if filter_field.type == 'items':
1076
            # unnest key/values
1077
            exploded_options = {}
1078
            for option_keys, option_label in options:
1079
                for option_key, option_label in zip(option_keys, option_label.split(', ')):
1080
                    exploded_options[option_key] = option_label
1081
            options = list(sorted(exploded_options.items(), key=lambda x: x[1]))
1082

  
1083
        return options
1084

  
1085
    def filter_options(self):
1086
        get_request().is_json_marker = True
1087
        field_id = get_request().form.get('filter_field_id')
1088
        for filter_field in self.get_formdef_fields():
1089
            if filter_field.id == field_id:
1090
                break
1091
        else:
1092
            raise errors.TraversalError()
1093

  
1094
        selected_filter = self.get_filter_from_query()
1095
        criterias = self.get_criterias_from_query()
1096
        options = self.get_item_filter_options(filter_field, selected_filter, criterias)
1097
        if get_request().form.get('_search'):  # select2
1098
            term = get_request().form.get('_search')
1099
            if term:
1100
                options = [x for x in options if term.lower() in x[1].lower()]
1101
            options = options[:15]
1102
        get_response().set_content_type('application/json')
1103
        return json.dumps({'err': 0, 'data': [{'id': x[0], 'text': x[1]} for x in options]})
1104

  
1105
    def get_filter_sidebar(self, selected_filter=None, mode='listing', query=None, criterias=None):
1053 1106
        r = TemplateIO(html=True)
1054 1107

  
1055 1108
        waitpoint_status = self.formdef.workflow.get_waitpoint_status()
......
1119 1172

  
1120 1173
            elif filter_field.type in ('item', 'items'):
1121 1174
                filter_field.required = False
1122
                options = filter_field.get_options()
1123
                if options:
1124
                    if len(options[0]) == 2:
1175

  
1176
                if get_publisher().is_using_postgresql():
1177
                    # Get options from existing formdatas.
1178
                    # This allows for options that don't appear anymore in the
1179
                    # data source to be listed (for example because the field
1180
                    # is using a parametrized URL depending on unavailable
1181
                    # variables, or simply returning different results now).
1182
                    display_mode = 'select'
1183
                    if filter_field.type == 'item' and filter_field.get_display_mode() == 'autocomplete':
1184
                        display_mode = 'select2'
1185

  
1186
                    if display_mode == 'select':
1187
                        options = self.get_item_filter_options(
1188
                                filter_field, selected_filter, criterias)
1125 1189
                        options = [(x[0], x[1], x[0]) for x in options]
1126
                    options.insert(0, (None, '', ''))
1190
                        options.insert(0, (None, '', ''))
1191
                        attrs = {'data-refresh-options': str(filter_field.id)}
1192
                    else:
1193
                        current_filter = get_request().form.get('filter-%s-value' % filter_field.id)
1194
                        options = [(current_filter, '', current_filter or '')]
1195
                        attrs = {'data-remote-options': str(filter_field.id)}
1196
                        get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
1197
                        get_response().add_css_include('../js/select2/select2.css')
1198

  
1127 1199
                    r += SingleSelectWidget(filter_field_key, title=filter_field.label,
1128 1200
                            options=options, value=filter_field_value,
1129
                            render_br=False).render()
1201
                            render_br=False, attrs=attrs).render()
1202

  
1130 1203
                else:
1131
                    # There may be no options because the field is using
1132
                    # a jsonp data source, or a json source using a
1133
                    # parametrized URL depending on unavailable variables.
1134
                    #
1135
                    # In that case fall back on a string widget.
1136
                    r += StringWidget(filter_field_key, title=filter_field.label,
1137
                            value=filter_field_value, render_br=False).render()
1204
                    # In pickle environments, get options from data source
1205
                    options = filter_field.get_options()
1206
                    if options:
1207
                        if len(options[0]) == 2:
1208
                            options = [(x[0], x[1], x[0]) for x in options]
1209
                        options.insert(0, (None, '', ''))
1210
                        r += SingleSelectWidget(filter_field_key, title=filter_field.label,
1211
                                options=options, value=filter_field_value,
1212
                                render_br=False).render()
1213
                    else:
1214
                        # and fall back on a string widget if there are none.
1215
                        r += StringWidget(filter_field_key, title=filter_field.label,
1216
                                value=filter_field_value, render_br=False).render()
1138 1217

  
1139 1218
            elif filter_field.type == 'bool':
1140 1219
                options = [(None, '', ''), (True, _('Yes'), 'true'), (False, _('No'), 'false')]
......
1160 1239
        return r.getvalue()
1161 1240

  
1162 1241
    def get_fields_sidebar(self, selected_filter, fields, offset=None,
1163
            limit=None, order_by=None, columns_settings_label=None):
1242
            limit=None, order_by=None, columns_settings_label=None,
1243
            query=None, criterias=None):
1164 1244
        get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'wcs.listing.js'])
1165 1245

  
1166 1246
        r = TemplateIO(html=True)
......
1186 1266
                r += htmltext('<input class="inline-input" name="q">')
1187 1267
            r += htmltext('<input type="submit" class="side-button" value="%s"/>') % _('Search')
1188 1268

  
1189
        r += self.get_filter_sidebar(selected_filter=selected_filter)
1269
        r += self.get_filter_sidebar(selected_filter=selected_filter, query=query, criterias=criterias)
1190 1270

  
1191 1271
        r += htmltext('<button class="refresh">%s</button>') % _('Refresh')
1192 1272

  
......
1421 1501

  
1422 1502
        get_response().filter['sidebar'] = self.get_formdata_sidebar(qs) + \
1423 1503
                self.get_fields_sidebar(selected_filter, fields, limit=limit,
1504
                        query=query, criterias=criterias,
1424 1505
                        offset=offset, order_by=order_by,
1425 1506
                        columns_settings_label=_('Columns Settings'))
1426 1507

  
wcs/fields.py
1307 1307
            return [(x, x) for x in self.items]
1308 1308
        return []
1309 1309

  
1310
    def perform_more_widget_changes(self, form, kwargs, edit=True):
1311
        data_source = data_sources.get_object(self.data_source)
1310
    def get_display_mode(self, data_source=None):
1311
        if not data_source:
1312
            data_source = data_sources.get_object(self.data_source)
1312 1313

  
1313 1314
        if data_source and data_source.type == 'jsonp':
1314 1315
            # a source defined as JSONP can only be used in autocomplete mode
1315
            self.display_mode = 'autocomplete'
1316
            return 'autocomplete'
1317

  
1318
        return self.display_mode
1319

  
1320
    def perform_more_widget_changes(self, form, kwargs, edit=True):
1321
        data_source = data_sources.get_object(self.data_source)
1322
        display_mode = self.get_display_mode(data_source)
1316 1323

  
1317
        if self.display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
1324
        if display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
1318 1325
            self.url = kwargs['url'] = data_source.get_jsonp_url()
1319 1326
            self.widget_class = JsonpSingleSelectWidget
1320 1327
            return
......
1330 1337
            kwargs['options'] = self.get_options()
1331 1338
        if not kwargs.get('options'):
1332 1339
            kwargs['options'] = [(None, '---')]
1333
        if self.display_mode == 'radio':
1340
        if display_mode == 'radio':
1334 1341
            self.widget_class = RadiobuttonsWidget
1335 1342
            if type(kwargs['options'][0]) is str:
1336 1343
                first_items = [x for x in kwargs['options'][:3]]
......
1340 1347
            if len(kwargs['options']) > 3 or length_first_items > 40:
1341 1348
                # TODO: absence/presence of delimitor should be an option
1342 1349
                kwargs['delim'] = htmltext('<br />')
1343
        elif self.display_mode == 'autocomplete':
1350
        elif display_mode == 'autocomplete':
1344 1351
            kwargs['select2'] = True
1345 1352

  
1346 1353
    def get_display_value(self, value):
wcs/formdata.py
1101 1101
        except KeyError:
1102 1102
            # give direct access to values from the data dictionary
1103 1103
            if attr[0] == 'f':
1104
                return self.__dict__['data'][attr[1:]]
1104
                field_id = attr[1:]
1105
                if field_id in self.__dict__['data']:
1106
                    return self.__dict__['data'][field_id]
1107
                # if field id is not in data dictionary it may still be a valid
1108
                # field, never initialized, check requested field id against
1109
                # existing fields ids.
1110
                formdef_fields = self.formdef.get_all_fields()
1111
                if field_id in [x.id for x in formdef_fields]:
1112
                    return None
1105 1113
            raise AttributeError(attr)
1106 1114

  
1107 1115
    # don't pickle _formdef cache
wcs/forms/backoffice.py
38 38
        partial_display = False
39 39
        using_postgresql = get_publisher().is_using_postgresql()
40 40

  
41

  
42 41
        if not items:
43 42
            if offset and not limit:
44 43
                limit = int(get_publisher().get_site_option('default-page-size') or 20)
wcs/qommon/static/js/wcs.listing.js
166 166
      if (window.history) {
167 167
        window.history.replaceState(null, null, pathname + '?' + listing_settings.qs);
168 168
      }
169
      /* refresh dynamic filters */
170
      $('[data-refresh-options]').each(function(idx, elem) {
171
        var $select = $(elem);
172
        var current_value = $select.val();
173
        var filter_path = pathname + 'filter-options?filter_field_id=' + $(elem).data('refresh-options') + '&' + listing_settings.qs;
174
        $.ajax({
175
          url: filter_path,
176
          success: function(data) {
177
            $select.empty();
178
            var $option = $('<option></option>', {value: ''});
179
            $option.appendTo($select);
180
            for (var i=0; i<data.data.length; i++) {
181
              var $option = $('<option></option>', {value: data.data[i].id, text: data.data[i].text});
182
              if (data.data[i].id == current_value) {
183
                $option.attr('selected', 'selected');
184
              }
185
              $option.appendTo($select);
186
            }
187
          }
188
        });
189
      });
190

  
169 191
      /* makes sure activity and disabled-during-submit are removed */
170 192
      $('#more-user-links, #listing, #statistics').removeClass('activity');
171 193
      $('form').removeClass('disabled-during-submit');
......
236 258
    return false;
237 259
  });
238 260

  
261
  /* set filter options from server (select2) */
262
  $('[data-remote-options]').each(function(idx, elem) {
263
    var filter_field_id = $(elem).data('remote-options');
264
    var options = {
265
      language: {
266
        errorLoading: function() { return WCS_I18N.s2_errorloading; },
267
        noResults: function () { return WCS_I18N.s2_nomatches; },
268
        inputTooShort: function (input, min) { return WCS_I18N.s2_tooshort; },
269
        loadingMore: function () { return WCS_I18N.s2_loadmore; },
270
        searching: function () { return WCS_I18N.s2_searching; },
271
      },
272
      placeholder: '',
273
      allowClear: true,
274
      minimumInputLength: 1,
275
      ajax: {
276
        url: function() {
277
          var pathname = window.location.pathname.replace(/^\/+/, '/');
278
          var filter_settings = $('form#listing-settings').serialize();
279
          return pathname + 'filter-options?filter_field_id=' + filter_field_id + '&' + filter_settings;
280
        },
281
        dataType: 'json',
282
        data: function(params) {
283
          var query = {
284
            _search: params.term,
285
          }
286
          return query;
287
        },
288
        processResults: function (data, params) {
289
          return {results: data.data};
290
        },
291
      },
292
    };
293
    $(elem).select2(options);
294
  });
295

  
239 296
  $('button.pdf').click(function() {
240 297
    if (window.location.href.indexOf('?') == -1) {
241 298
      window.location = window.location + '?pdf=on';
wcs/sql.py
1260 1260
        cur.close()
1261 1261
        return ids
1262 1262

  
1263
    @classmethod
1264
    @guard_postgres
1265
    def select_distinct(cls, columns, clause=None):
1266
        conn, cur = get_connection_and_cursor()
1267
        sql_statement = 'SELECT DISTINCT ON (%s) %s FROM %s' % (columns[0], ', '.join(columns), cls._table_name)
1268
        where_clauses, parameters, func_clause = parse_clause(clause)
1269
        assert not func_clause
1270
        if where_clauses:
1271
            sql_statement += ' WHERE ' + ' AND '.join(where_clauses)
1272
        sql_statement += ' ORDER BY %s' % columns[0]
1273
        cur.execute(sql_statement, parameters)
1274
        values = [x for x in cur.fetchall()]
1275
        conn.commit()
1276
        cur.close()
1277
        return values
1278

  
1263 1279

  
1264 1280
class SqlDataMixin(SqlMixin):
1265 1281
    _names = None # make sure StorableObject methods fail
1266
-