From 7aecf6608a8b2b747de0dad456e846539f7835b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 9 Dec 2019 17:03:09 +0100
Subject: [PATCH] backoffice: use actual data for item(s) field filters
 (#35703)

---
 tests/test_backoffice_pages.py      | 148 ++++++++++++++++++++++++++--
 wcs/backoffice/management.py        | 115 +++++++++++++++++----
 wcs/fields.py                       |  19 ++--
 wcs/formdata.py                     |  10 +-
 wcs/forms/backoffice.py             |   1 -
 wcs/qommon/static/js/wcs.listing.js |  57 +++++++++++
 wcs/sql.py                          |  16 +++
 7 files changed, 335 insertions(+), 31 deletions(-)

diff --git a/tests/test_backoffice_pages.py b/tests/test_backoffice_pages.py
index ee73680ea..54e633f17 100644
--- a/tests/test_backoffice_pages.py
+++ b/tests/test_backoffice_pages.py
@@ -694,6 +694,135 @@ def test_backoffice_bool_filter(pub):
     assert resp.text.count('<td>Yes</td>') == 0
     assert resp.text.count('<td>No</td>') > 0
 
+def test_backoffice_item_filter(pub):
+    create_superuser(pub)
+    create_environment(pub)
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
+        items=['a', 'b', 'c', 'd'],
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.store()
+
+    for i, formdata in enumerate(formdef.data_class().select()):
+        if i%4 == 0:
+            formdata.data['4'] = 'a'
+            formdata.data['4_display'] = 'a'
+        elif i%4 == 1:
+            formdata.data['4'] = 'b'
+            formdata.data['4_display'] = 'b'
+        elif i%4 == 2:
+            formdata.data['4'] = 'd'
+            formdata.data['4_display'] = 'd'
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp.form['filter-4'].checked = True
+    resp = resp.form.submit()
+
+    assert resp.form['filter-4-value'].value == ''
+
+    resp.form['filter-4-value'].value = 'a'
+    resp = resp.form.submit()
+    assert resp.text.count('<td>a</td>') > 0
+    assert resp.text.count('<td>b</td>') == 0
+    assert resp.text.count('<td>d</td>') == 0
+
+    resp.form['filter-4-value'].value = 'b'
+    resp = resp.form.submit()
+    assert resp.text.count('<td>a</td>') == 0
+    assert resp.text.count('<td>b</td>') > 0
+    assert resp.text.count('<td>d</td>') == 0
+
+    if pub.is_using_postgresql():
+        # option 'c' is never used so not even listed
+        with pytest.raises(ValueError):
+            resp.form['filter-4-value'].value = 'c'
+
+        # check json view used to fill select filters from javascript
+        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&' + resp.request.query_string)
+        assert [x['id'] for x in resp2.json['data']] == ['a', 'b', 'd']
+        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=4&_search=d&' + resp.request.query_string)
+        assert [x['id'] for x in resp2.json['data']] == ['d']
+        resp2 = app.get(resp.request.path + 'filter-options?filter_field_id=7&' + resp.request.query_string, status=404)
+
+    else:
+        # in pickle all options are always displayed
+        resp.form['filter-4-value'].value = 'c'
+        resp = resp.form.submit()
+        assert resp.text.count('<td>a</td>') == 0
+        assert resp.text.count('<td>b</td>') == 0
+        assert resp.text.count('<td>c</td>') == 0
+
+def test_backoffice_item_double_filter(pub):
+    if not pub.is_using_postgresql():
+        pytest.skip('this requires SQL')
+        return
+    create_superuser(pub)
+    create_environment(pub)
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.fields.append(fields.ItemField(id='4', label='4th field', type='item',
+        items=['a', 'b', 'c', 'd'],
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.ItemField(id='5', label='5th field', type='item',
+        items=['A', 'B', 'C', 'D'],
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.store()
+
+    for i, formdata in enumerate(formdef.data_class().select()):
+        if i%4 == 0:
+            formdata.data['4'] = 'a'
+            formdata.data['4_display'] = 'a'
+            formdata.data['5'] = 'A'
+            formdata.data['5_display'] = 'A'
+        elif i%4 == 1:
+            formdata.data['4'] = 'a'
+            formdata.data['4_display'] = 'a'
+            formdata.data['5'] = 'B'
+            formdata.data['5_display'] = 'B'
+        elif i%4 == 2:
+            formdata.data['4'] = 'a'
+            formdata.data['4_display'] = 'a'
+            formdata.data['5'] = 'C'
+            formdata.data['5_display'] = 'C'
+        elif i%4 == 3:
+            formdata.data['4'] = 'b'
+            formdata.data['4_display'] = 'b'
+            formdata.data['5'] = 'B'
+            formdata.data['5_display'] = 'B'
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp.form['filter-4'].checked = True
+    resp.form['filter-5'].checked = True
+    resp = resp.form.submit()
+
+    assert resp.form['filter-4-value'].value == ''
+    assert resp.form['filter-5-value'].value == ''
+    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
+    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
+
+    resp.form['filter-4-value'].value = 'a'
+    resp = resp.form.submit()
+    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
+    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
+
+    resp.form['filter-4-value'].value = 'b'
+    resp = resp.form.submit()
+    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
+    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
+
+    resp.form['filter-5-value'].value = 'B'
+    resp = resp.form.submit()
+    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
+    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'B']
+
+    resp.form['filter-4-value'].value = ''
+    resp = resp.form.submit()
+    assert [x[0] for x in resp.form['filter-4-value'].options] == ['', 'a', 'b']
+    assert [x[0] for x in resp.form['filter-5-value'].options] == ['', 'A', 'B', 'C']
+
 def test_backoffice_items_filter(pub):
     create_superuser(pub)
     create_environment(pub)
@@ -734,12 +863,17 @@ def test_backoffice_items_filter(pub):
     assert resp.text.count('<td>a</td>') == 0
     assert resp.text.count('<td>b, d</td>') > 0
 
-    resp.form['filter-4-value'].value = 'c'
-    resp = resp.form.submit()
-    assert resp.text.count('<td>a, b</td>') == 0
-    assert resp.text.count('<td>a</td>') == 0
-    assert resp.text.count('<td>b, d</td>') == 0
-    assert resp.text.count('data-link') == 0 # no rows
+    if pub.is_using_postgresql():
+        # option 'c' is never used so not even listed
+        with pytest.raises(ValueError):
+            resp.form['filter-4-value'].value = 'c'
+    else:
+        resp.form['filter-4-value'].value = 'c'
+        resp = resp.form.submit()
+        assert resp.text.count('<td>a, b</td>') == 0
+        assert resp.text.count('<td>a</td>') == 0
+        assert resp.text.count('<td>b, d</td>') == 0
+        assert resp.text.count('data-link') == 0 # no rows
 
 def test_backoffice_csv(pub):
     create_superuser(pub)
@@ -789,6 +923,8 @@ def test_backoffice_csv(pub):
 
     resp.forms[0]['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
     resp = resp.forms[0].submit()
+    resp.forms[0]['filter-2-value'] = 'baz'
+    resp = resp.forms[0].submit()
     resp_csv = resp.click('Export as CSV File')
     assert len(resp_csv.text.splitlines()) == 9
     assert 'Created' in resp_csv.text.splitlines()[0]
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index c5a9dd9dc..1107b3477 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -50,7 +50,7 @@ from ..qommon import errors
 from ..qommon import ods
 from ..qommon.form import *
 from ..qommon.storage import (Equal, NotEqual, LessOrEqual, GreaterOrEqual, Or,
-        Intersects, ILike, FtsMatch, Contains, Null)
+        Intersects, ILike, FtsMatch, Contains, Null, NotNull)
 
 from wcs.api_utils import get_user_from_api_query_string
 from wcs.conditions import Condition
@@ -998,7 +998,7 @@ class ManagementDirectory(Directory):
 
 class FormPage(Directory):
     _q_exports = ['', 'csv', 'stats', 'xls', 'ods', 'json', 'export', 'map',
-                  'geojson']
+                  'geojson', ('filter-options', 'filter_options')]
 
     def __init__(self, component):
         try:
@@ -1049,7 +1049,60 @@ class FormPage(Directory):
             return ('start', 'end')
         return ()
 
-    def get_filter_sidebar(self, selected_filter=None, mode='listing'):
+    def get_item_filter_options(self, filter_field, selected_filter, criterias):
+        criterias = (criterias or [])[:]
+        # remove potential filter on self (Equal for item, Intersects for items)
+        criterias = [x for x in criterias if not (isinstance(x, (Equal, Intersects)) and
+                                                  x.attribute == 'f%s' % filter_field.id)]
+        # apply other filters
+        criterias.append(Null('anonymised'))
+        if selected_filter == 'all':
+            criterias.append(NotEqual('status', 'draft'))
+        elif selected_filter in ('waiting', 'pending'):
+            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_not_endpoint_status()]
+            criterias.append(Contains('status', statuses))
+            if selected_filter == 'waniting':
+                user_roles = [logged_users_role().id] + user.get_roles()
+                criterias.append(Intersects('actions_roles_array', user_roles))
+        elif selected_filter == 'done':
+            statuses = ['wf-%s' % x.id for x in self.formdef.workflow.get_endpoint_status()]
+            criterias.append(Contains('status', statuses))
+        criterias.append(NotNull('f%s' % filter_field.id))
+        options = self.formdef.data_class().select_distinct(
+                ['f%s' % filter_field.id, 'f%s_display' % filter_field.id],
+                clause=criterias)
+
+        if filter_field.type == 'items':
+            # unnest key/values
+            exploded_options = {}
+            for option_keys, option_label in options:
+                for option_key, option_label in zip(option_keys, option_label.split(', ')):
+                    exploded_options[option_key] = option_label
+            options = list(sorted(exploded_options.items(), key=lambda x: x[1]))
+
+        return options
+
+    def filter_options(self):
+        get_request().is_json_marker = True
+        field_id = get_request().form.get('filter_field_id')
+        for filter_field in self.get_formdef_fields():
+            if filter_field.id == field_id:
+                break
+        else:
+            raise errors.TraversalError()
+
+        selected_filter = self.get_filter_from_query()
+        criterias = self.get_criterias_from_query()
+        options = self.get_item_filter_options(filter_field, selected_filter, criterias)
+        if get_request().form.get('_search'):  # select2
+            term = get_request().form.get('_search')
+            if term:
+                options = [x for x in options if term.lower() in x[1].lower()]
+            options = options[:15]
+        get_response().set_content_type('application/json')
+        return json.dumps({'err': 0, 'data': [{'id': x[0], 'text': x[1]} for x in options]})
+
+    def get_filter_sidebar(self, selected_filter=None, mode='listing', query=None, criterias=None):
         r = TemplateIO(html=True)
 
         waitpoint_status = self.formdef.workflow.get_waitpoint_status()
@@ -1119,22 +1172,48 @@ class FormPage(Directory):
 
             elif filter_field.type in ('item', 'items'):
                 filter_field.required = False
-                options = filter_field.get_options()
-                if options:
-                    if len(options[0]) == 2:
+
+                if get_publisher().is_using_postgresql():
+                    # Get options from existing formdatas.
+                    # This allows for options that don't appear anymore in the
+                    # data source to be listed (for example because the field
+                    # is using a parametrized URL depending on unavailable
+                    # variables, or simply returning different results now).
+                    display_mode = 'select'
+                    if filter_field.type == 'item' and filter_field.get_display_mode() == 'autocomplete':
+                        display_mode = 'select2'
+
+                    if display_mode == 'select':
+                        options = self.get_item_filter_options(
+                                filter_field, selected_filter, criterias)
                         options = [(x[0], x[1], x[0]) for x in options]
-                    options.insert(0, (None, '', ''))
+                        options.insert(0, (None, '', ''))
+                        attrs = {'data-refresh-options': str(filter_field.id)}
+                    else:
+                        current_filter = get_request().form.get('filter-%s-value' % filter_field.id)
+                        options = [(current_filter, '', current_filter or '')]
+                        attrs = {'data-remote-options': str(filter_field.id)}
+                        get_response().add_javascript(['jquery.js', '../../i18n.js', 'qommon.forms.js', 'select2.js'])
+                        get_response().add_css_include('../js/select2/select2.css')
+
                     r += SingleSelectWidget(filter_field_key, title=filter_field.label,
                             options=options, value=filter_field_value,
-                            render_br=False).render()
+                            render_br=False, attrs=attrs).render()
+
                 else:
-                    # There may be no options because the field is using
-                    # a jsonp data source, or a json source using a
-                    # parametrized URL depending on unavailable variables.
-                    #
-                    # In that case fall back on a string widget.
-                    r += StringWidget(filter_field_key, title=filter_field.label,
-                            value=filter_field_value, render_br=False).render()
+                    # In pickle environments, get options from data source
+                    options = filter_field.get_options()
+                    if options:
+                        if len(options[0]) == 2:
+                            options = [(x[0], x[1], x[0]) for x in options]
+                        options.insert(0, (None, '', ''))
+                        r += SingleSelectWidget(filter_field_key, title=filter_field.label,
+                                options=options, value=filter_field_value,
+                                render_br=False).render()
+                    else:
+                        # and fall back on a string widget if there are none.
+                        r += StringWidget(filter_field_key, title=filter_field.label,
+                                value=filter_field_value, render_br=False).render()
 
             elif filter_field.type == 'bool':
                 options = [(None, '', ''), (True, _('Yes'), 'true'), (False, _('No'), 'false')]
@@ -1160,7 +1239,8 @@ class FormPage(Directory):
         return r.getvalue()
 
     def get_fields_sidebar(self, selected_filter, fields, offset=None,
-            limit=None, order_by=None, columns_settings_label=None):
+            limit=None, order_by=None, columns_settings_label=None,
+            query=None, criterias=None):
         get_response().add_javascript(['jquery.js', 'jquery-ui.js', 'wcs.listing.js'])
 
         r = TemplateIO(html=True)
@@ -1186,7 +1266,7 @@ class FormPage(Directory):
                 r += htmltext('<input class="inline-input" name="q">')
             r += htmltext('<input type="submit" class="side-button" value="%s"/>') % _('Search')
 
-        r += self.get_filter_sidebar(selected_filter=selected_filter)
+        r += self.get_filter_sidebar(selected_filter=selected_filter, query=query, criterias=criterias)
 
         r += htmltext('<button class="refresh">%s</button>') % _('Refresh')
 
@@ -1421,6 +1501,7 @@ class FormPage(Directory):
 
         get_response().filter['sidebar'] = self.get_formdata_sidebar(qs) + \
                 self.get_fields_sidebar(selected_filter, fields, limit=limit,
+                        query=query, criterias=criterias,
                         offset=offset, order_by=order_by,
                         columns_settings_label=_('Columns Settings'))
 
diff --git a/wcs/fields.py b/wcs/fields.py
index 28f9545fa..9416fd78d 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -1307,14 +1307,21 @@ class ItemField(WidgetField):
             return [(x, x) for x in self.items]
         return []
 
-    def perform_more_widget_changes(self, form, kwargs, edit=True):
-        data_source = data_sources.get_object(self.data_source)
+    def get_display_mode(self, data_source=None):
+        if not data_source:
+            data_source = data_sources.get_object(self.data_source)
 
         if data_source and data_source.type == 'jsonp':
             # a source defined as JSONP can only be used in autocomplete mode
-            self.display_mode = 'autocomplete'
+            return 'autocomplete'
+
+        return self.display_mode
+
+    def perform_more_widget_changes(self, form, kwargs, edit=True):
+        data_source = data_sources.get_object(self.data_source)
+        display_mode = self.get_display_mode(data_source)
 
-        if self.display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
+        if display_mode == 'autocomplete' and data_source and data_source.can_jsonp():
             self.url = kwargs['url'] = data_source.get_jsonp_url()
             self.widget_class = JsonpSingleSelectWidget
             return
@@ -1330,7 +1337,7 @@ class ItemField(WidgetField):
             kwargs['options'] = self.get_options()
         if not kwargs.get('options'):
             kwargs['options'] = [(None, '---')]
-        if self.display_mode == 'radio':
+        if display_mode == 'radio':
             self.widget_class = RadiobuttonsWidget
             if type(kwargs['options'][0]) is str:
                 first_items = [x for x in kwargs['options'][:3]]
@@ -1340,7 +1347,7 @@ class ItemField(WidgetField):
             if len(kwargs['options']) > 3 or length_first_items > 40:
                 # TODO: absence/presence of delimitor should be an option
                 kwargs['delim'] = htmltext('<br />')
-        elif self.display_mode == 'autocomplete':
+        elif display_mode == 'autocomplete':
             kwargs['select2'] = True
 
     def get_display_value(self, value):
diff --git a/wcs/formdata.py b/wcs/formdata.py
index 5688924f2..3b48a8a28 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -1101,7 +1101,15 @@ class FormData(StorableObject):
         except KeyError:
             # give direct access to values from the data dictionary
             if attr[0] == 'f':
-                return self.__dict__['data'][attr[1:]]
+                field_id = attr[1:]
+                if field_id in self.__dict__['data']:
+                    return self.__dict__['data'][field_id]
+                # if field id is not in data dictionary it may still be a valid
+                # field, never initialized, check requested field id against
+                # existing fields ids.
+                formdef_fields = self.formdef.get_all_fields()
+                if field_id in [x.id for x in formdef_fields]:
+                    return None
             raise AttributeError(attr)
 
     # don't pickle _formdef cache
diff --git a/wcs/forms/backoffice.py b/wcs/forms/backoffice.py
index 15c96949b..68ed6c6e4 100644
--- a/wcs/forms/backoffice.py
+++ b/wcs/forms/backoffice.py
@@ -38,7 +38,6 @@ class FormDefUI(object):
         partial_display = False
         using_postgresql = get_publisher().is_using_postgresql()
 
-
         if not items:
             if offset and not limit:
                 limit = int(get_publisher().get_site_option('default-page-size') or 20)
diff --git a/wcs/qommon/static/js/wcs.listing.js b/wcs/qommon/static/js/wcs.listing.js
index 12ae46a4b..c21c86502 100644
--- a/wcs/qommon/static/js/wcs.listing.js
+++ b/wcs/qommon/static/js/wcs.listing.js
@@ -166,6 +166,28 @@ $(document).on('backoffice-filter-change', function(event, listing_settings) {
       if (window.history) {
         window.history.replaceState(null, null, pathname + '?' + listing_settings.qs);
       }
+      /* refresh dynamic filters */
+      $('[data-refresh-options]').each(function(idx, elem) {
+        var $select = $(elem);
+        var current_value = $select.val();
+        var filter_path = pathname + 'filter-options?filter_field_id=' + $(elem).data('refresh-options') + '&' + listing_settings.qs;
+        $.ajax({
+          url: filter_path,
+          success: function(data) {
+            $select.empty();
+            var $option = $('<option></option>', {value: ''});
+            $option.appendTo($select);
+            for (var i=0; i<data.data.length; i++) {
+              var $option = $('<option></option>', {value: data.data[i].id, text: data.data[i].text});
+              if (data.data[i].id == current_value) {
+                $option.attr('selected', 'selected');
+              }
+              $option.appendTo($select);
+            }
+          }
+        });
+      });
+
       /* makes sure activity and disabled-during-submit are removed */
       $('#more-user-links, #listing, #statistics').removeClass('activity');
       $('form').removeClass('disabled-during-submit');
@@ -236,6 +258,41 @@ $(function() {
     return false;
   });
 
+  /* set filter options from server (select2) */
+  $('[data-remote-options]').each(function(idx, elem) {
+    var filter_field_id = $(elem).data('remote-options');
+    var options = {
+      language: {
+        errorLoading: function() { return WCS_I18N.s2_errorloading; },
+        noResults: function () { return WCS_I18N.s2_nomatches; },
+        inputTooShort: function (input, min) { return WCS_I18N.s2_tooshort; },
+        loadingMore: function () { return WCS_I18N.s2_loadmore; },
+        searching: function () { return WCS_I18N.s2_searching; },
+      },
+      placeholder: '',
+      allowClear: true,
+      minimumInputLength: 1,
+      ajax: {
+        url: function() {
+          var pathname = window.location.pathname.replace(/^\/+/, '/');
+          var filter_settings = $('form#listing-settings').serialize();
+          return pathname + 'filter-options?filter_field_id=' + filter_field_id + '&' + filter_settings;
+        },
+        dataType: 'json',
+        data: function(params) {
+          var query = {
+            _search: params.term,
+          }
+          return query;
+        },
+        processResults: function (data, params) {
+          return {results: data.data};
+        },
+      },
+    };
+    $(elem).select2(options);
+  });
+
   $('button.pdf').click(function() {
     if (window.location.href.indexOf('?') == -1) {
       window.location = window.location + '?pdf=on';
diff --git a/wcs/sql.py b/wcs/sql.py
index 260ab78cb..22bf18b75 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -1260,6 +1260,22 @@ class SqlMixin(object):
         cur.close()
         return ids
 
+    @classmethod
+    @guard_postgres
+    def select_distinct(cls, columns, clause=None):
+        conn, cur = get_connection_and_cursor()
+        sql_statement = 'SELECT DISTINCT ON (%s) %s FROM %s' % (columns[0], ', '.join(columns), cls._table_name)
+        where_clauses, parameters, func_clause = parse_clause(clause)
+        assert not func_clause
+        if where_clauses:
+            sql_statement += ' WHERE ' + ' AND '.join(where_clauses)
+        sql_statement += ' ORDER BY %s' % columns[0]
+        cur.execute(sql_statement, parameters)
+        values = [x for x in cur.fetchall()]
+        conn.commit()
+        cur.close()
+        return values
+
 
 class SqlDataMixin(SqlMixin):
     _names = None # make sure StorableObject methods fail
-- 
2.24.0

