From b015aad5941179e17dcf9540b7ac5eb7d3a6a3de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 20 Jul 2020 22:18:51 +0200
Subject: [PATCH] backoffice: add possibility to display linked card fields as
 columns (#40036)

---
 tests/test_backoffice_pages.py      |  81 +++++++++++++++++++++
 wcs/backoffice/management.py        | 107 ++++++++++++++++++++++------
 wcs/forms/backoffice.py             |  15 ++--
 wcs/qommon/static/css/dc2/admin.css |  21 ++++++
 wcs/qommon/static/js/wcs.listing.js |  11 +++
 wcs/qommon/storage.py               |   2 +-
 wcs/sql.py                          |  64 +++++++++++++----
 7 files changed, 260 insertions(+), 41 deletions(-)

diff --git a/tests/test_backoffice_pages.py b/tests/test_backoffice_pages.py
index e35e01e17..0ce26928b 100644
--- a/tests/test_backoffice_pages.py
+++ b/tests/test_backoffice_pages.py
@@ -126,6 +126,7 @@ def create_environment(pub, set_receiver=True):
     Workflow.wipe()
     Category.wipe()
     FormDef.wipe()
+    CardDef.wipe()
     pub.custom_view_class.wipe()
     formdef = FormDef()
     formdef.name = 'form title'
@@ -646,6 +647,86 @@ def test_backoffice_image_column(pub):
     assert 'download?f=4&thumbnail=1' not in resp.text
 
 
+def test_backoffice_card_field_columns(pub):
+    user = create_superuser(pub)
+    create_environment(pub)
+
+    datasource = {
+        'type': 'formula',
+        'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])
+    }
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.CommentField(id='0', label='...', type='comment'),
+        fields.StringField(id='1', label='Test', type='string', varname='foo'),
+        fields.DateField(id='2', label='Date', type='date'),
+        fields.BoolField(id='3', label='Bool', type='bool'),
+        fields.ItemField(id='4', label='Item', type='item', data_source=datasource),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.digest_template = 'card {{form_var_foo}}'
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {
+        '1': 'plop',
+        '2': time.strptime('2020-04-24', '%Y-%m-%d'),
+        '3': True,
+        '4': 'A',
+        '4_display': 'aa',
+    }
+    card.store()
+
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields.append(
+        fields.ItemField(id='4', label='card field', type='item',
+            data_source={'type': 'carddef:foo', 'value': ''}))
+    formdef.store()
+
+    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
+        formdata.data['4'] = str(card.id)
+        formdata.data['4_display'] = formdef.fields[-1].store_display_value(formdata.data, '4')
+        formdata.data['4_structured'] = formdef.fields[-1].store_structured_value(formdata.data, '4')
+        formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    assert resp.text.count('</th>') == 8  # six columns
+    if not pub.is_using_postgresql():
+        # no support for relation columns unless using SQL
+        assert '4$1' not in resp.forms['listing-settings'].fields
+        return
+    assert '4$0' not in resp.forms['listing-settings'].fields
+    resp.forms['listing-settings']['4$1'].checked = True
+    resp.forms['listing-settings']['4$2'].checked = True
+    resp.forms['listing-settings']['4$3'].checked = True
+    resp.forms['listing-settings']['4$4'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    assert resp.text.count('</th>') == 12
+    assert resp.text.count('data-link') == 17  # 17 rows
+    assert resp.text.count('<td>plop</td>') == 17
+    assert resp.text.count('<td>2020-04-24</td>') == 17
+    assert resp.text.count('<td>Yes</td>') == 17
+    assert resp.text.count('<td>aa</td>') == 17
+
+    resp_csv = resp.click('Export as CSV File')
+    assert resp_csv.text.splitlines()[1].endswith(',plop,2020-04-24,Yes,aa')
+
+    resp_ods = resp.click('Export a Spreadsheet')
+
+    resp_map = resp.click('Plot on a Map')
+    geojson_url = re.findall(r'data-geojson-url="(.*?)"', resp_map.text)[0]
+    resp_geojson = app.get(geojson_url)
+    assert {'varname': None, 'label': 'card field - Test', 'value': 'plop', 'html_value': 'plop'} in resp_geojson.json['features'][0]['properties']['display_fields']
+
+
 def test_backoffice_filter(pub):
     create_superuser(pub)
     create_environment(pub)
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index e4baf76ff..2fb14e0ae 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -1190,7 +1190,7 @@ class FormPage(Directory):
         default_filters = self.get_default_filters(mode)
 
         filter_fields = []
-        for field in fake_fields + self.get_formdef_fields():
+        for field in fake_fields + list(self.get_formdef_fields()):
             field.enabled = False
             if field.type not in self.get_filterable_field_types() + ['status']:
                 continue
@@ -1415,14 +1415,29 @@ class FormPage(Directory):
                     return field_ids.index(x.id)
                 return 9999
 
+            seen_parents = set()
             for field in sorted(self.get_formdef_fields(), key=get_column_position):
                 if not hasattr(field, str('get_view_value')):
                     continue
-                r += htmltext('<li><span class="handle">⣿</span><label><input type="checkbox" name="%s"') % field.id
+                classnames = ''
+                attrs = ''
+                if isinstance(field, RelatedField):
+                    classnames = 'related-field'
+                    if field.parent_field.id in seen_parents:
+                        classnames += ' collapsed'
+                    attrs = 'data-relation-attr="%s"' % field.parent_field.id
+                elif getattr(field, 'has_relations', False):
+                    classnames = 'has-relations-field'
+                    attrs = 'data-field-id="%s"' % field.id
+                    seen_parents.add(field.id)
+                r += htmltext('<li class="%s" %s><span class="handle">⣿</span>' % (classnames, attrs))
+                r += htmltext('<label><input type="checkbox" name="%s"') % field.id
                 if field.id in field_ids:
                     r += htmltext(' checked="checked"')
                 r += htmltext('/>')
                 r += htmltext('%s</label>') % misc.ellipsize(field.label, 70)
+                if getattr(field, 'has_relations', False):
+                    r += htmltext('<button class="expand-relations"></button>')
                 r += htmltext('</li>')
                 column_order.append(str(field.id))
             r += htmltext('</ul>')
@@ -1511,20 +1526,34 @@ class FormPage(Directory):
             return redirect('..')
 
     def get_formdef_fields(self):
-        fields = []
-        fields.append(FakeField('id', 'id', _('Number')))
+        yield FakeField('id', 'id', _('Number'))
         if get_publisher().get_site_option('welco_url', 'variables'):
-            fields.append(FakeField('submission_channel', 'submission_channel', _('Channel')))
+            yield FakeField('submission_channel', 'submission_channel', _('Channel'))
         if self.formdef.backoffice_submission_roles:
-            fields.append(FakeField('submission_agent', 'submission_agent', _('Submission By')))
-        fields.append(FakeField('time', 'time', _('Created')))
-        fields.append(FakeField('last_update_time', 'last_update_time', _('Last Modified')))
-        fields.append(FakeField('user-label', 'user-label', _('User Label')))
-        fields.extend(self.formdef.get_all_fields())
-        fields.append(FakeField('status', 'status', _('Status')))
-        fields.append(FakeField('anonymised', 'anonymised', _('Anonymised')))
+            yield FakeField('submission_agent', 'submission_agent', _('Submission By'))
+        yield FakeField('time', 'time', _('Created'))
+        yield FakeField('last_update_time', 'last_update_time', _('Last Modified'))
+        yield FakeField('user-label', 'user-label', _('User Label'))
+        for field in self.formdef.get_all_fields():
+            yield field
+            if not get_publisher().is_using_postgresql():
+                continue
+            if not (field.type == 'item' and
+                    field.data_source and
+                    field.data_source.get('type', '').startswith('carddef:')):
+                continue
+            try:
+                carddef = CardDef.get_by_urlname(field.data_source['type'][8:])
+            except KeyError:
+                continue
+            for card_field in carddef.get_all_fields():
+                if not hasattr(card_field, 'get_view_value'):
+                    continue
+                field.has_relations = True
+                yield RelatedField(carddef, card_field, field)
 
-        return fields
+        yield FakeField('status', 'status', _('Status'))
+        yield FakeField('anonymised', 'anonymised', _('Anonymised'))
 
     def get_default_columns(self):
         if self.view:
@@ -1592,7 +1621,7 @@ class FormPage(Directory):
             filters_dict.update(self.view.get_filters_dict())
         filters_dict.update(get_request().form)
 
-        for filter_field in fake_fields + self.get_formdef_fields():
+        for filter_field in fake_fields + list(self.get_formdef_fields()):
             if filter_field.type not in self.get_filterable_field_types():
                 continue
 
@@ -1909,7 +1938,7 @@ class FormPage(Directory):
                 csv_output.writerow(self.formpage.csv_tuple_heading(self.fields))
 
                 items, total_count = FormDefUI(self.formdef).get_listing_items(
-                        self.selected_filter, user=user, query=query,
+                        fields, self.selected_filter, user=user, query=query,
                         criterias=criterias)
 
                 for filled in items:
@@ -2011,7 +2040,7 @@ class FormPage(Directory):
                     ws.write(0, i, f)
 
                 items, total_count = FormDefUI(self.formdef).get_listing_items(
-                        self.selected_filter, user=user, query=query,
+                        fields, self.selected_filter, user=user, query=query,
                         criterias=criterias)
 
                 for i, filled in enumerate(items):
@@ -2076,7 +2105,7 @@ class FormPage(Directory):
                     ws.write(0, i, f)
 
                 items, total_count = FormDefUI(self.formdef).get_listing_items(
-                        self.selected_filter, user=user, query=query,
+                        fields, self.selected_filter, user=user, query=query,
                         criterias=criterias)
 
                 for i, formdata in enumerate(items):
@@ -2132,7 +2161,7 @@ class FormPage(Directory):
         if 'limit' in get_request().form:
             limit = misc.get_int_or_400(get_request().form['limit'])
         items, total_count = FormDefUI(self.formdef).get_listing_items(
-            selected_filter, user=user, query=query, criterias=criterias,
+            None, selected_filter, user=user, query=query, criterias=criterias,
             order_by=order_by, anonymise=anonymise, offset=offset, limit=limit)
         if get_publisher().is_using_postgresql():
             self.formdef.data_class().load_all_evolutions(items)
@@ -2175,7 +2204,7 @@ class FormPage(Directory):
         query = get_request().form.get('q')
 
         items, total_count = FormDefUI(self.formdef).get_listing_items(
-                selected_filter, user=user, query=query, criterias=criterias)
+                fields, selected_filter, user=user, query=query, criterias=criterias)
 
         # only consider first key for now
         geoloc_key = list(self.formdef.geolocations.keys())[0]
@@ -2228,7 +2257,7 @@ class FormPage(Directory):
                     raise errors.TraversalError()
 
                 formdatas, total_count = FormDefUI(formdef).get_listing_items(
-                        selected_filter, user=user, query=query, criterias=criterias)
+                        fields, selected_filter, user=user, query=query, criterias=criterias)
 
                 cal = vobject.iCalendar()
                 cal.add('prodid').value = '-//Entr\'ouvert//NON SGML Publik'
@@ -3128,6 +3157,44 @@ class FakeField(object):
         return [element]
 
 
+class RelatedField:
+    is_related_field = True
+    type = 'related-field'
+    store_display_value = None
+    varname = None
+
+    def __init__(self, carddef, field, parent_field):
+        self.carddef = carddef
+        self.carddef_field = field
+        self.parent_field = parent_field
+
+    @property
+    def id(self):
+        return '%s$%s' % (self.parent_field.id, self.carddef_field.id)
+
+    @property
+    def label(self):
+        return '%s - %s' % (self.parent_field.label, self.carddef_field.label)
+
+    def get_view_value(self, value, **kwargs):
+        if value is None:
+            return ''
+        if isinstance(value, bool):
+            return _('Yes') if value else _('No')
+        if isinstance(value, datetime.date):
+            return misc.strftime(misc.date_format(), value)
+        return value
+
+    def get_view_short_value(self, value, max_len=30, **kwargs):
+        return self.get_view_value(value)
+
+    def get_csv_heading(self):
+        return [self.label]
+
+    def get_csv_value(self, value, **kwargs):
+        return [self.get_view_value(value)]
+
+
 def do_graphs_section(period_start=None, period_end=None, criterias=None):
     from wcs import sql
     r = TemplateIO(html=True)
diff --git a/wcs/forms/backoffice.py b/wcs/forms/backoffice.py
index 69f3d0f1b..8eed3edf6 100644
--- a/wcs/forms/backoffice.py
+++ b/wcs/forms/backoffice.py
@@ -47,7 +47,7 @@ class FormDefUI(object):
             if using_postgresql:
                 criterias.append(Null('anonymised'))
             items, total_count = self.get_listing_items(
-                            selected_filter, offset, limit, query, order_by,
+                            fields, selected_filter, offset, limit, query, order_by,
                             criterias=criterias)
             if (offset > 0) or (total_count > limit > 0):
                 partial_display = True
@@ -102,6 +102,8 @@ class FormDefUI(object):
                     field_sort_key = 'receipt_time'
                 elif f.id in ('user-label', 'submission_agent'):
                     field_sort_key = None
+            elif hasattr(f, 'column_id'):
+                field_sort_key = None
             else:
                 field_sort_key = 'f%s' % f.id
 
@@ -178,7 +180,7 @@ class FormDefUI(object):
 
         return item_ids
 
-    def get_listing_items(self, selected_filter='all', offset=None,
+    def get_listing_items(self, fields=None, selected_filter='all', offset=None,
             limit=None, query=None, order_by=None, user=None, criterias=None, anonymise=False):
         user = user or get_request().user
         formdata_class = self.formdef.data_class()
@@ -206,15 +208,18 @@ class FormDefUI(object):
         if not offset:
             offset = 0
 
+        kwargs = {}
+        if get_publisher().is_using_postgresql():
+            kwargs['fields'] = fields
+
         if limit:
             items = formdata_class.get_ids(item_ids[offset:offset+limit],
-                    keep_order=True)
+                    keep_order=True, **kwargs)
         else:
-            items = formdata_class.get_ids(item_ids, keep_order=True)
+            items = formdata_class.get_ids(item_ids, keep_order=True, **kwargs)
 
         return (items, total_count)
 
-
     def tbody(self, fields=None, items=None, url_action=None, include_checkboxes=False):
         r = TemplateIO(html=True)
         if url_action:
diff --git a/wcs/qommon/static/css/dc2/admin.css b/wcs/qommon/static/css/dc2/admin.css
index 7197b1b3f..84a74e27b 100644
--- a/wcs/qommon/static/css/dc2/admin.css
+++ b/wcs/qommon/static/css/dc2/admin.css
@@ -1121,6 +1121,27 @@ ul.columns-filter li label {
 	padding-left: 2em;
 }
 
+ul.columns-filter li.collapsed {
+	display: none;
+}
+
+ul.columns-filter button.expand-relations {
+	position: absolute;
+	right: 0;
+	top: 0;
+	height: 100%;
+	border: none;
+}
+
+ul.columns-filter button.expand-relations::before {
+	font-family: FontAwesome;
+	content: "\f107";  /* angle-down */
+}
+
+ul.columns-filter button.expand-relations.opened::before {
+	content: "\f106";  // angle-up
+}
+
 ul.multipage li {
 	margin-left: 2em;
 }
diff --git a/wcs/qommon/static/js/wcs.listing.js b/wcs/qommon/static/js/wcs.listing.js
index 8bcbc5d8f..f99843824 100644
--- a/wcs/qommon/static/js/wcs.listing.js
+++ b/wcs/qommon/static/js/wcs.listing.js
@@ -213,6 +213,17 @@ $(function() {
     var dialog = $('<form>');
     var $dialog_filter = $('#columns-filter').clone().attr('id', null);
     $dialog_filter.appendTo(dialog);
+    $dialog_filter.find('button.expand-relations').each(function(elem, i) {
+      $(this).removeClass('opened');
+      var field_id = $(this).parents('li.has-relations-field').data('field-id');
+      $(this).parents('li').find('~ li[data-relation-attr=' + field_id + ']').addClass('collapsed');
+    });
+    $dialog_filter.find('button.expand-relations').on('click', function() {
+      $(this).toggleClass('opened');
+      var field_id = $(this).parents('li.has-relations-field').data('field-id');
+      $(this).parents('li').find('~ li[data-relation-attr=' + field_id + ']').toggleClass('collapsed');
+      return false;
+    });
     $dialog_filter.sortable({handle: '.handle'})
     $(dialog).dialog({
             closeText: WCS_I18N.close,
diff --git a/wcs/qommon/storage.py b/wcs/qommon/storage.py
index 1431ad0cc..2a28ead0d 100644
--- a/wcs/qommon/storage.py
+++ b/wcs/qommon/storage.py
@@ -405,7 +405,7 @@ class StorableObject(object):
                                 **kwargs)
 
     @classmethod
-    def get_ids(cls, ids, ignore_errors=False, keep_order=False):
+    def get_ids(cls, ids, ignore_errors=False, **kwargs):
         objects = []
         for x in ids:
             obj = cls.get(x, ignore_errors=ignore_errors)
diff --git a/wcs/sql.py b/wcs/sql.py
index eaa7f4eab..79a74cb30 100644
--- a/wcs/sql.py
+++ b/wcs/sql.py
@@ -1138,19 +1138,45 @@ class SqlMixin(object):
 
     @classmethod
     @guard_postgres
-    def get_ids(cls, ids, ignore_errors=False, keep_order=False):
+    def get_ids(cls, ids, ignore_errors=False, keep_order=False, fields=None):
         if not ids:
             return []
+        tables = [cls._table_name]
+        columns = ['%s.%s' % (cls._table_name, column_name) for column_name in
+                    [x[0] for x in cls._table_static_fields] + cls.get_data_fields()]
+        extra_fields = []
+        if fields:
+            # look for relations
+            for field in fields:
+                if not getattr(field, 'is_related_field', False):
+                    continue
+                carddef_dataclass = field.carddef.data_class()
+                carddef_table_alias = 't%s' % id(field.carddef)
+                carddef_table_decl = 'LEFT JOIN %s AS %s ON (CAST(%s.%s AS INTEGER) = %s.id)' % (
+                        carddef_dataclass._table_name,
+                        carddef_table_alias,
+                        cls._table_name,
+                        get_field_id(field.parent_field),
+                        carddef_table_alias)
+                if carddef_table_decl not in tables:
+                    tables.append(carddef_table_decl)
+
+                column_field_id = get_field_id(field.carddef_field)
+                if field.carddef_field.store_display_value:
+                    column_field_id += '_display'
+                columns.append('%s.%s' % (carddef_table_alias, column_field_id))
+                extra_fields.append(field.id)
+
         conn, cur = get_connection_and_cursor()
         sql_statement = '''SELECT %s
                              FROM %s
-                            WHERE id IN (%s)''' % (
-                                    ', '.join([x[0] for x in cls._table_static_fields]
-                                              + cls.get_data_fields()),
+                            WHERE %s.id IN (%s)''' % (
+                                    ', '.join(columns),
+                                    ' '.join(tables),
                                     cls._table_name,
                                     ','.join([str(x) for x in ids]))
         cur.execute(sql_statement)
-        objects = cls.get_objects(cur)
+        objects = cls.get_objects(cur, extra_fields=extra_fields)
         conn.commit()
         cur.close()
         if ignore_errors:
@@ -1163,16 +1189,19 @@ class SqlMixin(object):
         return list(objects)
 
     @classmethod
-    def get_objects_iterator(cls, cur, ignore_errors=False):
+    def get_objects_iterator(cls, cur, ignore_errors=False, extra_fields=None):
         while True:
             row = cur.fetchone()
             if row is None:
                 break
-            yield cls._row2ob(row)
+            yield cls._row2ob(row, extra_fields=extra_fields)
 
     @classmethod
-    def get_objects(cls, cur, ignore_errors=False, iterator=False):
-        generator = cls.get_objects_iterator(cur=cur, ignore_errors=ignore_errors)
+    def get_objects(cls, cur, ignore_errors=False, iterator=False, extra_fields=None):
+        generator = cls.get_objects_iterator(
+                cur=cur,
+                ignore_errors=ignore_errors,
+                extra_fields=extra_fields)
         if iterator:
             return generator
         return list(generator)
@@ -1665,7 +1694,7 @@ class SqlDataMixin(SqlMixin):
         cur.close()
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, extra_fields=None):
         o = cls()
         for static_field, value in zip(cls._table_static_fields,
                                        tuple(row[:len(cls._table_static_fields)])):
@@ -1689,6 +1718,11 @@ class SqlDataMixin(SqlMixin):
                                      'lat': float(m.group(2))}
 
         o.data = cls._row2obdata(row, cls._formdef)
+        if extra_fields:
+            # extra fields are tuck at the end
+            for i, field_id in enumerate(reversed(extra_fields)):
+                o.data[field_id] = row[-(i + 1)]
+            pass
         del o._last_update_time
         return o
 
@@ -1940,7 +1974,7 @@ class SqlUser(SqlMixin, wcs.users.User):
         cur.close()
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, **kwargs):
         o = cls()
         (o.id, o.name, o.email, o.roles, o.is_admin, o.anonymous,
          o.name_identifiers, o.verified_fields, o.lasso_dump,
@@ -2084,7 +2118,7 @@ class Session(SqlMixin, wcs.sessions.BasicSession):
         cur.close()
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, **kwargs):
         o = cls.__new__(cls)
         cls.id = str_encode(row[0])
         session_data = pickle_loads(row[1])
@@ -2189,7 +2223,7 @@ class TrackingCode(SqlMixin, wcs.tracking_code.TrackingCode):
         cur.close()
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, **kwargs):
         o = cls()
         (o.id, o.formdef_id, o.formdata_id) = [str_encode(x) for x in tuple(row[:3])]
         return o
@@ -2264,7 +2298,7 @@ class CustomView(SqlMixin, wcs.custom_views.CustomView):
         cur.close()
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, **kwargs):
         o = cls()
         for field, value in zip(cls._table_static_fields, tuple(row)):
             if field[1] == 'varchar':
@@ -2316,7 +2350,7 @@ class AnyFormData(SqlMixin):
         return super(AnyFormData, cls).get_objects(*args, **kwargs)
 
     @classmethod
-    def _row2ob(cls, row):
+    def _row2ob(cls, row, **kwargs):
         formdef_id = row[1]
         from wcs.formdef import FormDef
         formdef = cls._formdef_cache.setdefault(formdef_id, FormDef.get(formdef_id))
-- 
2.28.0.rc1

