From b7d27c62a59ce7cb5555e7b9f46ff38409f2a5d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 18 Jul 2022 13:46:59 +0200
Subject: [PATCH 1/3] backoffice: add a json export format for cards (#60303)

---
 tests/backoffice_pages/test_export.py |  53 +++++++++
 wcs/backoffice/data_management.py     |   2 +
 wcs/backoffice/management.py          | 163 +++++++++++++++++++-------
 wcs/formdata.py                       |  59 +++++-----
 4 files changed, 209 insertions(+), 68 deletions(-)

diff --git a/tests/backoffice_pages/test_export.py b/tests/backoffice_pages/test_export.py
index 656c8a3d2..1bf8416dd 100644
--- a/tests/backoffice_pages/test_export.py
+++ b/tests/backoffice_pages/test_export.py
@@ -1,5 +1,6 @@
 import datetime
 import io
+import json
 import os
 import time
 import urllib.parse
@@ -10,8 +11,10 @@ import pytest
 
 from wcs import fields
 from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
 from wcs.formdef import FormDef
 from wcs.qommon import ods
+from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
 
@@ -867,3 +870,53 @@ def test_backoffice_header_line(pub):
     resp = resp.form.submit('submit')
     assert resp.headers['content-type'].startswith('text/')
     assert len(resp.text.splitlines()) == 3
+
+
+def test_backoffice_no_json(pub):
+    create_superuser(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export a Spreadsheet')
+    assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv']
+
+
+def test_backoffice_cards_json(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'card title'
+    carddef.fields = [
+        fields.StringField(id='1', label='1st field', type='string'),
+    ]
+    carddef.workflow_roles = {'_receiver': 1}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    for i in range(10):
+        carddata = carddef.data_class()()
+        carddata.data = {'1': 'foo %s' % i}
+        carddata.just_created()
+        carddata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/data/card-title/')
+    resp = resp.click('Export Data')
+    assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv', 'json']
+    resp.form['format'] = 'json'
+    resp = resp.form.submit('submit')
+    parsed_url = urllib.parse.urlparse(resp.location)
+    assert parsed_url.path == '/backoffice/processing'
+    job_id = urllib.parse.parse_qs(urllib.parse.urlparse(resp.location).query)['job'][0]
+    job = AfterJob.get(job_id)
+    assert job.completion_time
+    json_export = json.loads(job.file_content)
+    assert len(json_export['data']) == 10
diff --git a/wcs/backoffice/data_management.py b/wcs/backoffice/data_management.py
index 1d8a7d519..1ff46a9f0 100644
--- a/wcs/backoffice/data_management.py
+++ b/wcs/backoffice/data_management.py
@@ -121,8 +121,10 @@ class CardPage(FormPage):
     ]
     admin_permission = 'cards'
     formdef_class = CardDef
+    export_data_label = _('Export Data')
     search_label = _('Search in card content')
     formdef_view_label = _('View Card')
+    has_json_export_support = True
 
     @property
     def add(self):
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index f5ddcfc4e..5cc136b71 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -763,8 +763,10 @@ class FormPage(Directory):
     _view = None
     default_view = None
     use_default_view = False
+    has_json_export_support = False
     admin_permission = 'forms'
     formdef_class = FormDef
+    export_data_label = _('Export a Spreadsheet')
     search_label = _('Search in form content')
     formdef_view_label = _('View Form')
     WCS_SYNC_EXPORT_LIMIT = 100  # Arbitrary threshold
@@ -839,10 +841,7 @@ class FormPage(Directory):
             r += htmltext(
                 ' <li><a rel="popup" data-base-href="export-spreadsheet" data-autoclose-dialog="true" '
                 'href="export-spreadsheet%s">%s</a></li>'
-            ) % (
-                qs,
-                _('Export a Spreadsheet'),
-            )
+            ) % (qs, self.export_data_label)
         if self.formdef.geolocations:
             r += htmltext(' <li><a data-base-href="map" href="map%s">%s</a></li>') % (qs, _('Plot on a Map'))
         if 'stats' in self._q_exports and (
@@ -2205,26 +2204,38 @@ class FormPage(Directory):
             raise errors.AccessForbiddenError()
         form = Form()
         form.add_hidden('query_string', get_request().get_query())
+        formats = [
+            ('ods', _('OpenDocument (.ods)'), 'ods'),
+            ('csv', _('Text (.csv)'), 'csv'),
+        ]
+        if self.has_json_export_support:
+            formats.append(('json', _('JSON'), 'json'))
+
         form.add(
             RadiobuttonsWidget,
             'format',
-            options=[('ods', _('OpenDocument (.ods)'), 'ods'), ('csv', _('Text (.csv)'), 'csv')],
+            options=formats,
             value='ods',
             required=True,
             extra_css_class='widget-inline-radio',
+            attrs={'data-dynamic-display-parent': 'true'},
         )
         form.add(
             CheckboxWidget,
             'include_header_line',
             title=_('Include header line'),
             value=True,
+            attrs={
+                'data-dynamic-display-child-of': 'format',
+                'data-dynamic-display-value-in': 'csv|ods',
+            },
         )
         form.add_submit('submit', _('Export'))
         form.add_submit('cancel', _('Cancel'))
 
         if not form.is_submitted() or form.has_errors():
             r = TemplateIO(html=True)
-            r += htmltext('<h2>%s</h2>') % _('Spreadsheet Options')
+            r += htmltext('<h2>%s</h2>') % _('Export Options')
             r += form.render()
             return r.getvalue()
 
@@ -2233,6 +2244,8 @@ class FormPage(Directory):
         file_format = form.get_widget('format').parse()
         if file_format == 'csv':
             return self.csv()
+        elif file_format == 'json':
+            return self.export_json_file()
         else:
             return self.ods()
 
@@ -2335,6 +2348,29 @@ class FormPage(Directory):
             response.set_header('content-disposition', 'attachment; filename=%s.ods' % self.formdef.url_name)
             return job.file_content
 
+    def export_json_file(self):
+        fields = self.get_fields_from_query()
+        selected_filter = self.get_filter_from_query()
+        selected_filter_operator = self.get_filter_operator_from_query()
+        user = get_request().user
+        query = get_request().form.get('q')
+        criterias = self.get_criterias_from_query()
+        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
+
+        job = JsonFileExportAfterJob(
+            self.formdef,
+            fields=fields,
+            selected_filter=selected_filter,
+            selected_filter_operator=selected_filter_operator,
+            user_id=user.id,
+            query=query,
+            criterias=criterias,
+            order_by=order_by,
+        )
+        job = get_response().add_after_job(job)
+        job.store()
+        return redirect(job.get_processing_url())
+
     def json(self):
         self.view_type = 'json'
         anonymise = get_request().has_anonymised_data_api_restriction()
@@ -2373,40 +2409,15 @@ class FormPage(Directory):
             if view_digest_key in (self.formdef.digest_templates or {}):
                 digest_key = view_digest_key
         if get_request().form.get('full') == 'on':
-            output = []
-            prefetched_users = None
-            prefetched_roles = None
-            prefetched_users = {
-                str(x.id): x
-                for x in get_publisher().user_class.get_ids(
-                    [x.user_id for x in items if x.user_id], ignore_errors=True
-                )
-                if x is not None
-            }
-            role_ids = set((self.formdef.workflow_roles or {}).values())
-            for filled in items:
-                if filled.workflow_roles:
-                    for value in filled.workflow_roles.values():
-                        if not isinstance(value, list):
-                            value = [value]
-                        role_ids |= set(value)
-            prefetched_roles = {
-                str(x.id): x
-                for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True)
-                if x is not None
-            }
-            for filled in items:
-                data = filled.get_json_export_dict(
-                    include_files=False,
-                    anonymise=anonymise,
-                    user=user,
-                    digest_key=digest_key,
-                    prefetched_users=prefetched_users,
-                    prefetched_roles=prefetched_roles,
-                )
-                data.pop('digests')
-                data['digest'] = (filled.digests or {}).get(digest_key)
-                output.append(data)
+            output = JsonFileExportAfterJob(self.formdef).create_json_export(
+                items,
+                user=user,
+                anonymise=anonymise,
+                digest_key=digest_key,
+                include_evolution=True,
+                include_files=False,
+                include_roles=True,
+            )
         else:
             output = [
                 {
@@ -4187,3 +4198,75 @@ class OdsExportAfterJob(CsvExportAfterJob):
         self.file_content = output.getvalue()
         self.content_type = 'application/vnd.oasis.opendocument.spreadsheet'
         self.store()
+
+
+class JsonFileExportAfterJob(CsvExportAfterJob):
+    label = _('Exporting to JSON file')
+
+    def __init__(self, formdef, **kwargs):
+        super().__init__(formdef=formdef, **kwargs)
+        self.file_name = '%s.json' % formdef.url_name
+
+    def create_json_export(
+        self, items, user, anonymise, digest_key, include_evolution, include_files, include_roles
+    ):
+        formdef = self.kwargs['formdef_class'].get(self.kwargs['formdef_id'])
+        prefetched_users = None
+        prefetched_roles = None
+        prefetched_users = {
+            str(x.id): x
+            for x in get_publisher().user_class.get_ids(
+                [x.user_id for x in items if x.user_id], ignore_errors=True
+            )
+            if x is not None
+        }
+        if include_roles:
+            role_ids = set((formdef.workflow_roles or {}).values())
+            for formdata in items:
+                if formdata.workflow_roles:
+                    for value in formdata.workflow_roles.values():
+                        if not isinstance(value, list):
+                            value = [value]
+                        role_ids |= set(value)
+            prefetched_roles = {
+                str(x.id): x
+                for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True)
+                if x is not None
+            }
+        output = []
+        for formdata in items:
+            data = formdata.get_json_export_dict(
+                anonymise=anonymise,
+                user=user,
+                digest_key=digest_key,
+                prefetched_users=prefetched_users,
+                prefetched_roles=prefetched_roles,
+                include_evolution=include_evolution,
+                include_files=include_files,
+                include_roles=include_roles,
+            )
+            data.pop('digests')
+            if digest_key:
+                data['digest'] = (formdata.digests or {}).get(digest_key)
+            output.append(data)
+            self.increment_count()
+        return output
+
+    def create_export(self, formdef, fields, items, total_count):
+        self.file_content = json.dumps(
+            {
+                'data': self.create_json_export(
+                    items,
+                    user=None,
+                    anonymise=False,
+                    digest_key=None,
+                    include_evolution=False,
+                    include_files=True,
+                    include_roles=False,
+                )
+            },
+            indent=2,
+            cls=misc.JSONEncoder,
+        )
+        self.content_type = 'application/json'
+        self.store()
diff --git a/wcs/formdata.py b/wcs/formdata.py
index dbbdecc72..22fa1114a 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -1306,6 +1306,8 @@ class FormData(StorableObject):
         digest_key='default',
         prefetched_users=None,
         prefetched_roles=None,
+        include_evolution=True,
+        include_roles=True,
     ):
         data = {}
         data['id'] = str(self.id)
@@ -1351,33 +1353,34 @@ class FormData(StorableObject):
                 anonymise=anonymise,
             )
 
-        # add a roles dictionary, with workflow functions and two special
-        # entries for concerned/actions roles.
-        data['roles'] = {}
-        workflow_roles = {}
-        if self.formdef.workflow_roles:
-            workflow_roles.update(self.formdef.workflow_roles)
-        if self.workflow_roles:
-            workflow_roles.update(self.workflow_roles)
-        for workflow_role in workflow_roles:
-            value = workflow_roles.get(workflow_role)
-            if not isinstance(value, list):
-                value = [value]
-            data['roles'][workflow_role] = value
-        data['roles']['concerned'] = self.get_concerned_roles()
-        data['roles']['actions'] = self.get_actions_roles()
-
-        for role_key in data['roles']:
-            # exclude special _submitter value
-            role_list = [x for x in data['roles'][role_key] if x != '_submitter']
-            # get role objects
-            if prefetched_roles is not None:
-                role_list = [prefetched_roles.get(str(x)) for x in role_list]
-            else:
-                role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list]
-            # export as json dicts
-            role_list = [x.get_json_export_dict() for x in role_list if x is not None]
-            data['roles'][role_key] = role_list
+        if include_roles:
+            # add a roles dictionary, with workflow functions and two special
+            # entries for concerned/actions roles.
+            data['roles'] = {}
+            workflow_roles = {}
+            if self.formdef.workflow_roles:
+                workflow_roles.update(self.formdef.workflow_roles)
+            if self.workflow_roles:
+                workflow_roles.update(self.workflow_roles)
+            for workflow_role in workflow_roles:
+                value = workflow_roles.get(workflow_role)
+                if not isinstance(value, list):
+                    value = [value]
+                data['roles'][workflow_role] = value
+            data['roles']['concerned'] = self.get_concerned_roles()
+            data['roles']['actions'] = self.get_actions_roles()
+
+            for role_key in data['roles']:
+                # exclude special _submitter value
+                role_list = [x for x in data['roles'][role_key] if x != '_submitter']
+                # get role objects
+                if prefetched_roles is not None:
+                    role_list = [prefetched_roles.get(str(x)) for x in role_list]
+                else:
+                    role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list]
+                # export as json dicts
+                role_list = [x.get_json_export_dict() for x in role_list if x is not None]
+                data['roles'][role_key] = role_list
 
         data['submission'] = {
             'backoffice': self.backoffice_submission,
@@ -1391,7 +1394,7 @@ class FormData(StorableObject):
                 'api_url': parent.get_api_url(),
             }
 
-        if self.evolution:
+        if self.evolution and include_evolution:
             evolution = data['evolution'] = []
             for evo in self.evolution:
                 evolution.append(
-- 
2.36.1

