Projet

Général

Profil

0001-backoffice-add-a-json-export-format-for-cards-60303.patch

Frédéric Péters, 02 août 2022 17:51

Télécharger (16,1 ko)

Voir les différences:

Subject: [PATCH 1/3] backoffice: add a json export format for cards (#60303)

 tests/backoffice_pages/test_export.py |  53 +++++++++
 wcs/backoffice/data_management.py     |   2 +
 wcs/backoffice/management.py          | 163 +++++++++++++++++++-------
 wcs/formdata.py                       |  59 +++++-----
 4 files changed, 209 insertions(+), 68 deletions(-)
tests/backoffice_pages/test_export.py
1 1
import datetime
2 2
import io
3
import json
3 4
import os
4 5
import time
5 6
import urllib.parse
......
10 11

  
11 12
from wcs import fields
12 13
from wcs.blocks import BlockDef
14
from wcs.carddef import CardDef
13 15
from wcs.formdef import FormDef
14 16
from wcs.qommon import ods
17
from wcs.qommon.afterjobs import AfterJob
15 18
from wcs.qommon.http_request import HTTPRequest
16 19
from wcs.qommon.upload_storage import PicklableUpload
17 20

  
......
867 870
    resp = resp.form.submit('submit')
868 871
    assert resp.headers['content-type'].startswith('text/')
869 872
    assert len(resp.text.splitlines()) == 3
873

  
874

  
875
def test_backoffice_no_json(pub):
876
    create_superuser(pub)
877

  
878
    FormDef.wipe()
879
    formdef = FormDef()
880
    formdef.name = 'form title'
881
    formdef.fields = []
882
    formdef.workflow_roles = {'_receiver': 1}
883
    formdef.store()
884

  
885
    app = login(get_app(pub))
886
    resp = app.get('/backoffice/management/form-title/')
887
    resp = resp.click('Export a Spreadsheet')
888
    assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv']
889

  
890

  
891
def test_backoffice_cards_json(pub):
892
    create_superuser(pub)
893

  
894
    CardDef.wipe()
895
    carddef = CardDef()
896
    carddef.name = 'card title'
897
    carddef.fields = [
898
        fields.StringField(id='1', label='1st field', type='string'),
899
    ]
900
    carddef.workflow_roles = {'_receiver': 1}
901
    carddef.store()
902
    carddef.data_class().wipe()
903

  
904
    for i in range(10):
905
        carddata = carddef.data_class()()
906
        carddata.data = {'1': 'foo %s' % i}
907
        carddata.just_created()
908
        carddata.store()
909

  
910
    app = login(get_app(pub))
911
    resp = app.get('/backoffice/data/card-title/')
912
    resp = resp.click('Export Data')
913
    assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv', 'json']
914
    resp.form['format'] = 'json'
915
    resp = resp.form.submit('submit')
916
    parsed_url = urllib.parse.urlparse(resp.location)
917
    assert parsed_url.path == '/backoffice/processing'
918
    job_id = urllib.parse.parse_qs(urllib.parse.urlparse(resp.location).query)['job'][0]
919
    job = AfterJob.get(job_id)
920
    assert job.completion_time
921
    json_export = json.loads(job.file_content)
922
    assert len(json_export['data']) == 10
wcs/backoffice/data_management.py
121 121
    ]
122 122
    admin_permission = 'cards'
123 123
    formdef_class = CardDef
124
    export_data_label = _('Export Data')
124 125
    search_label = _('Search in card content')
125 126
    formdef_view_label = _('View Card')
127
    has_json_export_support = True
126 128

  
127 129
    @property
128 130
    def add(self):
wcs/backoffice/management.py
763 763
    _view = None
764 764
    default_view = None
765 765
    use_default_view = False
766
    has_json_export_support = False
766 767
    admin_permission = 'forms'
767 768
    formdef_class = FormDef
769
    export_data_label = _('Export a Spreadsheet')
768 770
    search_label = _('Search in form content')
769 771
    formdef_view_label = _('View Form')
770 772
    WCS_SYNC_EXPORT_LIMIT = 100  # Arbitrary threshold
......
839 841
            r += htmltext(
840 842
                ' <li><a rel="popup" data-base-href="export-spreadsheet" data-autoclose-dialog="true" '
841 843
                'href="export-spreadsheet%s">%s</a></li>'
842
            ) % (
843
                qs,
844
                _('Export a Spreadsheet'),
845
            )
844
            ) % (qs, self.export_data_label)
846 845
        if self.formdef.geolocations:
847 846
            r += htmltext(' <li><a data-base-href="map" href="map%s">%s</a></li>') % (qs, _('Plot on a Map'))
848 847
        if 'stats' in self._q_exports and (
......
2205 2204
            raise errors.AccessForbiddenError()
2206 2205
        form = Form()
2207 2206
        form.add_hidden('query_string', get_request().get_query())
2207
        formats = [
2208
            ('ods', _('OpenDocument (.ods)'), 'ods'),
2209
            ('csv', _('Text (.csv)'), 'csv'),
2210
        ]
2211
        if self.has_json_export_support:
2212
            formats.append(('json', _('JSON'), 'json'))
2213

  
2208 2214
        form.add(
2209 2215
            RadiobuttonsWidget,
2210 2216
            'format',
2211
            options=[('ods', _('OpenDocument (.ods)'), 'ods'), ('csv', _('Text (.csv)'), 'csv')],
2217
            options=formats,
2212 2218
            value='ods',
2213 2219
            required=True,
2214 2220
            extra_css_class='widget-inline-radio',
2221
            attrs={'data-dynamic-display-parent': 'true'},
2215 2222
        )
2216 2223
        form.add(
2217 2224
            CheckboxWidget,
2218 2225
            'include_header_line',
2219 2226
            title=_('Include header line'),
2220 2227
            value=True,
2228
            attrs={
2229
                'data-dynamic-display-child-of': 'format',
2230
                'data-dynamic-display-value-in': 'csv|ods',
2231
            },
2221 2232
        )
2222 2233
        form.add_submit('submit', _('Export'))
2223 2234
        form.add_submit('cancel', _('Cancel'))
2224 2235

  
2225 2236
        if not form.is_submitted() or form.has_errors():
2226 2237
            r = TemplateIO(html=True)
2227
            r += htmltext('<h2>%s</h2>') % _('Spreadsheet Options')
2238
            r += htmltext('<h2>%s</h2>') % _('Export Options')
2228 2239
            r += form.render()
2229 2240
            return r.getvalue()
2230 2241

  
......
2233 2244
        file_format = form.get_widget('format').parse()
2234 2245
        if file_format == 'csv':
2235 2246
            return self.csv()
2247
        elif file_format == 'json':
2248
            return self.export_json_file()
2236 2249
        else:
2237 2250
            return self.ods()
2238 2251

  
......
2335 2348
            response.set_header('content-disposition', 'attachment; filename=%s.ods' % self.formdef.url_name)
2336 2349
            return job.file_content
2337 2350

  
2351
    def export_json_file(self):
2352
        fields = self.get_fields_from_query()
2353
        selected_filter = self.get_filter_from_query()
2354
        selected_filter_operator = self.get_filter_operator_from_query()
2355
        user = get_request().user
2356
        query = get_request().form.get('q')
2357
        criterias = self.get_criterias_from_query()
2358
        order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None))
2359

  
2360
        job = JsonFileExportAfterJob(
2361
            self.formdef,
2362
            fields=fields,
2363
            selected_filter=selected_filter,
2364
            selected_filter_operator=selected_filter_operator,
2365
            user_id=user.id,
2366
            query=query,
2367
            criterias=criterias,
2368
            order_by=order_by,
2369
        )
2370
        job = get_response().add_after_job(job)
2371
        job.store()
2372
        return redirect(job.get_processing_url())
2373

  
2338 2374
    def json(self):
2339 2375
        self.view_type = 'json'
2340 2376
        anonymise = get_request().has_anonymised_data_api_restriction()
......
2373 2409
            if view_digest_key in (self.formdef.digest_templates or {}):
2374 2410
                digest_key = view_digest_key
2375 2411
        if get_request().form.get('full') == 'on':
2376
            output = []
2377
            prefetched_users = None
2378
            prefetched_roles = None
2379
            prefetched_users = {
2380
                str(x.id): x
2381
                for x in get_publisher().user_class.get_ids(
2382
                    [x.user_id for x in items if x.user_id], ignore_errors=True
2383
                )
2384
                if x is not None
2385
            }
2386
            role_ids = set((self.formdef.workflow_roles or {}).values())
2387
            for filled in items:
2388
                if filled.workflow_roles:
2389
                    for value in filled.workflow_roles.values():
2390
                        if not isinstance(value, list):
2391
                            value = [value]
2392
                        role_ids |= set(value)
2393
            prefetched_roles = {
2394
                str(x.id): x
2395
                for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True)
2396
                if x is not None
2397
            }
2398
            for filled in items:
2399
                data = filled.get_json_export_dict(
2400
                    include_files=False,
2401
                    anonymise=anonymise,
2402
                    user=user,
2403
                    digest_key=digest_key,
2404
                    prefetched_users=prefetched_users,
2405
                    prefetched_roles=prefetched_roles,
2406
                )
2407
                data.pop('digests')
2408
                data['digest'] = (filled.digests or {}).get(digest_key)
2409
                output.append(data)
2412
            output = JsonFileExportAfterJob(self.formdef).create_json_export(
2413
                items,
2414
                user=user,
2415
                anonymise=anonymise,
2416
                digest_key=digest_key,
2417
                include_evolution=True,
2418
                include_files=False,
2419
                include_roles=True,
2420
            )
2410 2421
        else:
2411 2422
            output = [
2412 2423
                {
......
4187 4198
        self.file_content = output.getvalue()
4188 4199
        self.content_type = 'application/vnd.oasis.opendocument.spreadsheet'
4189 4200
        self.store()
4201

  
4202

  
4203
class JsonFileExportAfterJob(CsvExportAfterJob):
4204
    label = _('Exporting to JSON file')
4205

  
4206
    def __init__(self, formdef, **kwargs):
4207
        super().__init__(formdef=formdef, **kwargs)
4208
        self.file_name = '%s.json' % formdef.url_name
4209

  
4210
    def create_json_export(
4211
        self, items, user, anonymise, digest_key, include_evolution, include_files, include_roles
4212
    ):
4213
        formdef = self.kwargs['formdef_class'].get(self.kwargs['formdef_id'])
4214
        prefetched_users = None
4215
        prefetched_roles = None
4216
        prefetched_users = {
4217
            str(x.id): x
4218
            for x in get_publisher().user_class.get_ids(
4219
                [x.user_id for x in items if x.user_id], ignore_errors=True
4220
            )
4221
            if x is not None
4222
        }
4223
        if include_roles:
4224
            role_ids = set((formdef.workflow_roles or {}).values())
4225
            for formdata in items:
4226
                if formdata.workflow_roles:
4227
                    for value in formdata.workflow_roles.values():
4228
                        if not isinstance(value, list):
4229
                            value = [value]
4230
                        role_ids |= set(value)
4231
            prefetched_roles = {
4232
                str(x.id): x
4233
                for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True)
4234
                if x is not None
4235
            }
4236
        output = []
4237
        for formdata in items:
4238
            data = formdata.get_json_export_dict(
4239
                anonymise=anonymise,
4240
                user=user,
4241
                digest_key=digest_key,
4242
                prefetched_users=prefetched_users,
4243
                prefetched_roles=prefetched_roles,
4244
                include_evolution=include_evolution,
4245
                include_files=include_files,
4246
                include_roles=include_roles,
4247
            )
4248
            data.pop('digests')
4249
            if digest_key:
4250
                data['digest'] = (formdata.digests or {}).get(digest_key)
4251
            output.append(data)
4252
            self.increment_count()
4253
        return output
4254

  
4255
    def create_export(self, formdef, fields, items, total_count):
4256
        self.file_content = json.dumps(
4257
            {
4258
                'data': self.create_json_export(
4259
                    items,
4260
                    user=None,
4261
                    anonymise=False,
4262
                    digest_key=None,
4263
                    include_evolution=False,
4264
                    include_files=True,
4265
                    include_roles=False,
4266
                )
4267
            },
4268
            indent=2,
4269
            cls=misc.JSONEncoder,
4270
        )
4271
        self.content_type = 'application/json'
4272
        self.store()
wcs/formdata.py
1306 1306
        digest_key='default',
1307 1307
        prefetched_users=None,
1308 1308
        prefetched_roles=None,
1309
        include_evolution=True,
1310
        include_roles=True,
1309 1311
    ):
1310 1312
        data = {}
1311 1313
        data['id'] = str(self.id)
......
1351 1353
                anonymise=anonymise,
1352 1354
            )
1353 1355

  
1354
        # add a roles dictionary, with workflow functions and two special
1355
        # entries for concerned/actions roles.
1356
        data['roles'] = {}
1357
        workflow_roles = {}
1358
        if self.formdef.workflow_roles:
1359
            workflow_roles.update(self.formdef.workflow_roles)
1360
        if self.workflow_roles:
1361
            workflow_roles.update(self.workflow_roles)
1362
        for workflow_role in workflow_roles:
1363
            value = workflow_roles.get(workflow_role)
1364
            if not isinstance(value, list):
1365
                value = [value]
1366
            data['roles'][workflow_role] = value
1367
        data['roles']['concerned'] = self.get_concerned_roles()
1368
        data['roles']['actions'] = self.get_actions_roles()
1369

  
1370
        for role_key in data['roles']:
1371
            # exclude special _submitter value
1372
            role_list = [x for x in data['roles'][role_key] if x != '_submitter']
1373
            # get role objects
1374
            if prefetched_roles is not None:
1375
                role_list = [prefetched_roles.get(str(x)) for x in role_list]
1376
            else:
1377
                role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list]
1378
            # export as json dicts
1379
            role_list = [x.get_json_export_dict() for x in role_list if x is not None]
1380
            data['roles'][role_key] = role_list
1356
        if include_roles:
1357
            # add a roles dictionary, with workflow functions and two special
1358
            # entries for concerned/actions roles.
1359
            data['roles'] = {}
1360
            workflow_roles = {}
1361
            if self.formdef.workflow_roles:
1362
                workflow_roles.update(self.formdef.workflow_roles)
1363
            if self.workflow_roles:
1364
                workflow_roles.update(self.workflow_roles)
1365
            for workflow_role in workflow_roles:
1366
                value = workflow_roles.get(workflow_role)
1367
                if not isinstance(value, list):
1368
                    value = [value]
1369
                data['roles'][workflow_role] = value
1370
            data['roles']['concerned'] = self.get_concerned_roles()
1371
            data['roles']['actions'] = self.get_actions_roles()
1372

  
1373
            for role_key in data['roles']:
1374
                # exclude special _submitter value
1375
                role_list = [x for x in data['roles'][role_key] if x != '_submitter']
1376
                # get role objects
1377
                if prefetched_roles is not None:
1378
                    role_list = [prefetched_roles.get(str(x)) for x in role_list]
1379
                else:
1380
                    role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list]
1381
                # export as json dicts
1382
                role_list = [x.get_json_export_dict() for x in role_list if x is not None]
1383
                data['roles'][role_key] = role_list
1381 1384

  
1382 1385
        data['submission'] = {
1383 1386
            'backoffice': self.backoffice_submission,
......
1391 1394
                'api_url': parent.get_api_url(),
1392 1395
            }
1393 1396

  
1394
        if self.evolution:
1397
        if self.evolution and include_evolution:
1395 1398
            evolution = data['evolution'] = []
1396 1399
            for evo in self.evolution:
1397 1400
                evolution.append(
1398
-