0001-backoffice-add-a-json-export-format-for-cards-60303.patch
tests/backoffice_pages/test_export.py | ||
---|---|---|
1 | 1 |
import datetime |
2 | 2 |
import io |
3 |
import json |
|
3 | 4 |
import os |
4 | 5 |
import time |
5 | 6 |
import urllib.parse |
... | ... | |
10 | 11 | |
11 | 12 |
from wcs import fields |
12 | 13 |
from wcs.blocks import BlockDef |
14 |
from wcs.carddef import CardDef |
|
13 | 15 |
from wcs.formdef import FormDef |
14 | 16 |
from wcs.qommon import ods |
17 |
from wcs.qommon.afterjobs import AfterJob |
|
15 | 18 |
from wcs.qommon.http_request import HTTPRequest |
16 | 19 |
from wcs.qommon.upload_storage import PicklableUpload |
17 | 20 | |
... | ... | |
867 | 870 |
resp = resp.form.submit('submit') |
868 | 871 |
assert resp.headers['content-type'].startswith('text/') |
869 | 872 |
assert len(resp.text.splitlines()) == 3 |
873 | ||
874 | ||
875 |
def test_backoffice_no_json(pub): |
|
876 |
create_superuser(pub) |
|
877 | ||
878 |
FormDef.wipe() |
|
879 |
formdef = FormDef() |
|
880 |
formdef.name = 'form title' |
|
881 |
formdef.fields = [] |
|
882 |
formdef.workflow_roles = {'_receiver': 1} |
|
883 |
formdef.store() |
|
884 | ||
885 |
app = login(get_app(pub)) |
|
886 |
resp = app.get('/backoffice/management/form-title/') |
|
887 |
resp = resp.click('Export a Spreadsheet') |
|
888 |
assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv'] |
|
889 | ||
890 | ||
891 |
def test_backoffice_cards_json(pub): |
|
892 |
create_superuser(pub) |
|
893 | ||
894 |
CardDef.wipe() |
|
895 |
carddef = CardDef() |
|
896 |
carddef.name = 'card title' |
|
897 |
carddef.fields = [ |
|
898 |
fields.StringField(id='1', label='1st field', type='string'), |
|
899 |
] |
|
900 |
carddef.workflow_roles = {'_receiver': 1} |
|
901 |
carddef.store() |
|
902 |
carddef.data_class().wipe() |
|
903 | ||
904 |
for i in range(10): |
|
905 |
carddata = carddef.data_class()() |
|
906 |
carddata.data = {'1': 'foo %s' % i} |
|
907 |
carddata.just_created() |
|
908 |
carddata.store() |
|
909 | ||
910 |
app = login(get_app(pub)) |
|
911 |
resp = app.get('/backoffice/data/card-title/') |
|
912 |
resp = resp.click('Export Data') |
|
913 |
assert [x[0] for x in resp.form['format'].options] == ['ods', 'csv', 'json'] |
|
914 |
resp.form['format'] = 'json' |
|
915 |
resp = resp.form.submit('submit') |
|
916 |
parsed_url = urllib.parse.urlparse(resp.location) |
|
917 |
assert parsed_url.path == '/backoffice/processing' |
|
918 |
job_id = urllib.parse.parse_qs(urllib.parse.urlparse(resp.location).query)['job'][0] |
|
919 |
job = AfterJob.get(job_id) |
|
920 |
assert job.completion_time |
|
921 |
json_export = json.loads(job.file_content) |
|
922 |
assert len(json_export['data']) == 10 |
wcs/backoffice/data_management.py | ||
---|---|---|
121 | 121 |
] |
122 | 122 |
admin_permission = 'cards' |
123 | 123 |
formdef_class = CardDef |
124 |
export_data_label = _('Export Data') |
|
124 | 125 |
search_label = _('Search in card content') |
125 | 126 |
formdef_view_label = _('View Card') |
127 |
has_json_export_support = True |
|
126 | 128 | |
127 | 129 |
@property |
128 | 130 |
def add(self): |
wcs/backoffice/management.py | ||
---|---|---|
763 | 763 |
_view = None |
764 | 764 |
default_view = None |
765 | 765 |
use_default_view = False |
766 |
has_json_export_support = False |
|
766 | 767 |
admin_permission = 'forms' |
767 | 768 |
formdef_class = FormDef |
769 |
export_data_label = _('Export a Spreadsheet') |
|
768 | 770 |
search_label = _('Search in form content') |
769 | 771 |
formdef_view_label = _('View Form') |
770 | 772 |
WCS_SYNC_EXPORT_LIMIT = 100 # Arbitrary threshold |
... | ... | |
839 | 841 |
r += htmltext( |
840 | 842 |
' <li><a rel="popup" data-base-href="export-spreadsheet" data-autoclose-dialog="true" ' |
841 | 843 |
'href="export-spreadsheet%s">%s</a></li>' |
842 |
) % ( |
|
843 |
qs, |
|
844 |
_('Export a Spreadsheet'), |
|
845 |
) |
|
844 |
) % (qs, self.export_data_label) |
|
846 | 845 |
if self.formdef.geolocations: |
847 | 846 |
r += htmltext(' <li><a data-base-href="map" href="map%s">%s</a></li>') % (qs, _('Plot on a Map')) |
848 | 847 |
if 'stats' in self._q_exports and ( |
... | ... | |
2205 | 2204 |
raise errors.AccessForbiddenError() |
2206 | 2205 |
form = Form() |
2207 | 2206 |
form.add_hidden('query_string', get_request().get_query()) |
2207 |
formats = [ |
|
2208 |
('ods', _('OpenDocument (.ods)'), 'ods'), |
|
2209 |
('csv', _('Text (.csv)'), 'csv'), |
|
2210 |
] |
|
2211 |
if self.has_json_export_support: |
|
2212 |
formats.append(('json', _('JSON'), 'json')) |
|
2213 | ||
2208 | 2214 |
form.add( |
2209 | 2215 |
RadiobuttonsWidget, |
2210 | 2216 |
'format', |
2211 |
options=[('ods', _('OpenDocument (.ods)'), 'ods'), ('csv', _('Text (.csv)'), 'csv')],
|
|
2217 |
options=formats,
|
|
2212 | 2218 |
value='ods', |
2213 | 2219 |
required=True, |
2214 | 2220 |
extra_css_class='widget-inline-radio', |
2221 |
attrs={'data-dynamic-display-parent': 'true'}, |
|
2215 | 2222 |
) |
2216 | 2223 |
form.add( |
2217 | 2224 |
CheckboxWidget, |
2218 | 2225 |
'include_header_line', |
2219 | 2226 |
title=_('Include header line'), |
2220 | 2227 |
value=True, |
2228 |
attrs={ |
|
2229 |
'data-dynamic-display-child-of': 'format', |
|
2230 |
'data-dynamic-display-value-in': 'csv|ods', |
|
2231 |
}, |
|
2221 | 2232 |
) |
2222 | 2233 |
form.add_submit('submit', _('Export')) |
2223 | 2234 |
form.add_submit('cancel', _('Cancel')) |
2224 | 2235 | |
2225 | 2236 |
if not form.is_submitted() or form.has_errors(): |
2226 | 2237 |
r = TemplateIO(html=True) |
2227 |
r += htmltext('<h2>%s</h2>') % _('Spreadsheet Options')
|
|
2238 |
r += htmltext('<h2>%s</h2>') % _('Export Options')
|
|
2228 | 2239 |
r += form.render() |
2229 | 2240 |
return r.getvalue() |
2230 | 2241 | |
... | ... | |
2233 | 2244 |
file_format = form.get_widget('format').parse() |
2234 | 2245 |
if file_format == 'csv': |
2235 | 2246 |
return self.csv() |
2247 |
elif file_format == 'json': |
|
2248 |
return self.export_json_file() |
|
2236 | 2249 |
else: |
2237 | 2250 |
return self.ods() |
2238 | 2251 | |
... | ... | |
2335 | 2348 |
response.set_header('content-disposition', 'attachment; filename=%s.ods' % self.formdef.url_name) |
2336 | 2349 |
return job.file_content |
2337 | 2350 | |
2351 |
def export_json_file(self): |
|
2352 |
fields = self.get_fields_from_query() |
|
2353 |
selected_filter = self.get_filter_from_query() |
|
2354 |
selected_filter_operator = self.get_filter_operator_from_query() |
|
2355 |
user = get_request().user |
|
2356 |
query = get_request().form.get('q') |
|
2357 |
criterias = self.get_criterias_from_query() |
|
2358 |
order_by = misc.get_order_by_or_400(get_request().form.get('order_by', None)) |
|
2359 | ||
2360 |
job = JsonFileExportAfterJob( |
|
2361 |
self.formdef, |
|
2362 |
fields=fields, |
|
2363 |
selected_filter=selected_filter, |
|
2364 |
selected_filter_operator=selected_filter_operator, |
|
2365 |
user_id=user.id, |
|
2366 |
query=query, |
|
2367 |
criterias=criterias, |
|
2368 |
order_by=order_by, |
|
2369 |
) |
|
2370 |
job = get_response().add_after_job(job) |
|
2371 |
job.store() |
|
2372 |
return redirect(job.get_processing_url()) |
|
2373 | ||
2338 | 2374 |
def json(self): |
2339 | 2375 |
self.view_type = 'json' |
2340 | 2376 |
anonymise = get_request().has_anonymised_data_api_restriction() |
... | ... | |
2373 | 2409 |
if view_digest_key in (self.formdef.digest_templates or {}): |
2374 | 2410 |
digest_key = view_digest_key |
2375 | 2411 |
if get_request().form.get('full') == 'on': |
2376 |
output = [] |
|
2377 |
prefetched_users = None |
|
2378 |
prefetched_roles = None |
|
2379 |
prefetched_users = { |
|
2380 |
str(x.id): x |
|
2381 |
for x in get_publisher().user_class.get_ids( |
|
2382 |
[x.user_id for x in items if x.user_id], ignore_errors=True |
|
2383 |
) |
|
2384 |
if x is not None |
|
2385 |
} |
|
2386 |
role_ids = set((self.formdef.workflow_roles or {}).values()) |
|
2387 |
for filled in items: |
|
2388 |
if filled.workflow_roles: |
|
2389 |
for value in filled.workflow_roles.values(): |
|
2390 |
if not isinstance(value, list): |
|
2391 |
value = [value] |
|
2392 |
role_ids |= set(value) |
|
2393 |
prefetched_roles = { |
|
2394 |
str(x.id): x |
|
2395 |
for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True) |
|
2396 |
if x is not None |
|
2397 |
} |
|
2398 |
for filled in items: |
|
2399 |
data = filled.get_json_export_dict( |
|
2400 |
include_files=False, |
|
2401 |
anonymise=anonymise, |
|
2402 |
user=user, |
|
2403 |
digest_key=digest_key, |
|
2404 |
prefetched_users=prefetched_users, |
|
2405 |
prefetched_roles=prefetched_roles, |
|
2406 |
) |
|
2407 |
data.pop('digests') |
|
2408 |
data['digest'] = (filled.digests or {}).get(digest_key) |
|
2409 |
output.append(data) |
|
2412 |
output = JsonFileExportAfterJob(self.formdef).create_json_export( |
|
2413 |
items, |
|
2414 |
user=user, |
|
2415 |
anonymise=anonymise, |
|
2416 |
digest_key=digest_key, |
|
2417 |
include_evolution=True, |
|
2418 |
include_files=False, |
|
2419 |
include_roles=True, |
|
2420 |
) |
|
2410 | 2421 |
else: |
2411 | 2422 |
output = [ |
2412 | 2423 |
{ |
... | ... | |
4187 | 4198 |
self.file_content = output.getvalue() |
4188 | 4199 |
self.content_type = 'application/vnd.oasis.opendocument.spreadsheet' |
4189 | 4200 |
self.store() |
4201 | ||
4202 | ||
4203 |
class JsonFileExportAfterJob(CsvExportAfterJob): |
|
4204 |
label = _('Exporting to JSON file') |
|
4205 | ||
4206 |
def __init__(self, formdef, **kwargs): |
|
4207 |
super().__init__(formdef=formdef, **kwargs) |
|
4208 |
self.file_name = '%s.json' % formdef.url_name |
|
4209 | ||
4210 |
def create_json_export( |
|
4211 |
self, items, user, anonymise, digest_key, include_evolution, include_files, include_roles |
|
4212 |
): |
|
4213 |
formdef = self.kwargs['formdef_class'].get(self.kwargs['formdef_id']) |
|
4214 |
prefetched_users = None |
|
4215 |
prefetched_roles = None |
|
4216 |
prefetched_users = { |
|
4217 |
str(x.id): x |
|
4218 |
for x in get_publisher().user_class.get_ids( |
|
4219 |
[x.user_id for x in items if x.user_id], ignore_errors=True |
|
4220 |
) |
|
4221 |
if x is not None |
|
4222 |
} |
|
4223 |
if include_roles: |
|
4224 |
role_ids = set((formdef.workflow_roles or {}).values()) |
|
4225 |
for formdata in items: |
|
4226 |
if formdata.workflow_roles: |
|
4227 |
for value in formdata.workflow_roles.values(): |
|
4228 |
if not isinstance(value, list): |
|
4229 |
value = [value] |
|
4230 |
role_ids |= set(value) |
|
4231 |
prefetched_roles = { |
|
4232 |
str(x.id): x |
|
4233 |
for x in get_publisher().role_class.get_ids(role_ids, ignore_errors=True) |
|
4234 |
if x is not None |
|
4235 |
} |
|
4236 |
output = [] |
|
4237 |
for formdata in items: |
|
4238 |
data = formdata.get_json_export_dict( |
|
4239 |
anonymise=anonymise, |
|
4240 |
user=user, |
|
4241 |
digest_key=digest_key, |
|
4242 |
prefetched_users=prefetched_users, |
|
4243 |
prefetched_roles=prefetched_roles, |
|
4244 |
include_evolution=include_evolution, |
|
4245 |
include_files=include_files, |
|
4246 |
include_roles=include_roles, |
|
4247 |
) |
|
4248 |
data.pop('digests') |
|
4249 |
if digest_key: |
|
4250 |
data['digest'] = (formdata.digests or {}).get(digest_key) |
|
4251 |
output.append(data) |
|
4252 |
self.increment_count() |
|
4253 |
return output |
|
4254 | ||
4255 |
def create_export(self, formdef, fields, items, total_count): |
|
4256 |
self.file_content = json.dumps( |
|
4257 |
{ |
|
4258 |
'data': self.create_json_export( |
|
4259 |
items, |
|
4260 |
user=None, |
|
4261 |
anonymise=False, |
|
4262 |
digest_key=None, |
|
4263 |
include_evolution=False, |
|
4264 |
include_files=True, |
|
4265 |
include_roles=False, |
|
4266 |
) |
|
4267 |
}, |
|
4268 |
indent=2, |
|
4269 |
cls=misc.JSONEncoder, |
|
4270 |
) |
|
4271 |
self.content_type = 'application/json' |
|
4272 |
self.store() |
wcs/formdata.py | ||
---|---|---|
1306 | 1306 |
digest_key='default', |
1307 | 1307 |
prefetched_users=None, |
1308 | 1308 |
prefetched_roles=None, |
1309 |
include_evolution=True, |
|
1310 |
include_roles=True, |
|
1309 | 1311 |
): |
1310 | 1312 |
data = {} |
1311 | 1313 |
data['id'] = str(self.id) |
... | ... | |
1351 | 1353 |
anonymise=anonymise, |
1352 | 1354 |
) |
1353 | 1355 | |
1354 |
# add a roles dictionary, with workflow functions and two special |
|
1355 |
# entries for concerned/actions roles. |
|
1356 |
data['roles'] = {} |
|
1357 |
workflow_roles = {} |
|
1358 |
if self.formdef.workflow_roles: |
|
1359 |
workflow_roles.update(self.formdef.workflow_roles) |
|
1360 |
if self.workflow_roles: |
|
1361 |
workflow_roles.update(self.workflow_roles) |
|
1362 |
for workflow_role in workflow_roles: |
|
1363 |
value = workflow_roles.get(workflow_role) |
|
1364 |
if not isinstance(value, list): |
|
1365 |
value = [value] |
|
1366 |
data['roles'][workflow_role] = value |
|
1367 |
data['roles']['concerned'] = self.get_concerned_roles() |
|
1368 |
data['roles']['actions'] = self.get_actions_roles() |
|
1369 | ||
1370 |
for role_key in data['roles']: |
|
1371 |
# exclude special _submitter value |
|
1372 |
role_list = [x for x in data['roles'][role_key] if x != '_submitter'] |
|
1373 |
# get role objects |
|
1374 |
if prefetched_roles is not None: |
|
1375 |
role_list = [prefetched_roles.get(str(x)) for x in role_list] |
|
1376 |
else: |
|
1377 |
role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list] |
|
1378 |
# export as json dicts |
|
1379 |
role_list = [x.get_json_export_dict() for x in role_list if x is not None] |
|
1380 |
data['roles'][role_key] = role_list |
|
1356 |
if include_roles: |
|
1357 |
# add a roles dictionary, with workflow functions and two special |
|
1358 |
# entries for concerned/actions roles. |
|
1359 |
data['roles'] = {} |
|
1360 |
workflow_roles = {} |
|
1361 |
if self.formdef.workflow_roles: |
|
1362 |
workflow_roles.update(self.formdef.workflow_roles) |
|
1363 |
if self.workflow_roles: |
|
1364 |
workflow_roles.update(self.workflow_roles) |
|
1365 |
for workflow_role in workflow_roles: |
|
1366 |
value = workflow_roles.get(workflow_role) |
|
1367 |
if not isinstance(value, list): |
|
1368 |
value = [value] |
|
1369 |
data['roles'][workflow_role] = value |
|
1370 |
data['roles']['concerned'] = self.get_concerned_roles() |
|
1371 |
data['roles']['actions'] = self.get_actions_roles() |
|
1372 | ||
1373 |
for role_key in data['roles']: |
|
1374 |
# exclude special _submitter value |
|
1375 |
role_list = [x for x in data['roles'][role_key] if x != '_submitter'] |
|
1376 |
# get role objects |
|
1377 |
if prefetched_roles is not None: |
|
1378 |
role_list = [prefetched_roles.get(str(x)) for x in role_list] |
|
1379 |
else: |
|
1380 |
role_list = [get_publisher().role_class.get(x, ignore_errors=True) for x in role_list] |
|
1381 |
# export as json dicts |
|
1382 |
role_list = [x.get_json_export_dict() for x in role_list if x is not None] |
|
1383 |
data['roles'][role_key] = role_list |
|
1381 | 1384 | |
1382 | 1385 |
data['submission'] = { |
1383 | 1386 |
'backoffice': self.backoffice_submission, |
... | ... | |
1391 | 1394 |
'api_url': parent.get_api_url(), |
1392 | 1395 |
} |
1393 | 1396 | |
1394 |
if self.evolution: |
|
1397 |
if self.evolution and include_evolution:
|
|
1395 | 1398 |
evolution = data['evolution'] = [] |
1396 | 1399 |
for evo in self.evolution: |
1397 | 1400 |
evolution.append( |
1398 |
- |