From f55fc91887bc3b485ba35eb9beba81cf05cf49e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Thu, 15 Oct 2020 15:59:17 +0200
Subject: [PATCH 2/4] misc: split backoffice pages tests

---
 tests/backoffice_pages/test_all.py    | 303 ---------------------
 tests/backoffice_pages/test_export.py | 363 ++++++++++++++++++++++++++
 2 files changed, 363 insertions(+), 303 deletions(-)
 create mode 100644 tests/backoffice_pages/test_export.py

diff --git a/tests/backoffice_pages/test_all.py b/tests/backoffice_pages/test_all.py
index 14015704..e28b5988 100644
--- a/tests/backoffice_pages/test_all.py
+++ b/tests/backoffice_pages/test_all.py
@@ -5,7 +5,6 @@ import os
 import re
 import time
 import random
-import xml.etree.ElementTree as ET
 import zipfile
 
 import mock
@@ -18,13 +17,11 @@ except ImportError:
 
 from django.utils.six import StringIO, BytesIO
 from django.utils.six.moves.urllib import parse as urllib
-from django.utils.six.moves.urllib import parse as urlparse
 
 from webtest import Upload
 
 from quixote import get_publisher
 from quixote.http_request import Upload as QuixoteUpload
-from wcs.qommon import ods
 from wcs.api_utils import sign_url
 from wcs.blocks import BlockDef
 from wcs.qommon.form import PicklableUpload
@@ -1363,306 +1360,6 @@ def test_backoffice_submission_agent_filter(pub):
     assert resp.text.count('>userB<') == 0
 
 
-def test_backoffice_csv(pub):
-    create_superuser(pub)
-    create_environment(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export as CSV File')
-    assert resp.headers['content-type'].startswith('text/')
-    assert len(resp.text.splitlines()) == 18 # 17 + header line
-    assert len(resp.text.splitlines()[0].split(',')) == 7
-
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields[-1].display_locations = ['validation', 'summary', 'listings']
-    formdef.store()
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export as CSV File')
-    assert len(resp.text.splitlines()[0].split(',')) == 9
-
-    # check item fields with datasources get two columns (id & text)
-    assert resp.text.splitlines()[0].split(',')[6] == '3rd field'
-    assert resp.text.splitlines()[0].split(',')[7] == '' # 3rd field, continue
-    assert resp.text.splitlines()[1].split(',')[6] == 'A'
-    assert resp.text.splitlines()[1].split(',')[7] == 'aa'
-
-    resp = app.get('/backoffice/management/form-title/')
-    resp.forms['listing-settings']['filter'] = 'all'
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 51
-
-    # test status filter
-    resp.forms['listing-settings']['filter'] = 'pending'
-    resp.forms['listing-settings']['filter-2'].checked = True
-    resp = resp.forms['listing-settings'].submit()
-    resp.forms['listing-settings']['filter-2-value'] = 'baz'
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 9
-
-    # test criteria filters
-    resp.forms['listing-settings']['filter-start'].checked = True
-    resp = resp.forms['listing-settings'].submit()
-    resp.forms['listing-settings']['filter-start-value'] = datetime.datetime(2015, 2, 1).strftime('%Y-%m-%d')
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 1
-
-    resp.forms['listing-settings']['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
-    resp = resp.forms['listing-settings'].submit()
-    resp.forms['listing-settings']['filter-2-value'] = 'baz'
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 9
-    assert 'Created' in resp_csv.text.splitlines()[0]
-
-    # test column selection
-    resp.forms['listing-settings']['time'].checked = False
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert 'Created' not in resp_csv.text.splitlines()[0]
-
-
-def test_backoffice_export_long_listings(pub):
-    create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    for i in range(100):
-        formdata = formdef.data_class()()
-        formdata.just_created()
-        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
-        formdata.data = {'1': 'BAZ BAZ %d' % i}
-        formdata.jump_status('new')
-        formdata.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export as CSV File')
-    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
-    resp = resp.follow()
-    assert 'completed' in resp.text
-    resp = resp.click('Download Export')
-    resp_lines = resp.text.splitlines()
-    assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,2nd field,Status'
-    assert len(resp_lines) == 118
-    assert resp_lines[1].split(',')[1].startswith(
-            time.strftime('%Y-%m-%d', formdata.receipt_time))
-    assert resp_lines[1].split(',')[2].startswith(
-            time.strftime('%Y-%m-%d', formdata.last_update_time))
-
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export a Spreadsheet')
-    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
-    job_id = urlparse.parse_qs(urlparse.urlparse(resp.location).query)['job'][0]
-    resp = resp.follow()
-    assert 'completed' in resp.text
-    resp = resp.click('Download Export')
-    assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'
-
-    # check afterjob ajax call
-    status_resp = app.get('/afterjobs/' + job_id)
-    assert status_resp.text == 'completed|completed'
-
-    # check error handling
-    app.get('/afterjobs/whatever', status=404)
-
-
-def test_backoffice_csv_export_channel(pub):
-    if not pub.site_options.has_section('variables'):
-        pub.site_options.add_section('variables')
-    pub.site_options.set('variables', 'welco_url', 'xxx')
-    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
-    pub.site_options.write(fd)
-    fd.close()
-
-    create_superuser(pub)
-    create_environment(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp_csv = resp.click('Export as CSV File')
-    assert 'Channel' not in resp_csv.text.splitlines()[0]
-
-    # add submission channel column
-    resp.forms['listing-settings']['submission_channel'].checked = True
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert resp_csv.text.splitlines()[0].split(',')[-1] == 'Channel'
-    assert resp_csv.text.splitlines()[1].split(',')[-1] == 'Web'
-
-
-def test_backoffice_csv_export_anonymised(pub):
-    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
-    pub.site_options.write(fd)
-    fd.close()
-
-    create_superuser(pub)
-    create_environment(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp_csv = resp.click('Export as CSV File')
-    assert resp_csv.text.splitlines()[0].split(',')[-1] != 'Anonymised'
-
-    # add anonymised column
-    resp.forms['listing-settings']['anonymised'].checked = True
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert resp_csv.text.splitlines()[0].split(',')[-1] == 'Anonymised'
-    assert resp_csv.text.splitlines()[1].split(',')[-1] == 'No'
-
-
-def test_backoffice_csv_export_block(pub):
-    create_superuser(pub)
-    create_environment(pub)
-
-    block = BlockDef()
-    block.name = 'foobar'
-    block.fields = [
-        fields.StringField(id='123', required=True, label='Test', type='string', varname='foo'),
-        fields.StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
-    ]
-    block.digest_template = 'X{{foobar_var_foo}}Y'
-    block.store()
-
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields = []
-    formdef.store()  # make sure sql columns are removed
-
-    formdef.fields = [
-        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
-    ]
-    formdef.store()
-
-    formdef.data_class().wipe()
-    formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.data['1'] = {
-        'data': [
-            {'123': 'foo', '234': 'bar'},
-            {'123': 'foo2', '234': 'bar2'},
-        ],
-        'schema': {'123': 'string', '234': 'string'},
-    }
-    formdata.just_created()
-    formdata.jump_status('new')
-    formdata.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp_csv = resp.click('Export as CSV File')
-    resp.forms['listing-settings']['1'].checked = True
-    resp = resp.forms['listing-settings'].submit()
-    resp_csv = resp.click('Export as CSV File')
-    assert resp_csv.text.splitlines()[0].split(',')[-3:] == ['test - 1', 'test - 2', 'test - 3']
-    assert resp_csv.text.splitlines()[1].split(',')[-3:] == ['XfooY', 'Xfoo2Y', '']
-
-
-def test_backoffice_ods(pub):
-    create_superuser(pub)
-    create_environment(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export a Spreadsheet')
-    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
-    assert 'filename=form-title.ods' in resp.headers['content-disposition']
-    assert resp.body[:2] == b'PK' # ods has a zip container
-
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(fields.FileField(id='4', label='file field', type='file',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.DateField(id='5', label='date field', type='date',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='6', label='number field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='7', label='phone field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.DateField(id='8', label='very old field', type='date',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='9', label='string field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.store()
-
-    formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
-    formdata.data['4'] = PicklableUpload('/foo/bar', content_type='text/plain')
-    formdata.data['4'].receive([b'hello world'])
-    formdata.data['5'] = time.strptime('2015-05-12', '%Y-%m-%d')
-    formdata.data['6'] = '12345'
-    formdata.data['7'] = '0102030405'
-    formdata.data['8'] = time.strptime('1871-03-18', '%Y-%m-%d')
-    formdata.data['9'] = 'plop\npl\x1dop'  # with control characters
-    formdata.store()
-
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Export a Spreadsheet')
-    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
-    assert 'filename=form-title.ods' in resp.headers['content-disposition']
-    assert resp.body[:2] == b'PK' # ods has a zip container
-
-    zipf = zipfile.ZipFile(BytesIO(resp.body))
-    ods_sheet = ET.parse(zipf.open('content.xml'))
-    # check the ods contains a link to the document
-    elem = ods_sheet.findall('.//{%s}a' % ods.NS['text'])[0]
-    assert elem.attrib['{%s}href' % ods.NS['xlink']] == 'http://example.net/backoffice/management/form-title/%s/files/4/bar' % formdata.id
-    resp = app.get(elem.attrib['{%s}href' % ods.NS['xlink']])
-    assert resp.text == 'hello world'
-
-    all_texts = [x.text for x in ods_sheet.findall('.//{%s}table-row//{%s}p' % (ods.NS['table'], ods.NS['text']))]
-    created_column = all_texts.index('Created')
-    date_column = all_texts.index('date field')
-    number_column = all_texts.index('number field')
-    phone_column = all_texts.index('phone field')
-    old_column = all_texts.index('very old field')
-    string_column = all_texts.index('string field')
-
-    for row in ods_sheet.findall('.//{%s}table-row' % ods.NS['table']):
-        if row.findall('.//{%s}table-cell/{%s}p' % (
-                ods.NS['table'], ods.NS['text']))[0].text == formdata.get_display_id():
-            break
-    else:
-        assert False, 'failed to find data row'
-
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
-            '{%s}value-type' % ods.NS['office']] == 'date'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
-            '{%s}style-name' % ods.NS['table']] == 'DateTime'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
-            '{%s}value-type' % ods.NS['office']] == 'date'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
-            '{%s}style-name' % ods.NS['table']] == 'Date'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
-            '{%s}value-type' % ods.NS['office']] == 'float'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
-            '{%s}value' % ods.NS['office']] == '12345'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[phone_column].attrib[
-            '{%s}value-type' % ods.NS['office']] == 'string'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
-            '{%s}value-type' % ods.NS['office']] == 'date'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
-            '{%s}date-value' % ods.NS['office']] == '1871-03-18'
-    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[
-            string_column].find('{%s}p' % ods.NS['text']).text == 'plop\nplop'
-
-
-@pytest.mark.skipif('xlwt is None')
-def test_backoffice_xls(pub):
-    create_superuser(pub)
-    create_environment(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/management/form-title/')
-    assert not 'Excel Export' in resp.text
-
-    if not pub.site_options.has_section('options'):
-        pub.site_options.add_section('options')
-        pub.site_options.set('options', 'legacy-excel-export', 'true')
-        fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
-        pub.site_options.write(fd)
-        fd.close()
-
-    resp = app.get('/backoffice/management/form-title/')
-    resp = resp.click('Excel Export')
-    assert resp.headers['content-type'].startswith('application/vnd.ms-excel')
-
-
 def test_backoffice_statistics(pub):
     create_superuser(pub)
     create_environment(pub)
diff --git a/tests/backoffice_pages/test_export.py b/tests/backoffice_pages/test_export.py
new file mode 100644
index 00000000..11d7f662
--- /dev/null
+++ b/tests/backoffice_pages/test_export.py
@@ -0,0 +1,363 @@
+# -*- coding: utf-8 -*-
+import datetime
+import os
+import time
+import xml.etree.ElementTree as ET
+import zipfile
+
+import pytest
+
+try:
+    import xlwt
+except ImportError:
+    xlwt = None
+
+from django.utils.six import BytesIO
+from django.utils.six.moves.urllib import parse as urlparse
+
+from wcs.qommon import ods
+from wcs.blocks import BlockDef
+from wcs.qommon.form import PicklableUpload
+from wcs.qommon.http_request import HTTPRequest
+from wcs.formdef import FormDef
+from wcs import fields
+
+from utilities import get_app, login, create_temporary_pub, clean_temporary_pub
+from test_all import create_superuser, create_environment
+
+
+def pytest_generate_tests(metafunc):
+    if 'pub' in metafunc.fixturenames:
+        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates'], indirect=True)
+
+
+@pytest.fixture
+def pub(request, emails):
+    pub = create_temporary_pub(
+            sql_mode=bool('sql' in request.param),
+            templates_mode=bool('templates' in request.param)
+            )
+
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    pub.set_app_dir(req)
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
+    fd.write('''
+[api-secrets]
+coucou = 1234
+''')
+    fd.close()
+
+    return pub
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+def test_backoffice_csv(pub):
+    create_superuser(pub)
+    create_environment(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export as CSV File')
+    assert resp.headers['content-type'].startswith('text/')
+    assert len(resp.text.splitlines()) == 18  # 17 + header line
+    assert len(resp.text.splitlines()[0].split(',')) == 7
+
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.fields[-1].display_locations = ['validation', 'summary', 'listings']
+    formdef.store()
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export as CSV File')
+    assert len(resp.text.splitlines()[0].split(',')) == 9
+
+    # check item fields with datasources get two columns (id & text)
+    assert resp.text.splitlines()[0].split(',')[6] == '3rd field'
+    assert resp.text.splitlines()[0].split(',')[7] == ''  # 3rd field, continue
+    assert resp.text.splitlines()[1].split(',')[6] == 'A'
+    assert resp.text.splitlines()[1].split(',')[7] == 'aa'
+
+    resp = app.get('/backoffice/management/form-title/')
+    resp.forms['listing-settings']['filter'] = 'all'
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert len(resp_csv.text.splitlines()) == 51
+
+    # test status filter
+    resp.forms['listing-settings']['filter'] = 'pending'
+    resp.forms['listing-settings']['filter-2'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp.forms['listing-settings']['filter-2-value'] = 'baz'
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert len(resp_csv.text.splitlines()) == 9
+
+    # test criteria filters
+    resp.forms['listing-settings']['filter-start'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp.forms['listing-settings']['filter-start-value'] = datetime.datetime(2015, 2, 1).strftime('%Y-%m-%d')
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert len(resp_csv.text.splitlines()) == 1
+
+    resp.forms['listing-settings']['filter-start-value'] = datetime.datetime(2014, 2, 1).strftime('%Y-%m-%d')
+    resp = resp.forms['listing-settings'].submit()
+    resp.forms['listing-settings']['filter-2-value'] = 'baz'
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert len(resp_csv.text.splitlines()) == 9
+    assert 'Created' in resp_csv.text.splitlines()[0]
+
+    # test column selection
+    resp.forms['listing-settings']['time'].checked = False
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert 'Created' not in resp_csv.text.splitlines()[0]
+
+
+def test_backoffice_export_long_listings(pub):
+    create_superuser(pub)
+    create_environment(pub)
+    formdef = FormDef.get_by_urlname('form-title')
+    for i in range(100):
+        formdata = formdef.data_class()()
+        formdata.just_created()
+        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+        formdata.data = {'1': 'BAZ BAZ %d' % i}
+        formdata.jump_status('new')
+        formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export as CSV File')
+    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
+    resp = resp.follow()
+    assert 'completed' in resp.text
+    resp = resp.click('Download Export')
+    resp_lines = resp.text.splitlines()
+    assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,2nd field,Status'
+    assert len(resp_lines) == 118
+    assert resp_lines[1].split(',')[1].startswith(
+            time.strftime('%Y-%m-%d', formdata.receipt_time))
+    assert resp_lines[1].split(',')[2].startswith(
+            time.strftime('%Y-%m-%d', formdata.last_update_time))
+
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export a Spreadsheet')
+    assert resp.location.startswith('http://example.net/backoffice/management/form-title/export?job=')
+    job_id = urlparse.parse_qs(urlparse.urlparse(resp.location).query)['job'][0]
+    resp = resp.follow()
+    assert 'completed' in resp.text
+    resp = resp.click('Download Export')
+    assert resp.content_type == 'application/vnd.oasis.opendocument.spreadsheet'
+
+    # check afterjob ajax call
+    status_resp = app.get('/afterjobs/' + job_id)
+    assert status_resp.text == 'completed|completed'
+
+    # check error handling
+    app.get('/afterjobs/whatever', status=404)
+
+
+def test_backoffice_csv_export_channel(pub):
+    if not pub.site_options.has_section('variables'):
+        pub.site_options.add_section('variables')
+    pub.site_options.set('variables', 'welco_url', 'xxx')
+    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
+    pub.site_options.write(fd)
+    fd.close()
+
+    create_superuser(pub)
+    create_environment(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp_csv = resp.click('Export as CSV File')
+    assert 'Channel' not in resp_csv.text.splitlines()[0]
+
+    # add submission channel column
+    resp.forms['listing-settings']['submission_channel'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert resp_csv.text.splitlines()[0].split(',')[-1] == 'Channel'
+    assert resp_csv.text.splitlines()[1].split(',')[-1] == 'Web'
+
+
+def test_backoffice_csv_export_anonymised(pub):
+    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
+    pub.site_options.write(fd)
+    fd.close()
+
+    create_superuser(pub)
+    create_environment(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp_csv = resp.click('Export as CSV File')
+    assert resp_csv.text.splitlines()[0].split(',')[-1] != 'Anonymised'
+
+    # add anonymised column
+    resp.forms['listing-settings']['anonymised'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert resp_csv.text.splitlines()[0].split(',')[-1] == 'Anonymised'
+    assert resp_csv.text.splitlines()[1].split(',')[-1] == 'No'
+
+
+def test_backoffice_csv_export_block(pub):
+    create_superuser(pub)
+    create_environment(pub)
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.StringField(id='123', required=True, label='Test', type='string', varname='foo'),
+        fields.StringField(id='234', required=True, label='Test2', type='string', varname='bar'),
+    ]
+    block.digest_template = 'X{{foobar_var_foo}}Y'
+    block.store()
+
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.fields = []
+    formdef.store()  # make sure sql columns are removed
+
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
+    ]
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {}
+    formdata.data['1'] = {
+        'data': [
+            {'123': 'foo', '234': 'bar'},
+            {'123': 'foo2', '234': 'bar2'},
+        ],
+        'schema': {'123': 'string', '234': 'string'},
+    }
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp_csv = resp.click('Export as CSV File')
+    resp.forms['listing-settings']['1'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+    resp_csv = resp.click('Export as CSV File')
+    assert resp_csv.text.splitlines()[0].split(',')[-3:] == ['test - 1', 'test - 2', 'test - 3']
+    assert resp_csv.text.splitlines()[1].split(',')[-3:] == ['XfooY', 'Xfoo2Y', '']
+
+
+def test_backoffice_ods(pub):
+    create_superuser(pub)
+    create_environment(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export a Spreadsheet')
+    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
+    assert 'filename=form-title.ods' in resp.headers['content-disposition']
+    assert resp.body[:2] == b'PK'  # ods has a zip container
+
+    formdef = FormDef.get_by_urlname('form-title')
+    formdef.fields.append(fields.FileField(
+        id='4', label='file field', type='file',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.DateField(
+        id='5', label='date field', type='date',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.StringField(
+        id='6', label='number field', type='string',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.StringField(
+        id='7', label='phone field', type='string',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.DateField(
+        id='8', label='very old field', type='date',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.fields.append(fields.StringField(
+        id='9', label='string field', type='string',
+        display_locations=['validation', 'summary', 'listings']))
+    formdef.store()
+
+    formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
+    formdata.data['4'] = PicklableUpload('/foo/bar', content_type='text/plain')
+    formdata.data['4'].receive([b'hello world'])
+    formdata.data['5'] = time.strptime('2015-05-12', '%Y-%m-%d')
+    formdata.data['6'] = '12345'
+    formdata.data['7'] = '0102030405'
+    formdata.data['8'] = time.strptime('1871-03-18', '%Y-%m-%d')
+    formdata.data['9'] = 'plop\npl\x1dop'  # with control characters
+    formdata.store()
+
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Export a Spreadsheet')
+    assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
+    assert 'filename=form-title.ods' in resp.headers['content-disposition']
+    assert resp.body[:2] == b'PK'  # ods has a zip container
+
+    zipf = zipfile.ZipFile(BytesIO(resp.body))
+    ods_sheet = ET.parse(zipf.open('content.xml'))
+    # check the ods contains a link to the document
+    elem = ods_sheet.findall('.//{%s}a' % ods.NS['text'])[0]
+    assert elem.attrib['{%s}href' % ods.NS['xlink']] == 'http://example.net/backoffice/management/form-title/%s/files/4/bar' % formdata.id
+    resp = app.get(elem.attrib['{%s}href' % ods.NS['xlink']])
+    assert resp.text == 'hello world'
+
+    all_texts = [x.text for x in ods_sheet.findall('.//{%s}table-row//{%s}p' % (ods.NS['table'], ods.NS['text']))]
+    created_column = all_texts.index('Created')
+    date_column = all_texts.index('date field')
+    number_column = all_texts.index('number field')
+    phone_column = all_texts.index('phone field')
+    old_column = all_texts.index('very old field')
+    string_column = all_texts.index('string field')
+
+    for row in ods_sheet.findall('.//{%s}table-row' % ods.NS['table']):
+        if row.findall('.//{%s}table-cell/{%s}p' % (
+                ods.NS['table'], ods.NS['text']))[0].text == formdata.get_display_id():
+            break
+    else:
+        assert False, 'failed to find data row'
+
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
+            '{%s}value-type' % ods.NS['office']] == 'date'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[created_column].attrib[
+            '{%s}style-name' % ods.NS['table']] == 'DateTime'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
+            '{%s}value-type' % ods.NS['office']] == 'date'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[date_column].attrib[
+            '{%s}style-name' % ods.NS['table']] == 'Date'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
+            '{%s}value-type' % ods.NS['office']] == 'float'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[number_column].attrib[
+            '{%s}value' % ods.NS['office']] == '12345'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[phone_column].attrib[
+            '{%s}value-type' % ods.NS['office']] == 'string'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
+            '{%s}value-type' % ods.NS['office']] == 'date'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[old_column].attrib[
+            '{%s}date-value' % ods.NS['office']] == '1871-03-18'
+    assert row.findall('.//{%s}table-cell' % ods.NS['table'])[
+            string_column].find('{%s}p' % ods.NS['text']).text == 'plop\nplop'
+
+
+@pytest.mark.skipif('xlwt is None')
+def test_backoffice_xls(pub):
+    create_superuser(pub)
+    create_environment(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/management/form-title/')
+    assert 'Excel Export' not in resp.text
+
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+        pub.site_options.set('options', 'legacy-excel-export', 'true')
+        fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
+        pub.site_options.write(fd)
+        fd.close()
+
+    resp = app.get('/backoffice/management/form-title/')
+    resp = resp.click('Excel Export')
+    assert resp.headers['content-type'].startswith('application/vnd.ms-excel')
-- 
2.28.0

