From 17cb16927eaaa0f33ddbd73de546203de72c44fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Thu, 15 Oct 2020 16:20:12 +0200
Subject: [PATCH 2/3] misc: speed up backoffice export tests

---
 tests/backoffice_pages/test_export.py | 226 +++++++++++++++++++-------
 wcs/backoffice/management.py          |   9 +-
 2 files changed, 171 insertions(+), 64 deletions(-)

diff --git a/tests/backoffice_pages/test_export.py b/tests/backoffice_pages/test_export.py
index 2b014e51..ab806ebf 100644
--- a/tests/backoffice_pages/test_export.py
+++ b/tests/backoffice_pages/test_export.py
@@ -23,7 +23,7 @@ from wcs.formdef import FormDef
 from wcs import fields
 
 from utilities import get_app, login, create_temporary_pub, clean_temporary_pub
-from test_all import create_superuser, create_environment
+from test_all import create_superuser
 
 
 def pytest_generate_tests(metafunc):
@@ -59,12 +59,53 @@ def teardown_module(module):
 
 def test_backoffice_csv(pub):
     create_superuser(pub)
-    create_environment(pub)
+
+    datasource = {
+        'type': 'formula',
+        'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])
+    }
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(
+            id='1', label='1st field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.ItemField(
+            id='2', label='2nd field', type='item',
+            items=['foo', 'bar', 'baz'],
+            display_locations=['validation', 'summary', 'listings']),
+        fields.ItemField(
+            id='3', label='3rd field', type='item',
+            data_source=datasource, varname='foo'),
+    ]
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    for i in range(3):
+        formdata = formdef.data_class()()
+        formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
+        formdata.data = {'1': 'FOO BAR %d' % i}
+        if i == 0:
+            formdata.data['2'] = 'foo'
+            formdata.data['2_display'] = 'foo'
+            formdata.data['3'] = 'A'
+            formdata.data['3_display'] = 'aa'
+        else:
+            formdata.data['2'] = 'baz'
+            formdata.data['2_display'] = 'baz'
+            formdata.data['3'] = 'C'
+            formdata.data['3_display'] = 'cc'
+        if i < 2:
+            formdata.jump_status('new')
+        formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Export as CSV File')
     assert resp.headers['content-type'].startswith('text/')
-    assert len(resp.text.splitlines()) == 18 # 17 + header line
+    assert len(resp.text.splitlines()) == 3  # 3 + header line
     assert len(resp.text.splitlines()[0].split(',')) == 7
 
     formdef = FormDef.get_by_urlname('form-title')
@@ -76,15 +117,15 @@ def test_backoffice_csv(pub):
 
     # check item fields with datasources get two columns (id & text)
     assert resp.text.splitlines()[0].split(',')[6] == '3rd field'
-    assert resp.text.splitlines()[0].split(',')[7] == '' # 3rd field, continue
-    assert resp.text.splitlines()[1].split(',')[6] == 'A'
-    assert resp.text.splitlines()[1].split(',')[7] == 'aa'
+    assert resp.text.splitlines()[0].split(',')[7] == ''  # 3rd field, continue
+    assert resp.text.splitlines()[1].split(',')[6] == 'C'
+    assert resp.text.splitlines()[1].split(',')[7] == 'cc'
 
     resp = app.get('/backoffice/management/form-title/')
     resp.forms['listing-settings']['filter'] = 'all'
     resp = resp.forms['listing-settings'].submit()
     resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 51
+    assert len(resp_csv.text.splitlines()) == 4
 
     # test status filter
     resp.forms['listing-settings']['filter'] = 'pending'
@@ -93,7 +134,7 @@ def test_backoffice_csv(pub):
     resp.forms['listing-settings']['filter-2-value'] = 'baz'
     resp = resp.forms['listing-settings'].submit()
     resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 9
+    assert len(resp_csv.text.splitlines()) == 2
 
     # test criteria filters
     resp.forms['listing-settings']['filter-start'].checked = True
@@ -108,7 +149,7 @@ def test_backoffice_csv(pub):
     resp.forms['listing-settings']['filter-2-value'] = 'baz'
     resp = resp.forms['listing-settings'].submit()
     resp_csv = resp.click('Export as CSV File')
-    assert len(resp_csv.text.splitlines()) == 9
+    assert len(resp_csv.text.splitlines()) == 2
     assert 'Created' in resp_csv.text.splitlines()[0]
 
     # test column selection
@@ -118,13 +159,31 @@ def test_backoffice_csv(pub):
     assert 'Created' not in resp_csv.text.splitlines()[0]
 
 
-def test_backoffice_export_long_listings(pub):
+@pytest.fixture
+def threshold():
+    from wcs.backoffice.management import FormPage
+    FormPage.WCS_SYNC_EXPORT_LIMIT = 1
+    yield
+    FormPage.WCS_SYNC_EXPORT_LIMIT = 100
+
+
+def test_backoffice_export_long_listings(pub, threshold):
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    for i in range(100):
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.StringField(
+            id='1', label='1st field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+    ]
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    for i in range(2):
         formdata = formdef.data_class()()
-        formdata.just_created()
         formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
         formdata.data = {'1': 'BAZ BAZ %d' % i}
         formdata.jump_status('new')
@@ -138,8 +197,8 @@ def test_backoffice_export_long_listings(pub):
     assert 'completed' in resp.text
     resp = resp.click('Download Export')
     resp_lines = resp.text.splitlines()
-    assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,2nd field,Status'
-    assert len(resp_lines) == 118
+    assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,Status'
+    assert len(resp_lines) == 3
     assert resp_lines[1].split(',')[1].startswith(
             time.strftime('%Y-%m-%d', formdata.receipt_time))
     assert resp_lines[1].split(',')[2].startswith(
@@ -171,7 +230,19 @@ def test_backoffice_csv_export_channel(pub):
     fd.close()
 
     create_superuser(pub)
-    create_environment(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     resp_csv = resp.click('Export as CSV File')
@@ -191,7 +262,19 @@ def test_backoffice_csv_export_anonymised(pub):
     fd.close()
 
     create_superuser(pub)
-    create_environment(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     resp_csv = resp.click('Export as CSV File')
@@ -207,7 +290,6 @@ def test_backoffice_csv_export_anonymised(pub):
 
 def test_backoffice_csv_export_block(pub):
     create_superuser(pub)
-    create_environment(pub)
 
     block = BlockDef()
     block.name = 'foobar'
@@ -218,24 +300,25 @@ def test_backoffice_csv_export_block(pub):
     block.digest_template = 'X{{foobar_var_foo}}Y'
     block.store()
 
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields = []
-    formdef.store()  # make sure sql columns are removed
-
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
     formdef.fields = [
         fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
     ]
+    formdef.workflow_roles = {'_receiver': 1}
     formdef.store()
 
     formdef.data_class().wipe()
     formdata = formdef.data_class()()
-    formdata.data = {}
-    formdata.data['1'] = {
-        'data': [
-            {'123': 'foo', '234': 'bar'},
-            {'123': 'foo2', '234': 'bar2'},
-        ],
-        'schema': {'123': 'string', '234': 'string'},
+    formdata.data = {
+        '1': {
+            'data': [
+                {'123': 'foo', '234': 'bar'},
+                {'123': 'foo2', '234': 'bar2'},
+            ],
+            'schema': {'123': 'string', '234': 'string'},
+        }
     }
     formdata.just_created()
     formdata.jump_status('new')
@@ -253,47 +336,64 @@ def test_backoffice_csv_export_block(pub):
 
 def test_backoffice_ods(pub):
     create_superuser(pub)
-    create_environment(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.FileField(
+            id='4', label='file field', type='file',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.DateField(
+            id='5', label='date field', type='date',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.StringField(
+            id='6', label='number field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.StringField(
+            id='7', label='phone field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.DateField(
+            id='8', label='very old field', type='date',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.StringField(
+            id='9', label='string field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+        fields.StringField(
+            id='10', label='number with comma field', type='string',
+            display_locations=['validation', 'summary', 'listings']),
+    ]
+    formdef.workflow_roles = {'_receiver': 1}
+    formdef.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Export a Spreadsheet')
     assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
     assert 'filename=form-title.ods' in resp.headers['content-disposition']
-    assert resp.body[:2] == b'PK' # ods has a zip container
+    assert resp.body[:2] == b'PK'  # ods has a zip container
 
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(fields.FileField(id='4', label='file field', type='file',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.DateField(id='5', label='date field', type='date',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='6', label='number field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='7', label='phone field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.DateField(id='8', label='very old field', type='date',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='9', label='string field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.fields.append(fields.StringField(id='10', label='number with comma field', type='string',
-        display_locations=['validation', 'summary', 'listings']))
-    formdef.store()
-
-    formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
-    formdata.data['4'] = PicklableUpload('/foo/bar', content_type='text/plain')
+    formdef.data_class().wipe()
+    formdata = formdef.data_class()()
+    formdata.data = {
+        '4': PicklableUpload('/foo/bar', content_type='text/plain'),
+        '5': time.strptime('2015-05-12', '%Y-%m-%d'),
+        '6': '12345',
+        '7': '0102030405',
+        '8': time.strptime('1871-03-18', '%Y-%m-%d'),
+        '9': 'plop\npl\x1dop',  # with control characters
+        '10': '123,45',
+    }
     formdata.data['4'].receive([b'hello world'])
-    formdata.data['5'] = time.strptime('2015-05-12', '%Y-%m-%d')
-    formdata.data['6'] = '12345'
-    formdata.data['7'] = '0102030405'
-    formdata.data['8'] = time.strptime('1871-03-18', '%Y-%m-%d')
-    formdata.data['9'] = 'plop\npl\x1dop'  # with control characters
-    formdata.data['10'] = '123,45'
+    formdata.just_created()
+    formdata.jump_status('new')
     formdata.store()
 
     resp = app.get('/backoffice/management/form-title/')
     resp = resp.click('Export a Spreadsheet')
     assert resp.headers['content-type'] == 'application/vnd.oasis.opendocument.spreadsheet'
     assert 'filename=form-title.ods' in resp.headers['content-disposition']
-    assert resp.body[:2] == b'PK' # ods has a zip container
+    assert resp.body[:2] == b'PK'  # ods has a zip container
 
     zipf = zipfile.ZipFile(BytesIO(resp.body))
     ods_sheet = ET.parse(zipf.open('content.xml'))
@@ -346,10 +446,16 @@ def test_backoffice_ods(pub):
 @pytest.mark.skipif('xlwt is None')
 def test_backoffice_xls(pub):
     create_superuser(pub)
-    create_environment(pub)
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = []
+    formdef.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert not 'Excel Export' in resp.text
+    assert 'Excel Export' not in resp.text
 
     if not pub.site_options.has_section('options'):
         pub.site_options.add_section('options')
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index 8dbb2726..c7c2cd2f 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -28,6 +28,7 @@ try:
 except ImportError:
     xlwt = None
 
+from django.conf import settings
 from django.utils import six
 from django.utils.encoding import force_text
 from django.utils.six.moves.urllib import parse as urllib
@@ -1033,6 +1034,7 @@ class FormPage(Directory):
     admin_permission = 'forms'
     formdef_class = FormDef
     search_label = N_('Search in form content')
+    WCS_SYNC_EXPORT_LIMIT = 100  # Arbitrary threshold
 
     def __init__(self, component=None, formdef=None, view=None, update_breadcrumbs=True):
         self.view_type = None
@@ -2062,7 +2064,7 @@ class FormPage(Directory):
 
         count = self.formdef.data_class().count()
         exporter = Exporter(self, self.formdef, fields, selected_filter)
-        if count > 100: # Arbitrary threshold
+        if count > self.WCS_SYNC_EXPORT_LIMIT:
             job = get_response().add_after_job(
                 str(N_('Exporting forms in CSV')),
                 exporter.export)
@@ -2172,7 +2174,7 @@ class FormPage(Directory):
 
         count = self.formdef.data_class().count()
         exporter = Exporter(self, self.formdef, fields, selected_filter)
-        if count > 100: # Arbitrary threshold
+        if count > self.WCS_SYNC_EXPORT_LIMIT:
             job = get_response().add_after_job(
                 str(N_('Exporting forms in Excel format')),
                 exporter.export)
@@ -2235,8 +2237,7 @@ class FormPage(Directory):
 
         count = self.formdef.data_class().count()
         exporter = Exporter(self, self.formdef, fields, selected_filter)
-        if count > 100 and not get_request().is_api_url():
-            # The "100" threshold is arbitrary
+        if count > self.WCS_SYNC_EXPORT_LIMIT and not get_request().is_api_url():
             job = get_response().add_after_job(
                 str(N_('Exporting forms in Open Document format')),
                 exporter.export)
-- 
2.28.0

