0003-misc-speed-up-backoffice-export-tests.patch

Lauréline Guérin, 15 octobre 2020 17:06

Voir les différences: en ligne côte à côte

Subject: [PATCH 3/4] misc: speed up backoffice export tests

 tests/backoffice_pages/test_export.py | 210 ++++++++++++++++++--------
 wcs/backoffice/management.py          |   9 +-
 2 files changed, 156 insertions(+), 63 deletions(-)

     from wcs import fields
     from utilities import get_app, login, create_temporary_pub, clean_temporary_pub
     from test_all import create_superuser, create_environment
     from test_all import create_superuser
     def pytest_generate_tests(metafunc):
-...
     def test_backoffice_csv(pub):
         create_superuser(pub)
         create_environment(pub)
         datasource = {
             'type': 'formula',
             'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])
+        }
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = [
             fields.StringField(
                 id='1', label='1st field', type='string',
                 display_locations=['validation', 'summary', 'listings']),
             fields.ItemField(
                 id='2', label='2nd field', type='item',
                 items=['foo', 'bar', 'baz'],
                 display_locations=['validation', 'summary', 'listings']),
             fields.ItemField(
                 id='3', label='3rd field', type='item',
                 data_source=datasource, varname='foo'),
+        ]
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         formdef.data_class().wipe()
         for i in range(3):
             formdata = formdef.data_class()()
             formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
             formdata.data = {'1': 'FOO BAR %d' % i}
             if i == 0:
                 formdata.data['2'] = 'foo'
                 formdata.data['2_display'] = 'foo'
                 formdata.data['3'] = 'A'
                 formdata.data['3_display'] = 'aa'
             else:
                 formdata.data['2'] = 'baz'
                 formdata.data['2_display'] = 'baz'
                 formdata.data['3'] = 'C'
                 formdata.data['3_display'] = 'cc'
             if i < 2:
                 formdata.jump_status('new')
             formdata.store()
         app = login(get_app(pub))
         resp = app.get('/backoffice/management/form-title/')
         resp = resp.click('Export as CSV File')
         assert resp.headers['content-type'].startswith('text/')
         assert len(resp.text.splitlines()) == 18  # 17 + header line
         assert len(resp.text.splitlines()) == 3  # 3 + header line
         assert len(resp.text.splitlines()[0].split(',')) == 7
         formdef = FormDef.get_by_urlname('form-title')
-...
         # check item fields with datasources get two columns (id & text)
         assert resp.text.splitlines()[0].split(',')[6] == '3rd field'
         assert resp.text.splitlines()[0].split(',')[7] == ''  # 3rd field, continue
         assert resp.text.splitlines()[1].split(',')[6] == 'A'
         assert resp.text.splitlines()[1].split(',')[7] == 'aa'
         assert resp.text.splitlines()[1].split(',')[6] == 'C'
         assert resp.text.splitlines()[1].split(',')[7] == 'cc'
         resp = app.get('/backoffice/management/form-title/')
         resp.forms['listing-settings']['filter'] = 'all'
         resp = resp.forms['listing-settings'].submit()
         resp_csv = resp.click('Export as CSV File')
         assert len(resp_csv.text.splitlines()) == 51
         assert len(resp_csv.text.splitlines()) == 4
         # test status filter
         resp.forms['listing-settings']['filter'] = 'pending'
-...
         resp.forms['listing-settings']['filter-2-value'] = 'baz'
         resp = resp.forms['listing-settings'].submit()
         resp_csv = resp.click('Export as CSV File')
         assert len(resp_csv.text.splitlines()) == 9
         assert len(resp_csv.text.splitlines()) == 2
         # test criteria filters
         resp.forms['listing-settings']['filter-start'].checked = True
-...
         resp.forms['listing-settings']['filter-2-value'] = 'baz'
         resp = resp.forms['listing-settings'].submit()
         resp_csv = resp.click('Export as CSV File')
         assert len(resp_csv.text.splitlines()) == 9
         assert len(resp_csv.text.splitlines()) == 2
         assert 'Created' in resp_csv.text.splitlines()[0]
         # test column selection
-...
         assert 'Created' not in resp_csv.text.splitlines()[0]
     def test_backoffice_export_long_listings(pub):
     def test_backoffice_export_long_listings(settings, pub):
         settings.WCS_SYNC_EXPORT_LIMIT = 1
         create_superuser(pub)
         create_environment(pub)
         formdef = FormDef.get_by_urlname('form-title')
         for i in range(100):
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = [
             fields.StringField(
                 id='1', label='1st field', type='string',
                 display_locations=['validation', 'summary', 'listings']),
+        ]
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         formdef.data_class().wipe()
         for i in range(2):
             formdata = formdef.data_class()()
             formdata.just_created()
             formdata.receipt_time = datetime.datetime(2015, 1, 1).timetuple()
             formdata.data = {'1': 'BAZ BAZ %d' % i}
             formdata.jump_status('new')
-...
         assert 'completed' in resp.text
         resp = resp.click('Download Export')
         resp_lines = resp.text.splitlines()
         assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,2nd field,Status'
         assert len(resp_lines) == 118
         assert resp_lines[0] == 'Number,Created,Last Modified,User Label,1st field,Status'
         assert len(resp_lines) == 3
         assert resp_lines[1].split(',')[1].startswith(
                 time.strftime('%Y-%m-%d', formdata.receipt_time))
         assert resp_lines[1].split(',')[2].startswith(
-...
         fd.close()
         create_superuser(pub)
         create_environment(pub)
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = []
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         formdef.data_class().wipe()
         formdata = formdef.data_class()()
         formdata.jump_status('new')
         formdata.store()
         app = login(get_app(pub))
         resp = app.get('/backoffice/management/form-title/')
         resp_csv = resp.click('Export as CSV File')
-...
         fd.close()
         create_superuser(pub)
         create_environment(pub)
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = []
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         formdef.data_class().wipe()
         formdata = formdef.data_class()()
         formdata.jump_status('new')
         formdata.store()
         app = login(get_app(pub))
         resp = app.get('/backoffice/management/form-title/')
         resp_csv = resp.click('Export as CSV File')
-...
     def test_backoffice_csv_export_block(pub):
         create_superuser(pub)
         create_environment(pub)
         block = BlockDef()
         block.name = 'foobar'
-...
         block.digest_template = 'X{{foobar_var_foo}}Y'
         block.store()
         formdef = FormDef.get_by_urlname('form-title')
         formdef.fields = []
         formdef.store()  # make sure sql columns are removed
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = [
             fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
+        ]
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         formdef.data_class().wipe()
         formdata = formdef.data_class()()
         formdata.data = {}
         formdata.data['1'] = {
             'data': [
                 {'123': 'foo', '234': 'bar'},
                 {'123': 'foo2', '234': 'bar2'},
             ],
             'schema': {'123': 'string', '234': 'string'},
         formdata.data = {
             '1': {
                 'data': [
                     {'123': 'foo', '234': 'bar'},
                     {'123': 'foo2', '234': 'bar2'},
                 ],
                 'schema': {'123': 'string', '234': 'string'},
+            }
+        }
         formdata.just_created()
         formdata.jump_status('new')
-...
     def test_backoffice_ods(pub):
         create_superuser(pub)
         create_environment(pub)
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = [
             fields.FileField(
                 id='4', label='file field', type='file',
                 display_locations=['validation', 'summary', 'listings']),
             fields.DateField(
                 id='5', label='date field', type='date',
                 display_locations=['validation', 'summary', 'listings']),
             fields.StringField(
                 id='6', label='number field', type='string',
                 display_locations=['validation', 'summary', 'listings']),
             fields.StringField(
                 id='7', label='phone field', type='string',
                 display_locations=['validation', 'summary', 'listings']),
             fields.DateField(
                 id='8', label='very old field', type='date',
                 display_locations=['validation', 'summary', 'listings']),
             fields.StringField(
                 id='9', label='string field', type='string',
                 display_locations=['validation', 'summary', 'listings']),
+        ]
         formdef.workflow_roles = {'_receiver': 1}
         formdef.store()
         app = login(get_app(pub))
         resp = app.get('/backoffice/management/form-title/')
         resp = resp.click('Export a Spreadsheet')
-...
         assert 'filename=form-title.ods' in resp.headers['content-disposition']
         assert resp.body[:2] == b'PK'  # ods has a zip container
         formdef = FormDef.get_by_urlname('form-title')
         formdef.fields.append(fields.FileField(
             id='4', label='file field', type='file',
             display_locations=['validation', 'summary', 'listings']))
         formdef.fields.append(fields.DateField(
             id='5', label='date field', type='date',
             display_locations=['validation', 'summary', 'listings']))
         formdef.fields.append(fields.StringField(
             id='6', label='number field', type='string',
             display_locations=['validation', 'summary', 'listings']))
         formdef.fields.append(fields.StringField(
             id='7', label='phone field', type='string',
             display_locations=['validation', 'summary', 'listings']))
         formdef.fields.append(fields.DateField(
             id='8', label='very old field', type='date',
             display_locations=['validation', 'summary', 'listings']))
         formdef.fields.append(fields.StringField(
             id='9', label='string field', type='string',
             display_locations=['validation', 'summary', 'listings']))
         formdef.store()
         formdata = formdef.data_class().select(lambda x: x.status == 'wf-new')[0]
         formdata.data['4'] = PicklableUpload('/foo/bar', content_type='text/plain')
         formdef.data_class().wipe()
         formdata = formdef.data_class()()
         formdata.data = {
             '4': PicklableUpload('/foo/bar', content_type='text/plain'),
             '5': time.strptime('2015-05-12', '%Y-%m-%d'),
             '6': '12345',
             '7': '0102030405',
             '8': time.strptime('1871-03-18', '%Y-%m-%d'),
             '9': 'plop\npl\x1dop',  # with control characters
+        }
         formdata.data['4'].receive([b'hello world'])
         formdata.data['5'] = time.strptime('2015-05-12', '%Y-%m-%d')
         formdata.data['6'] = '12345'
         formdata.data['7'] = '0102030405'
         formdata.data['8'] = time.strptime('1871-03-18', '%Y-%m-%d')
         formdata.data['9'] = 'plop\npl\x1dop'  # with control characters
         formdata.just_created()
         formdata.jump_status('new')
         formdata.store()
         resp = app.get('/backoffice/management/form-title/')
-...
     @pytest.mark.skipif('xlwt is None')
     def test_backoffice_xls(pub):
         create_superuser(pub)
         create_environment(pub)
         FormDef.wipe()
         formdef = FormDef()
         formdef.name = 'form title'
         formdef.fields = []
         formdef.store()
         app = login(get_app(pub))
         resp = app.get('/backoffice/management/form-title/')
         assert 'Excel Export' not in resp.text

     except ImportError:
         xlwt = None
     from django.conf import settings
     from django.utils import six
     from django.utils.encoding import force_text
     from django.utils.six.moves.urllib import parse as urllib
-...
             count = self.formdef.data_class().count()
             exporter = Exporter(self, self.formdef, fields, selected_filter)
             if count > 100: # Arbitrary threshold
             if count > getattr(settings, 'WCS_SYNC_EXPORT_LIMIT', 100):  # Arbitrary threshold, but parametrized
                 job = get_response().add_after_job(
                     str(N_('Exporting forms in CSV')),
                     exporter.export)
-...
             count = self.formdef.data_class().count()
             exporter = Exporter(self, self.formdef, fields, selected_filter)
             if count > 100: # Arbitrary threshold
             if count > getattr(settings, 'WCS_SYNC_EXPORT_LIMIT', 100):  # Arbitrary threshold, but parametrized
                 job = get_response().add_after_job(
                     str(N_('Exporting forms in Excel format')),
                     exporter.export)
-...
             count = self.formdef.data_class().count()
             exporter = Exporter(self, self.formdef, fields, selected_filter)
             if count > 100 and not get_request().is_api_url():
                 # The "100" threshold is arbitrary
             if count > getattr(settings, 'WCS_SYNC_EXPORT_LIMIT', 100)  and not get_request().is_api_url():
                 # The "100" threshold is arbitrary, but parametrized
                 job = get_response().add_after_job(
                     str(N_('Exporting forms in Open Document format')),
                     exporter.export)
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0003-misc-speed-up-backoffice-export-tests.patch