From bf2c805c53130fe0a9751c3e955156fafb7756c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 17 Jan 2022 10:53:43 +0100
Subject: [PATCH 2/3] misc: faster tests

---
 tests/backoffice_pages/test_columns.py | 276 +++++++++++++++------
 tests/backoffice_pages/test_filters.py | 330 +++++++++++++++++++------
 2 files changed, 454 insertions(+), 152 deletions(-)

diff --git a/tests/backoffice_pages/test_columns.py b/tests/backoffice_pages/test_columns.py
index 183074fb..76451910 100644
--- a/tests/backoffice_pages/test_columns.py
+++ b/tests/backoffice_pages/test_columns.py
@@ -5,6 +5,7 @@ import time
 import pytest
 
 from wcs import fields
+from wcs.admin.settings import UserFieldsFormDef
 from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.formdef import FormDef
@@ -12,7 +13,7 @@ from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.upload_storage import PicklableUpload
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
-from .test_all import create_environment, create_superuser, create_user
+from .test_all import create_superuser
 
 
 def pytest_generate_tests(metafunc):
@@ -45,75 +46,139 @@ def teardown_module(module):
 
 
 def test_backoffice_columns(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = [
+        fields.StringField(id='1', label='1st field', type='string', display_locations=['listings']),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {'1': 'Foo Bar'}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert resp.text.count('</th>') == 8  # six columns
+    assert resp.text.count('</th>') == 7  # five columns
     resp.forms['listing-settings']['1'].checked = False
     assert 'submission_channel' not in resp.forms['listing-settings'].fields
     assert 'last_update_time' in resp.forms['listing-settings'].fields
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('</th>') == 7  # fixe columns
-    assert resp.text.count('data-link') == 17  # 17 rows
+    assert resp.text.count('</th>') == 6  # fixe columns
+    assert resp.text.count('data-link') == 1  # 1 rows
     assert resp.text.count('FOO BAR') == 0  # no field 1 column
 
     # change column order
     assert (
         resp.forms['listing-settings']['columns-order'].value
-        == 'id,time,last_update_time,user-label,2,status,1,3,anonymised'
+        == 'id,time,last_update_time,user-label,status,1,anonymised'
     )
     resp.forms['listing-settings'][
         'columns-order'
-    ].value = 'user-label,id,time,last_update_time,2,status,1,3,anonymised'
+    ].value = 'user-label,id,time,last_update_time,status,1,anonymised'
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.find('<span>User Label</span>') < resp.text.find('<span>Number</span>')
 
 
 def test_backoffice_channel_column(pub):
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
     if not pub.site_options.has_section('variables'):
         pub.site_options.add_section('variables')
     pub.site_options.set('variables', 'welco_url', 'xxx')
     with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
         pub.site_options.write(fd)
 
-    create_superuser(pub)
-    create_environment(pub)
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert resp.text.count('</th>') == 8  # six columns
+    assert resp.text.count('</th>') == 6  # four columns
     resp.forms['listing-settings']['submission_channel'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('</th>') == 9  # seven columns
-    assert resp.text.count('data-link') == 17  # 17 rows
-    assert resp.text.count('<td>Web</td>') == 17
+    assert resp.text.count('</th>') == 7  # five columns
+    assert resp.text.count('data-link') == 1  # 1 row
+    assert resp.text.count('<td>Web</td>') == 1
 
 
 def test_backoffice_submission_agent_column(pub):
-    user = create_user(pub)
-    create_environment(pub)
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
 
     agent = pub.user_class(name='agent')
     agent.store()
 
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     assert 'submission_agent' not in resp.forms['listing-settings'].fields
 
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.backoffice_submission_roles = user.roles
+    formdef.backoffice_submission_roles = [role]
     formdef.store()
+
     resp = app.get('/backoffice/management/form-title/')
-    assert resp.text.count('</th>') == 8  # six columns
+    assert resp.text.count('</th>') == 6  # four columns
     resp.forms['listing-settings']['submission_agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('</th>') == 9  # seven columns
-    assert resp.text.count('data-link') == 17  # 17 rows
+    assert resp.text.count('</th>') == 7  # five columns
+    assert resp.text.count('data-link') == 1  # 1 row
     assert '>agent<' not in resp.text
     resp = resp.click('Export a Spreadsheet')
     resp.form['format'] = 'csv'
     resp = resp.form.submit('submit')
-    assert len(resp.text.splitlines()) == 18  # 17 + header line
+    assert len(resp.text.splitlines()) == 2  # 1 + header line
     assert ',"agent",' not in resp.text
 
     for formdata in formdef.data_class().select():
@@ -123,53 +188,83 @@ def test_backoffice_submission_agent_column(pub):
     resp = app.get('/backoffice/management/form-title/')
     resp.forms['listing-settings']['submission_agent'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('>agent<') == 17
+    assert resp.text.count('>agent<') == 1
 
     resp = resp.click('Export a Spreadsheet')
     resp.form['format'] = 'csv'
     resp = resp.form.submit('submit')
-    assert len(resp.text.splitlines()) == 18  # 17 + header line
-    assert resp.text.count(',"agent"') == 17
+    assert len(resp.text.splitlines()) == 2  # 1 + header line
+    assert resp.text.count(',"agent"') == 1
 
 
 def test_backoffice_image_column(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = [
         fields.FileField(
             id='4', label='file field', type='file', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
     upload = PicklableUpload('test.jpeg', 'image/jpeg')
     with open(os.path.join(os.path.dirname(__file__), '..', 'image-with-gps-data.jpeg'), 'rb') as fd:
         upload.receive([fd.read()])
-    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
-        formdata.data['4'] = upload
-        formdata.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {'4': upload}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     assert 'download?f=4&thumbnail=1' not in resp.text
 
 
 def test_backoffice_file_column(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = [
         fields.FileField(
             id='4', label='file field', type='file', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
     upload = PicklableUpload('a filename that is too long "and" will be ellipsised.txt', 'text/plain')
     upload.receive([b'text'])
-    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
-        formdata.data['4'] = upload
-        formdata.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {'4': upload}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     assert 'title="a filename that is too long &quot;and&quot; will be ellipsised.txt"' in resp
@@ -177,10 +272,11 @@ def test_backoffice_file_column(pub):
 
 
 def test_backoffice_user_columns(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-
-    from wcs.admin.settings import UserFieldsFormDef
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
 
     user_formdef = UserFieldsFormDef(pub)
     user_formdef.fields.append(fields.StringField(id='_first_name', label='name', type='string'))
@@ -198,14 +294,28 @@ def test_backoffice_user_columns(pub):
     user2.set_attributes_from_formdata(user2.form_data)
     user2.store()
 
-    formdef = FormDef.get_by_urlname('form-title')
-    for i, formdata in enumerate(formdef.data_class().select()):
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.geolocations = {'base': 'Geolocation'}
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
         formdata.user_id = user1.id if bool(i % 2) else user2.id
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert resp.text.count('</th>') == 8  # six columns
+    assert resp.text.count('</th>') == 6  # four columns
 
     if not pub.is_using_postgresql():
         # no support for relation columns unless using SQL
@@ -213,13 +323,16 @@ def test_backoffice_user_columns(pub):
         return
     resp.forms['listing-settings']['user-label$3'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('</th>') == 9
+    assert resp.text.count('</th>') == 7
     assert '<td>nono</td' in resp
 
 
 def test_backoffice_card_field_columns(pub):
-    user = create_superuser(pub)
-    create_environment(pub)
+    pub.user_class.wipe()
+    create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
 
     datasource = {'type': 'formula', 'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])}
 
@@ -233,8 +346,6 @@ def test_backoffice_card_field_columns(pub):
         fields.BoolField(id='3', label='Bool', type='bool'),
         fields.ItemField(id='4', label='Item', type='item', data_source=datasource),
     ]
-    carddef.backoffice_submission_roles = user.roles
-    carddef.workflow_roles = {'_editor': user.roles[0]}
     carddef.digest_templates = {'default': 'card {{ form_var_foo }}'}
     carddef.store()
     carddef.data_class().wipe()
@@ -249,25 +360,35 @@ def test_backoffice_card_field_columns(pub):
     }
     card.store()
 
-    formdef = FormDef.get_by_urlname('form-title')
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
     formdef.geolocations = {'base': 'Geolocation'}
-    formdef.fields.append(
+    formdef.fields = [
         fields.ItemField(
             id='4', label='card field', type='item', data_source={'type': 'carddef:foo', 'value': ''}
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
-        formdata.data['4'] = str(card.id)
-        formdata.data['4_display'] = formdef.fields[-1].store_display_value(formdata.data, '4')
-        formdata.data['4_structured'] = formdef.fields[-1].store_structured_value(formdata.data, '4')
-        formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
-        formdata.store()
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    formdata = data_class()
+    formdata.data = {
+        '4': str(card.id),
+    }
+    formdata.data['4_display'] = formdef.fields[-1].store_display_value(formdata.data, '4')
+    formdata.data['4_structured'] = formdef.fields[-1].store_structured_value(formdata.data, '4')
+    formdata.geolocations = {'base': {'lat': 48.83, 'lon': 2.32}}
+    formdata.just_created()
+    formdata.jump_status('new')
+    formdata.store()
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
-    assert resp.text.count('</th>') == 8  # six columns
+    assert resp.text.count('</th>') == 6  # four columns
     if not pub.is_using_postgresql():
         # no support for relation columns unless using SQL
         assert '4$1' not in resp.forms['listing-settings'].fields
@@ -278,12 +399,12 @@ def test_backoffice_card_field_columns(pub):
     resp.forms['listing-settings']['4$3'].checked = True
     resp.forms['listing-settings']['4$4'].checked = True
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('</th>') == 12
-    assert resp.text.count('data-link') == 17  # 17 rows
-    assert resp.text.count('<td>plop</td>') == 17
-    assert resp.text.count('<td>2020-04-24</td>') == 17
-    assert resp.text.count('<td>Yes</td>') == 17
-    assert resp.text.count('<td>aa</td>') == 17
+    assert resp.text.count('</th>') == 10
+    assert resp.text.count('data-link') == 1  # 1 row
+    assert resp.text.count('<td>plop</td>') == 1
+    assert resp.text.count('<td>2020-04-24</td>') == 1
+    assert resp.text.count('<td>Yes</td>') == 1
+    assert resp.text.count('<td>aa</td>') == 1
 
     resp_csv = resp.click('Export a Spreadsheet')
     resp_csv.form['format'] = 'csv'
@@ -302,8 +423,11 @@ def test_backoffice_card_field_columns(pub):
 
 
 def test_backoffice_block_columns(pub):
-    create_environment(pub)
+    pub.user_class.wipe()
     create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
 
     BlockDef.wipe()
     block = BlockDef()
@@ -313,20 +437,15 @@ def test_backoffice_block_columns(pub):
     ]
     block.store()
 
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.BlockField(id='8', label='Block', type='block:foobar', varname='data', max_items=3),
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for formdata in formdef.data_class().select(lambda x: x.status == 'wf-new'):
-        formdata.data['8'] = {
-            'data': [{'123': 'blah'}],
-            'schema': {'123': 'string'},
-        }
-        formdata.data['8_display'] = 'blah'
-        formdata.store()
-
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     # check Block / Test is not part of the possible columns
@@ -335,10 +454,7 @@ def test_backoffice_block_columns(pub):
         'Created',
         'Last Modified',
         'User Label',
-        '1st field',
-        '2nd field',
         'Status',
-        '3rd field',
         'Block',
         'Anonymised',
     ]
diff --git a/tests/backoffice_pages/test_filters.py b/tests/backoffice_pages/test_filters.py
index a9140ebf..e987fab7 100644
--- a/tests/backoffice_pages/test_filters.py
+++ b/tests/backoffice_pages/test_filters.py
@@ -95,14 +95,33 @@ def test_backoffice_filter(pub):
 
 
 def test_backoffice_default_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
+        fields.ItemField(
+            id='2',
+            label='2nd field',
+            type='item',
+            items=['foo', 'bar', 'baz'],
+            display_locations=['validation', 'summary', 'listings'],
+        ),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/')
     assert 'filter-2-value' not in resp.forms['listing-settings'].fields
 
     formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields[1].in_filters = True
+    formdef.fields[0].in_filters = True
     formdef.store()
     resp = app.get('/backoffice/management/form-title/')
     assert 'filter-2-value' in resp.forms['listing-settings'].fields
@@ -115,25 +134,38 @@ def test_backoffice_default_filter(pub):
     resp = app.get('/backoffice/management/form-title/')
     assert 'filter-4-value' not in resp.forms['listing-settings'].fields
 
-    formdef.fields[-1].in_filters = True
+    formdef.fields[1].in_filters = True
     formdef.store()
     resp = app.get('/backoffice/management/form-title/')
     assert 'filter-4-value' in resp.forms['listing-settings'].fields
 
 
 def test_backoffice_bool_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.BoolField(
             id='4', label='4th field', type='bool', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
-        formdata.data['4'] = bool(i % 2)
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {'4': bool(i % 2)}
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -155,10 +187,16 @@ def test_backoffice_bool_filter(pub):
 
 
 def test_backoffice_item_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.ItemField(
             id='4',
             label='4th field',
@@ -166,10 +204,16 @@ def test_backoffice_item_filter(pub):
             items=['â', 'b', 'c', 'd'],
             display_locations=['validation', 'summary', 'listings'],
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 12):
+        formdata = data_class()
+        formdata.data = {}
         if i % 4 == 0:
             formdata.data['4'] = 'â'
             formdata.data['4_display'] = 'â'
@@ -182,6 +226,11 @@ def test_backoffice_item_filter(pub):
         else:
             formdata.data['4'] = ''
             formdata.data['4_display'] = ''
+        formdata.just_created()
+        if i % 3 == 0:
+            formdata.jump_status('new')
+        else:
+            formdata.jump_status('finished')
         formdata.store()
 
     app = login(get_app(pub))
@@ -243,31 +292,40 @@ def test_backoffice_item_double_filter(pub):
     if not pub.is_using_postgresql():
         pytest.skip('this requires SQL')
         return
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.ItemField(
             id='4',
             label='4th field',
             type='item',
-            items=['a', 'b', 'c', 'd'],
+            items=['â', 'b', 'c', 'd'],
             display_locations=['validation', 'summary', 'listings'],
-        )
-    )
-    formdef.fields.append(
+        ),
         fields.ItemField(
             id='5',
             label='5th field',
             type='item',
             items=['E', 'F', 'G', 'H'],
             display_locations=['validation', 'summary', 'listings'],
-        )
-    )
+        ),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
-        formdata.jump_status('new')
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 4):
+        formdata = data_class()
+        formdata.data = {}
         if i % 4 == 0:
             formdata.data['4'] = 'a'
             formdata.data['4_display'] = 'a'
@@ -288,6 +346,8 @@ def test_backoffice_item_double_filter(pub):
             formdata.data['4_display'] = 'b'
             formdata.data['5'] = 'F'
             formdata.data['5_display'] = 'F'
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -323,9 +383,12 @@ def test_backoffice_item_double_filter(pub):
 
 
 def test_backoffice_bofield_item_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
     workflow = Workflow.get_default_workflow()
     workflow.id = '2'
     workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
@@ -339,10 +402,21 @@ def test_backoffice_bofield_item_filter(pub):
         )
     ]
     workflow.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.workflow_id = workflow.id
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 12):
+        formdata = data_class()
+        formdata.data = {}
         if i % 4 == 0:
             formdata.data['bo0-1'] = 'â'
             formdata.data['bo0-1_display'] = 'â'
@@ -352,6 +426,11 @@ def test_backoffice_bofield_item_filter(pub):
         elif i % 4 == 2:
             formdata.data['bo0-1'] = 'd'
             formdata.data['bo0-1_display'] = 'd'
+        formdata.just_created()
+        if i % 3 == 0:
+            formdata.jump_status('new')
+        else:
+            formdata.jump_status('finished')
         formdata.store()
 
     app = login(get_app(pub))
@@ -409,10 +488,16 @@ def test_backoffice_bofield_item_filter(pub):
 
 
 def test_backoffice_items_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.ItemsField(
             id='4',
             label='4th field',
@@ -420,10 +505,16 @@ def test_backoffice_items_filter(pub):
             items=['â', 'b', 'c', 'd'],
             display_locations=['validation', 'summary', 'listings'],
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 4):
+        formdata = data_class()
+        formdata.data = {}
         if i % 4 == 0:
             formdata.data['4'] = ['â', 'b']
             formdata.data['4_display'] = 'â, b'
@@ -436,6 +527,8 @@ def test_backoffice_items_filter(pub):
         else:
             formdata.data['4'] = []
             formdata.data['4_display'] = None
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -471,18 +564,32 @@ def test_backoffice_items_filter(pub):
 
 
 def test_backoffice_string_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.StringField(
             id='4', label='4th field', type='string', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
         formdata.data['4'] = 'a' if bool(i % 2) else 'b'
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -504,18 +611,32 @@ def test_backoffice_string_filter(pub):
 
 
 def test_backoffice_email_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.EmailField(
             id='4', label='4th field', type='email', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
         formdata.data['4'] = 'a@localhost' if bool(i % 2) else 'b@localhost'
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -540,18 +661,32 @@ def test_backoffice_date_filter(pub):
     if not pub.is_using_postgresql():
         pytest.skip('this requires SQL')
         return
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
-    formdef.fields.append(
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
         fields.DateField(
             id='4', label='4th field', type='date', display_locations=['validation', 'summary', 'listings']
         )
-    )
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
         formdata.data['4'] = time.strptime('2020-04-24' if bool(i % 2) else '2015-05-12', '%Y-%m-%d')
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -579,9 +714,17 @@ def test_backoffice_date_filter(pub):
 
 
 def test_backoffice_user_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
     user1 = pub.user_class(name='userA')
@@ -589,8 +732,15 @@ def test_backoffice_user_filter(pub):
     user2 = pub.user_class(name='userB')
     user2.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
         formdata.user_id = user1.id if bool(i % 2) else user2.id
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -630,8 +780,15 @@ def test_backoffice_user_filter(pub):
 def test_backoffice_submission_agent_filter(pub):
     pub.user_class.wipe()
     user = create_superuser(pub)
-    create_environment(pub)
-    formdef = FormDef.get_by_urlname('form-title')
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = []
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.backoffice_submission_roles = user.roles
     formdef.store()
 
@@ -640,9 +797,15 @@ def test_backoffice_submission_agent_filter(pub):
     user2 = pub.user_class(name='userB')
     user2.store()
 
-    for i, formdata in enumerate(formdef.data_class().select()):
-        formdata.submission_agent_id = str(user1.id if i < 10 else user2.id)
-        formdata.status = 'wf-new'
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
+        formdata.submission_agent_id = str(user1.id if bool(i % 2) else user2.id)
+        formdata.just_created()
+        formdata.jump_status('new')
         formdata.store()
 
     app = login(get_app(pub))
@@ -659,24 +822,24 @@ def test_backoffice_submission_agent_filter(pub):
     resp = app.get(base_url + '&filter-submission-agent=on&filter-submission-agent-value=%s' % user1.id)
     assert resp.text.count('>userA<') > 0
     assert resp.text.count('>userB<') == 0
-    assert resp.text.count('<tr') == 11
+    assert resp.text.count('<tr') == 2
     assert resp.pyquery.find('input[value=userA]')  # displayed in sidebar
     # check it persits on filter changes
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('>userA<') > 0
     assert resp.text.count('>userB<') == 0
-    assert resp.text.count('<tr') == 11
+    assert resp.text.count('<tr') == 2
 
     resp = app.get(base_url + '&filter-submission-agent=on&filter-submission-agent-value=%s' % user2.id)
     assert resp.text.count('>userA<') == 0
     assert resp.text.count('>userB<') > 0
-    assert resp.text.count('<tr') == 42
+    assert resp.text.count('<tr') == 2
 
     resp = app.get(
         '/backoffice/management/form-title/?limit=100&filter-submission-agent=on&filter-submission-agent-value=%s'
         % user2.id
     )
-    assert resp.text.count('<tr') == 42
+    assert resp.text.count('<tr') == 2
 
     # filter on uuid
     user1.name_identifiers = ['0123456789']
@@ -697,26 +860,56 @@ def test_backoffice_submission_agent_filter(pub):
 
 
 def test_backoffice_table_varname_filter(pub):
+    pub.user_class.wipe()
     create_superuser(pub)
-    create_environment(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
+    datasource = {'type': 'formula', 'value': repr([('A', 'aa'), ('B', 'bb'), ('C', 'cc')])}
+    FormDef.wipe()
+    formdef = FormDef()
+    formdef.name = 'form-title'
+    formdef.fields = [
+        fields.ItemField(id='3', label='3rd field', type='item', data_source=datasource, varname='foo'),
+    ]
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    data_class = formdef.data_class()
+    data_class.wipe()
+
+    for i in range(0, 2):
+        formdata = data_class()
+        formdata.data = {}
+        formdata.data['3'] = 'A' if bool(i % 2) else 'B'
+        formdata.data['3_display'] = 'aa' if bool(i % 2) else 'bb'
+        formdata.just_created()
+        formdata.jump_status('new')
+        formdata.store()
 
     app = login(get_app(pub))
     resp = app.get('/backoffice/management/form-title/?filter-foo=A')
     # check filter is applied
-    assert resp.text.count('<tr') == 6
+    assert resp.text.count('<tr') == 2
     # and kept in parameters
     assert resp.forms['listing-settings']['filter-3'].checked
     assert resp.forms['listing-settings']['filter-3-value'].value == 'A'
 
     resp = resp.forms['listing-settings'].submit()
-    assert resp.text.count('<tr') == 6
+    assert resp.text.count('<tr') == 2
 
 
-def test_backoffice_block_field_filter(pub, local_user):
+def test_backoffice_block_field_filter(pub):
     if not pub.is_using_postgresql():
         pytest.skip('this requires SQL')
 
+    pub.user_class.wipe()
     create_superuser(pub)
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+
     NamedDataSource.wipe()
     data_source = NamedDataSource(name='foobar')
     data_source.data_source = {
@@ -737,13 +930,6 @@ def test_backoffice_block_field_filter(pub, local_user):
     ]
     block.store()
 
-    pub.role_class.wipe()
-    role = pub.role_class(name='test')
-    role.store()
-
-    local_user.roles = [role.id]
-    local_user.store()
-
     FormDef.wipe()
     formdef = FormDef()
     formdef.name = 'form-title'
@@ -751,6 +937,7 @@ def test_backoffice_block_field_filter(pub, local_user):
     formdef.fields = [
         fields.BlockField(id='0', label='Block Data', varname='blockdata', type='block:foobar', max_items=3),
     ]
+    formdef.workflow_roles = {'_receiver': role.id}
     formdef.store()
 
     data_class = formdef.data_class()
@@ -787,7 +974,6 @@ def test_backoffice_block_field_filter(pub, local_user):
                     '5': 'a@localhost',
                 },
             )
-        formdata.user_id = local_user.id
         formdata.just_created()
         formdata.jump_status('new')
         formdata.store()
-- 
2.34.1

