From 840774c253e79e4b4ce2f6dd5662eabb15be1d78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 16 Nov 2020 15:14:12 +0100
Subject: [PATCH 1/2] misc: split backoffice_pages tests (carddata)

---
 tests/backoffice_pages/test_all.py      | 454 +--------------------
 tests/backoffice_pages/test_carddata.py | 504 ++++++++++++++++++++++++
 2 files changed, 505 insertions(+), 453 deletions(-)
 create mode 100644 tests/backoffice_pages/test_carddata.py

diff --git a/tests/backoffice_pages/test_all.py b/tests/backoffice_pages/test_all.py
index 8a7597fe..b5cd4702 100644
--- a/tests/backoffice_pages/test_all.py
+++ b/tests/backoffice_pages/test_all.py
@@ -18,8 +18,6 @@ except ImportError:
 from django.utils.six import StringIO, BytesIO
 from django.utils.six.moves.urllib import parse as urllib
 
-from webtest import Upload
-
 from quixote import get_publisher
 from quixote.http_request import Upload as QuixoteUpload
 from wcs.api_utils import sign_url
@@ -46,7 +44,7 @@ from wcs.wf.resubmit import ResubmitWorkflowStatusItem
 from wcs.wf.create_formdata import CreateFormdataWorkflowStatusItem, Mapping
 from wcs.wf.create_carddata import CreateCarddataWorkflowStatusItem
 from wcs.carddef import CardDef
-from wcs.categories import Category, CardDefCategory
+from wcs.categories import Category
 from wcs.formdef import FormDef
 from wcs.logged_errors import LoggedError
 from wcs import fields
@@ -6287,394 +6285,6 @@ def test_workflow_comment_required(pub):
     assert 'HELLO WORLD 2' in resp.text
 
 
-def test_carddata_management(pub, studio):
-    CardDef.wipe()
-    user = create_user(pub)
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/')
-    assert 'Cards' not in resp.text
-    carddef = CardDef()
-    carddef.name = 'foo'
-    carddef.fields = [
-        fields.StringField(id='1', label='Test', type='string', varname='foo'),
-        fields.StringField(id='2', label='Condi', type='string', varname='bar',
-            required=True, condition={'type': 'django', 'value': 'form_var_foo == "ok"'}),
-    ]
-    carddef.store()
-    carddef.data_class().wipe()
-
-    resp = app.get('/backoffice/')
-    assert 'Cards' not in resp.text
-
-    carddef.backoffice_submission_roles = user.roles
-    carddef.store()
-    resp = app.get('/backoffice/')
-    assert 'Cards' in resp.text
-
-    carddef.backoffice_submission_roles = None
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.store()
-    resp = app.get('/backoffice/')
-    assert 'Cards' in resp.text
-
-    resp = app.get('/backoffice/data/')
-    resp = resp.click('foo')
-    assert 'Add' not in resp.text
-
-    carddef.backoffice_submission_roles = user.roles
-    carddef.store()
-
-    resp = app.get('/backoffice/data/')
-    resp = resp.click('foo')
-    assert resp.text.count('<tr') == 1  # header
-    assert 'Add' in resp.text
-    resp = resp.click('Add')
-    resp.form['f1'] = 'blah'
-
-    live_url = resp.html.find('form').attrs['data-live-url']
-    assert '/backoffice/data/foo/add/live' in live_url
-    live_resp = app.post(live_url, params=resp.form.submit_fields())
-    assert live_resp.json['result']['1']['visible']
-    assert not live_resp.json['result']['2']['visible']
-    resp.form['f1'] = 'ok'
-    live_resp = app.post(live_url, params=resp.form.submit_fields())
-    assert live_resp.json['result']['1']['visible']
-    assert live_resp.json['result']['2']['visible']
-    resp.form['f2'] = 'blah'
-
-    resp = resp.form.submit('submit')
-    assert resp.location.endswith('/backoffice/data/foo/1/')
-    resp = resp.follow()
-    assert 'Edit Card' in resp.text
-    assert 'Delete Card' in resp.text
-
-    carddata = carddef.data_class().select()[0]
-    assert carddata.data == {'1': 'ok', '2': 'blah'}
-    assert carddata.user_id is None
-    assert carddata.submission_agent_id == str(user.id)
-    assert carddata.evolution[0].who == str(user.id)
-    assert 'Original Submitter' not in resp.text
-
-    resp = app.get('/backoffice/data/')
-    resp = resp.click('foo')
-    assert resp.text.count('<tr') == 2  # header + row of data
-
-
-def test_carddata_management_categories(pub, studio):
-    user = create_user(pub)
-
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.name = 'foo'
-    carddef.fields = []
-    carddef.backoffice_submission_roles = None
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.store()
-
-    carddef2 = CardDef()
-    carddef2.name = 'card title 2'
-    carddef2.fields = []
-    carddef2.backoffice_submission_roles = None
-    carddef2.workflow_roles = {'_editor': user.roles[0]}
-    carddef2.store()
-
-    CardDefCategory.wipe()
-    cat = CardDefCategory(name='Foo')
-    cat.store()
-    cat2 = CardDefCategory(name='Bar')
-    cat2.store()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/data/')
-    assert '<h3>Misc</h3>' not in resp.text
-    assert '<h3>Foo</h3>' not in resp.text
-    assert '<h3>Bar</h3>' not in resp.text
-
-    carddef.category = cat2
-    carddef.store()
-    resp = app.get('/backoffice/data/')
-    assert '<h3>Misc</h3>' in resp.text
-    assert '<h3>Foo</h3>' not in resp.text
-    assert '<h3>Bar</h3>' in resp.text
-
-    carddef2.category = cat
-    carddef2.store()
-    resp = app.get('/backoffice/data/')
-    assert '<h3>Misc</h3>' not in resp.text
-    assert '<h3>Foo</h3>' in resp.text
-    assert '<h3>Bar</h3>' in resp.text
-
-
-def test_studio_card_item_link(pub, studio):
-    user = create_user(pub)
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.name = 'foo'
-    carddef.fields = [
-        fields.StringField(id='1', label='Test', type='string', varname='foo'),
-    ]
-    carddef.backoffice_submission_roles = user.roles
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.digest_template = 'card {{form_var_foo}}'
-    carddef.store()
-    carddef.data_class().wipe()
-
-    card = carddef.data_class()()
-    card.data = {'1': 'plop'}
-    card.just_created()
-    card.store()
-
-    carddef2 = CardDef()
-    carddef2.name = 'bar'
-    carddef2.fields = [
-        fields.ItemField(id='1', label='Test', type='item',
-            data_source={'type': 'carddef:foo', 'value': ''}),
-    ]
-    carddef2.backoffice_submission_roles = user.roles
-    carddef2.workflow_roles = {'_editor': user.roles[0]}
-    carddef2.store()
-    carddef2.data_class().wipe()
-
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/data/')
-    resp = resp.click('bar')
-    resp = resp.click('Add')
-    resp.form['f1'] = card.id
-    resp = resp.form.submit('submit')
-    assert resp.location.endswith('/backoffice/data/bar/1/')
-    resp = resp.follow()
-    resp = resp.click('card plop')
-    assert '<div class="value">plop</div>' in resp
-
-    # link to a unknown carddef
-    carddef2.fields = [
-        fields.ItemField(id='1', label='Test', type='item',
-            data_source={'type': 'carddef:unknown', 'value': ''}),
-    ]
-    carddef2.store()
-    app = login(get_app(pub))
-    resp = app.get('/backoffice/data/')
-    resp = resp.click('bar')
-    resp = resp.click('Add')  # no error
-
-    # look without access rights
-    carddef.backoffice_submission_roles = None
-    carddef.workflow_roles = {'_editor': None}
-    carddef.store()
-    resp = app.get('/backoffice/data/bar/1/')
-    with pytest.raises(IndexError):
-        resp.click('card plop')
-
-
-def test_backoffice_cards_import_data_from_csv(pub, studio):
-    user = create_user(pub)
-
-    data_source = {
-        'type': 'formula',
-        'value': repr([
-            {'id': '1', 'text': 'un', 'more': 'foo'},
-            {'id': '2', 'text': 'deux', 'more': 'bar'}])
-    }
-
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.name = 'test'
-    carddef.fields = [
-        fields.TableField(id='0', label='Table'),
-        fields.MapField(id='1', label='Map'),
-        fields.StringField(id='2', label='Test'),
-        fields.BoolField(id='3', label='Boolean'),
-        fields.ItemField(id='4', label='List',
-                         items=['item1', 'item2']),
-        fields.DateField(id='5', label='Date'),
-        fields.TitleField(id='6', label='Title', type='title'),
-        fields.FileField(id='7', label='File'),
-        fields.EmailField(id='8', label='Email'),
-        fields.TextField(id='9', label='Long'),
-        fields.ItemField(id='10', label='List2', data_source=data_source),
-    ]
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.store()
-    carddef.data_class().wipe()
-
-    app = login(get_app(pub))
-
-    resp = app.get(carddef.get_url())
-    assert 'Import data from a CSV file' not in resp.text
-    resp = app.get(carddef.get_url() + 'import-csv', status=403)
-
-    carddef.backoffice_submission_roles = user.roles
-    carddef.store()
-
-    resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
-
-    assert 'Table, File are required but cannot be filled from CSV.' in resp
-    assert 'Download sample file for this card' not in resp
-    carddef.fields[0].required = False
-    carddef.fields[7].required = False
-    carddef.store()
-
-    resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
-    sample_resp = resp.click('Download sample file for this card')
-    today = datetime.date.today()
-    assert sample_resp.text == (
-            "Table,Map,Test,Boolean,List,Date,File,Email,Long,List2\r\n"
-            "will be ignored - type Table not supported,"
-            "%s,"
-            "value,"
-            "Yes,"
-            "value,"
-            "%s,"
-            "will be ignored - type File Upload not supported,"
-            "foo@example.com,"
-            "value,"
-            "value\r\n" % (pub.get_default_position(), today))
-
-    # missing file
-    resp = resp.forms[0].submit()
-    assert '>required field<' in resp
-
-    resp.forms[0]['file'] = Upload('test.csv', b'\0', 'text/csv')
-    resp = resp.forms[0].submit()
-    assert 'Invalid file format.' in resp
-
-    resp.forms[0]['file'] = Upload('test.csv', b'', 'text/csv')
-    resp = resp.forms[0].submit()
-    assert 'Invalid CSV file.' in resp
-
-    resp.forms[0]['file'] = Upload('test.csv',
-                                   b'Test,List,Date\ndata1,item1,invalid',
-                                   'text/csv')
-    resp = resp.forms[0].submit()
-    assert 'CSV file contains less columns than card fields.' in resp.text
-
-    data = [b'Table,Map,Test,Boolean,List,Date,File,Email,Long,List2']
-    for i in range(1, 150):
-        data.append(b'table,48.81;2.37,data%d ,%s,item%d,2020-01-%02d,filename-%d,test@localhost,"plop\nplop",1' % (
-            i, str(bool(i % 2)).encode('utf-8'), i, i % 31 + 1, i))
-
-    resp.forms[0]['file'] = Upload('test.csv', b'\n'.join(data),
-                                   'text/csv')
-    resp = resp.forms[0].submit().follow()
-    assert 'Importing data into cards' in resp
-    assert 'Column File will be ignored: type File Upload not supported.' in resp
-    assert carddef.data_class().count() == 149
-    card1, card2 = carddef.data_class().select(order_by='id')[:2]
-    assert card1.data['1'] == '48.81;2.37'
-    assert card1.data['2'] == 'data1'
-    assert card1.data['3'] is True
-    assert card1.data['5'].tm_mday == 2
-    assert card1.data['9'] == 'plop\nplop'
-    assert card1.data['10'] == '1'
-    assert card1.data['10_display'] == 'un'
-    assert card1.data['10_structured'] == {'id': '1', 'text': 'un', 'more': 'foo'}
-    assert card2.data['2'] == 'data2'
-    assert card2.data['3'] is False
-    assert card2.data['5'].tm_mday == 3
-
-
-def test_backoffice_cards_import_data_csv_invalid_columns(pub):
-    user = create_user(pub)
-
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.backoffice_submission_roles = user.roles
-    carddef.name = 'test'
-    carddef.fields = [
-        fields.StringField(id='1', label='String1'),
-        fields.StringField(id='2', label='String2'),
-        fields.TextField(id='3', label='Text'),
-    ]
-    carddef.store()
-
-    app = login(get_app(pub))
-    resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
-
-    csv_data = '''String1,String2,Text
-1,2,3
-4,5,6
-7,
-8,9,10,11
-12,13,14
-
-'''
-    resp.forms[0]['file'] = Upload('test.csv', csv_data.encode('utf-8'), 'text/csv')
-    resp = resp.forms[0].submit()
-    assert 'CSV file contains lines with wrong number of columns.' in resp.text
-    assert '(line numbers 4, 5, 7)' in resp.text
-
-    csv_data += '\n' * 10
-    resp.forms[0]['file'] = Upload('test.csv', csv_data.encode('utf-8'), 'text/csv')
-    resp = resp.forms[0].submit()
-    assert 'CSV file contains lines with wrong number of columns.' in resp.text
-    assert '(line numbers 4, 5, 7, 8, 9 and more)' in resp.text
-
-
-def test_backoffice_cards_wscall_failure_display(http_requests, pub, studio):
-    LoggedError.wipe()
-    user = create_user(pub)
-
-    Workflow.wipe()
-    workflow = Workflow(name='wscall')
-    workflow.roles = {
-        '_viewer': 'Viewer',
-        '_editor': 'Editor',
-    }
-    st1 = workflow.add_status('Recorded', 'recorded')
-
-    wscall = WebserviceCallStatusItem()
-    wscall.id = '_wscall'
-    wscall.varname = 'xxx'
-    wscall.url = 'http://remote.example.net/xml'
-    wscall.action_on_bad_data = ':stop'
-    wscall.record_errors = True
-    st1.items.append(wscall)
-    wscall.parent = st1
-
-    again = ChoiceWorkflowStatusItem()
-    again.id = '_again'
-    again.label = 'Again'
-    again.by = ['_editor']
-    again.status = st1.id
-    st1.items.append(again)
-    again.parent = st1
-
-    workflow.store()
-
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.name = 'foo'
-    carddef.fields = [
-        fields.StringField(id='1', label='Test', type='string', varname='foo'),
-    ]
-    carddef.backoffice_submission_roles = user.roles
-    carddef.workflow_id = workflow.id
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.digest_template = 'card {{form_var_foo}}'
-    carddef.store()
-    carddef.data_class().wipe()
-
-    carddata = carddef.data_class()()
-    carddata.data = {'1': 'plop'}
-    carddata.just_created()
-    carddata.store()
-
-    app = login(get_app(pub))
-
-    resp = app.get('/backoffice/data/foo/%s/' % carddata.id)
-    assert 'Again' in resp.text
-    resp = resp.forms[0].submit('button_again')
-    resp = resp.follow()
-    assert 'Error during webservice call' in resp.text
-
-    assert LoggedError.count() == 1
-    assert LoggedError.select()[0].get_formdata().data == {'1': 'plop'}
-
-
 def test_lazy_eval_with_conditional_workflow_form(pub):
     role = Role(name='foobar')
     role.store()
@@ -7150,65 +6760,3 @@ def test_backoffice_table_varname_filter(pub):
 
     resp = resp.forms['listing-settings'].submit()
     assert resp.text.count('<tr') == 6
-
-
-def test_block_card_item_link(pub, studio, blocks_feature):
-    user = create_user(pub)
-    CardDef.wipe()
-    carddef = CardDef()
-    carddef.name = 'foo'
-    carddef.fields = [
-        fields.StringField(id='1', label='Test', type='string', varname='foo'),
-    ]
-    carddef.backoffice_submission_roles = user.roles
-    carddef.workflow_roles = {'_editor': user.roles[0]}
-    carddef.digest_template = 'card {{form_var_foo}}'
-    carddef.store()
-    carddef.data_class().wipe()
-
-    card = carddef.data_class()()
-    card.data = {'1': 'plop'}
-    card.just_created()
-    card.store()
-
-    card2 = carddef.data_class()()
-    card2.data = {'1': 'plop2'}
-    card2.just_created()
-    card2.store()
-
-    BlockDef.wipe()
-    block = BlockDef()
-    block.name = 'foobar'
-    block.fields = [
-        fields.ItemField(id='1', label='Test', type='item',
-            data_source={'type': 'carddef:foo', 'value': ''}),
-    ]
-    block.store()
-
-    formdef = FormDef()
-    formdef.name = 'bar'
-    formdef.fields = [
-        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
-    ]
-    formdef.store()
-    formdef.data_class().wipe()
-
-    app = login(get_app(pub))
-    resp = app.get('/bar/')
-    resp.form['f1$element0$f1'].value = card.id
-    resp = resp.form.submit('f1$add_element')
-    resp.form['f1$element1$f1'].value = card2.id
-    resp = resp.form.submit('submit')  # -> validation page
-    assert resp.form['f1$element0$f1'].value == str(card.id)
-    assert resp.form['f1$element0$f1_label'].value == 'card plop'
-    assert resp.form['f1$element1$f1'].value == str(card2.id)
-    assert resp.form['f1$element1$f1_label'].value == 'card plop2'
-    resp = resp.form.submit('submit')  # -> final submit
-    resp = resp.follow()
-    assert '<div class="value">card plop</div>' in resp
-    assert '<div class="value">card plop2</div>' in resp
-
-    # check cards are links in backoffice
-    resp = app.get('/backoffice/management' + resp.request.path)
-    assert '<div class="value"><a href="http://example.net/backoffice/data/foo/%s/">card plop</a></div></div>' % card.id in resp
-    assert '<div class="value"><a href="http://example.net/backoffice/data/foo/%s/">card plop2</a></div></div>' % card2.id in resp
diff --git a/tests/backoffice_pages/test_carddata.py b/tests/backoffice_pages/test_carddata.py
new file mode 100644
index 00000000..4ddd591e
--- /dev/null
+++ b/tests/backoffice_pages/test_carddata.py
@@ -0,0 +1,504 @@
+# -*- coding: utf-8 -*-
+import datetime
+import os
+
+import pytest
+from webtest import Upload
+
+from wcs import fields
+from wcs.blocks import BlockDef
+from wcs.carddef import CardDef
+from wcs.categories import CardDefCategory
+from wcs.formdef import FormDef
+from wcs.logged_errors import LoggedError
+from wcs.qommon.http_request import HTTPRequest
+from wcs.wf.wscall import WebserviceCallStatusItem
+from wcs.workflows import ChoiceWorkflowStatusItem, Workflow
+
+from utilities import clean_temporary_pub, create_temporary_pub, get_app, login
+from .test_all import create_user
+
+
+def pytest_generate_tests(metafunc):
+    if 'pub' in metafunc.fixturenames:
+        metafunc.parametrize('pub', ['pickle', 'sql', 'pickle-templates'], indirect=True)
+
+
+@pytest.fixture
+def pub(request, emails):
+    pub = create_temporary_pub(
+            sql_mode=bool('sql' in request.param),
+            templates_mode=bool('templates' in request.param)
+            )
+
+    req = HTTPRequest(None, {'SCRIPT_NAME': '/', 'SERVER_NAME': 'example.net'})
+    pub.set_app_dir(req)
+    pub.cfg['identification'] = {'methods': ['password']}
+    pub.cfg['language'] = {'language': 'en'}
+    pub.write_cfg()
+    fd = open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w')
+    fd.write('''
+[api-secrets]
+coucou = 1234
+''')
+    fd.close()
+
+    return pub
+
+
+def teardown_module(module):
+    clean_temporary_pub()
+
+
+def test_carddata_management(pub, studio):
+    CardDef.wipe()
+    user = create_user(pub)
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/')
+    assert 'Cards' not in resp.text
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', type='string', varname='foo'),
+        fields.StringField(
+            id='2', label='Condi', type='string', varname='bar',
+            required=True, condition={'type': 'django', 'value': 'form_var_foo == "ok"'}),
+    ]
+    carddef.store()
+    carddef.data_class().wipe()
+
+    resp = app.get('/backoffice/')
+    assert 'Cards' not in resp.text
+
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    resp = app.get('/backoffice/')
+    assert 'Cards' in resp.text
+
+    carddef.backoffice_submission_roles = None
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.store()
+    resp = app.get('/backoffice/')
+    assert 'Cards' in resp.text
+
+    resp = app.get('/backoffice/data/')
+    resp = resp.click('foo')
+    assert 'Add' not in resp.text
+
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+
+    resp = app.get('/backoffice/data/')
+    resp = resp.click('foo')
+    assert resp.text.count('<tr') == 1  # header
+    assert 'Add' in resp.text
+    resp = resp.click('Add')
+    resp.form['f1'] = 'blah'
+
+    live_url = resp.html.find('form').attrs['data-live-url']
+    assert '/backoffice/data/foo/add/live' in live_url
+    live_resp = app.post(live_url, params=resp.form.submit_fields())
+    assert live_resp.json['result']['1']['visible']
+    assert not live_resp.json['result']['2']['visible']
+    resp.form['f1'] = 'ok'
+    live_resp = app.post(live_url, params=resp.form.submit_fields())
+    assert live_resp.json['result']['1']['visible']
+    assert live_resp.json['result']['2']['visible']
+    resp.form['f2'] = 'blah'
+
+    resp = resp.form.submit('submit')
+    assert resp.location.endswith('/backoffice/data/foo/1/')
+    resp = resp.follow()
+    assert 'Edit Card' in resp.text
+    assert 'Delete Card' in resp.text
+
+    carddata = carddef.data_class().select()[0]
+    assert carddata.data == {'1': 'ok', '2': 'blah'}
+    assert carddata.user_id is None
+    assert carddata.submission_agent_id == str(user.id)
+    assert carddata.evolution[0].who == str(user.id)
+    assert 'Original Submitter' not in resp.text
+
+    resp = app.get('/backoffice/data/')
+    resp = resp.click('foo')
+    assert resp.text.count('<tr') == 2  # header + row of data
+
+
+def test_carddata_management_categories(pub, studio):
+    user = create_user(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = []
+    carddef.backoffice_submission_roles = None
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.store()
+
+    carddef2 = CardDef()
+    carddef2.name = 'card title 2'
+    carddef2.fields = []
+    carddef2.backoffice_submission_roles = None
+    carddef2.workflow_roles = {'_editor': user.roles[0]}
+    carddef2.store()
+
+    CardDefCategory.wipe()
+    cat = CardDefCategory(name='Foo')
+    cat.store()
+    cat2 = CardDefCategory(name='Bar')
+    cat2.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/data/')
+    assert '<h3>Misc</h3>' not in resp.text
+    assert '<h3>Foo</h3>' not in resp.text
+    assert '<h3>Bar</h3>' not in resp.text
+
+    carddef.category = cat2
+    carddef.store()
+    resp = app.get('/backoffice/data/')
+    assert '<h3>Misc</h3>' in resp.text
+    assert '<h3>Foo</h3>' not in resp.text
+    assert '<h3>Bar</h3>' in resp.text
+
+    carddef2.category = cat
+    carddef2.store()
+    resp = app.get('/backoffice/data/')
+    assert '<h3>Misc</h3>' not in resp.text
+    assert '<h3>Foo</h3>' in resp.text
+    assert '<h3>Bar</h3>' in resp.text
+
+
+def test_studio_card_item_link(pub, studio):
+    user = create_user(pub)
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', type='string', varname='foo'),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.digest_template = 'card {{form_var_foo}}'
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop'}
+    card.just_created()
+    card.store()
+
+    carddef2 = CardDef()
+    carddef2.name = 'bar'
+    carddef2.fields = [
+        fields.ItemField(
+            id='1', label='Test', type='item',
+            data_source={'type': 'carddef:foo', 'value': ''}),
+    ]
+    carddef2.backoffice_submission_roles = user.roles
+    carddef2.workflow_roles = {'_editor': user.roles[0]}
+    carddef2.store()
+    carddef2.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/data/')
+    resp = resp.click('bar')
+    resp = resp.click('Add')
+    resp.form['f1'] = card.id
+    resp = resp.form.submit('submit')
+    assert resp.location.endswith('/backoffice/data/bar/1/')
+    resp = resp.follow()
+    resp = resp.click('card plop')
+    assert '<div class="value">plop</div>' in resp
+
+    # link to a unknown carddef
+    carddef2.fields = [
+        fields.ItemField(
+            id='1', label='Test', type='item',
+            data_source={'type': 'carddef:unknown', 'value': ''}),
+    ]
+    carddef2.store()
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/data/')
+    resp = resp.click('bar')
+    resp = resp.click('Add')  # no error
+
+    # look without access rights
+    carddef.backoffice_submission_roles = None
+    carddef.workflow_roles = {'_editor': None}
+    carddef.store()
+    resp = app.get('/backoffice/data/bar/1/')
+    with pytest.raises(IndexError):
+        resp.click('card plop')
+
+
+def test_backoffice_cards_import_data_from_csv(pub, studio):
+    user = create_user(pub)
+
+    data_source = {
+        'type': 'formula',
+        'value': repr([
+            {'id': '1', 'text': 'un', 'more': 'foo'},
+            {'id': '2', 'text': 'deux', 'more': 'bar'}])
+    }
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.TableField(id='0', label='Table'),
+        fields.MapField(id='1', label='Map'),
+        fields.StringField(id='2', label='Test'),
+        fields.BoolField(id='3', label='Boolean'),
+        fields.ItemField(id='4', label='List',
+                         items=['item1', 'item2']),
+        fields.DateField(id='5', label='Date'),
+        fields.TitleField(id='6', label='Title', type='title'),
+        fields.FileField(id='7', label='File'),
+        fields.EmailField(id='8', label='Email'),
+        fields.TextField(id='9', label='Long'),
+        fields.ItemField(id='10', label='List2', data_source=data_source),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+
+    resp = app.get(carddef.get_url())
+    assert 'Import data from a CSV file' not in resp.text
+    resp = app.get(carddef.get_url() + 'import-csv', status=403)
+
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a CSV file')
+
+    assert 'Table, File are required but cannot be filled from CSV.' in resp
+    assert 'Download sample file for this card' not in resp
+    carddef.fields[0].required = False
+    carddef.fields[7].required = False
+    carddef.store()
+
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a CSV file')
+    sample_resp = resp.click('Download sample file for this card')
+    today = datetime.date.today()
+    assert sample_resp.text == (
+            "Table,Map,Test,Boolean,List,Date,File,Email,Long,List2\r\n"
+            "will be ignored - type Table not supported,"
+            "%s,"
+            "value,"
+            "Yes,"
+            "value,"
+            "%s,"
+            "will be ignored - type File Upload not supported,"
+            "foo@example.com,"
+            "value,"
+            "value\r\n" % (pub.get_default_position(), today))
+
+    # missing file
+    resp = resp.forms[0].submit()
+    assert '>required field<' in resp
+
+    resp.forms[0]['file'] = Upload('test.csv', b'\0', 'text/csv')
+    resp = resp.forms[0].submit()
+    assert 'Invalid file format.' in resp
+
+    resp.forms[0]['file'] = Upload('test.csv', b'', 'text/csv')
+    resp = resp.forms[0].submit()
+    assert 'Invalid CSV file.' in resp
+
+    resp.forms[0]['file'] = Upload('test.csv',
+                                   b'Test,List,Date\ndata1,item1,invalid',
+                                   'text/csv')
+    resp = resp.forms[0].submit()
+    assert 'CSV file contains less columns than card fields.' in resp.text
+
+    data = [b'Table,Map,Test,Boolean,List,Date,File,Email,Long,List2']
+    for i in range(1, 150):
+        data.append(b'table,48.81;2.37,data%d ,%s,item%d,2020-01-%02d,filename-%d,test@localhost,"plop\nplop",1' % (
+            i, str(bool(i % 2)).encode('utf-8'), i, i % 31 + 1, i))
+
+    resp.forms[0]['file'] = Upload('test.csv', b'\n'.join(data),
+                                   'text/csv')
+    resp = resp.forms[0].submit().follow()
+    assert 'Importing data into cards' in resp
+    assert 'Column File will be ignored: type File Upload not supported.' in resp
+    assert carddef.data_class().count() == 149
+    card1, card2 = carddef.data_class().select(order_by='id')[:2]
+    assert card1.data['1'] == '48.81;2.37'
+    assert card1.data['2'] == 'data1'
+    assert card1.data['3'] is True
+    assert card1.data['5'].tm_mday == 2
+    assert card1.data['9'] == 'plop\nplop'
+    assert card1.data['10'] == '1'
+    assert card1.data['10_display'] == 'un'
+    assert card1.data['10_structured'] == {'id': '1', 'text': 'un', 'more': 'foo'}
+    assert card2.data['2'] == 'data2'
+    assert card2.data['3'] is False
+    assert card2.data['5'].tm_mday == 3
+
+
+def test_backoffice_cards_import_data_csv_invalid_columns(pub):
+    user = create_user(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.backoffice_submission_roles = user.roles
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='String1'),
+        fields.StringField(id='2', label='String2'),
+        fields.TextField(id='3', label='Text'),
+    ]
+    carddef.store()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a CSV file')
+
+    csv_data = '''String1,String2,Text
+1,2,3
+4,5,6
+7,
+8,9,10,11
+12,13,14
+
+'''
+    resp.forms[0]['file'] = Upload('test.csv', csv_data.encode('utf-8'), 'text/csv')
+    resp = resp.forms[0].submit()
+    assert 'CSV file contains lines with wrong number of columns.' in resp.text
+    assert '(line numbers 4, 5, 7)' in resp.text
+
+    csv_data += '\n' * 10
+    resp.forms[0]['file'] = Upload('test.csv', csv_data.encode('utf-8'), 'text/csv')
+    resp = resp.forms[0].submit()
+    assert 'CSV file contains lines with wrong number of columns.' in resp.text
+    assert '(line numbers 4, 5, 7, 8, 9 and more)' in resp.text
+
+
+def test_backoffice_cards_wscall_failure_display(http_requests, pub, studio):
+    LoggedError.wipe()
+    user = create_user(pub)
+
+    Workflow.wipe()
+    workflow = Workflow(name='wscall')
+    workflow.roles = {
+        '_viewer': 'Viewer',
+        '_editor': 'Editor',
+    }
+    st1 = workflow.add_status('Recorded', 'recorded')
+
+    wscall = WebserviceCallStatusItem()
+    wscall.id = '_wscall'
+    wscall.varname = 'xxx'
+    wscall.url = 'http://remote.example.net/xml'
+    wscall.action_on_bad_data = ':stop'
+    wscall.record_errors = True
+    st1.items.append(wscall)
+    wscall.parent = st1
+
+    again = ChoiceWorkflowStatusItem()
+    again.id = '_again'
+    again.label = 'Again'
+    again.by = ['_editor']
+    again.status = st1.id
+    st1.items.append(again)
+    again.parent = st1
+
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', type='string', varname='foo'),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.workflow_id = workflow.id
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.digest_template = 'card {{form_var_foo}}'
+    carddef.store()
+    carddef.data_class().wipe()
+
+    carddata = carddef.data_class()()
+    carddata.data = {'1': 'plop'}
+    carddata.just_created()
+    carddata.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get('/backoffice/data/foo/%s/' % carddata.id)
+    assert 'Again' in resp.text
+    resp = resp.forms[0].submit('button_again')
+    resp = resp.follow()
+    assert 'Error during webservice call' in resp.text
+
+    assert LoggedError.count() == 1
+    assert LoggedError.select()[0].get_formdata().data == {'1': 'plop'}
+
+
+def test_block_card_item_link(pub, studio, blocks_feature):
+    user = create_user(pub)
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', type='string', varname='foo'),
+    ]
+    carddef.backoffice_submission_roles = user.roles
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.digest_template = 'card {{form_var_foo}}'
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop'}
+    card.just_created()
+    card.store()
+
+    card2 = carddef.data_class()()
+    card2.data = {'1': 'plop2'}
+    card2.just_created()
+    card2.store()
+
+    BlockDef.wipe()
+    block = BlockDef()
+    block.name = 'foobar'
+    block.fields = [
+        fields.ItemField(
+            id='1', label='Test', type='item',
+            data_source={'type': 'carddef:foo', 'value': ''}),
+    ]
+    block.store()
+
+    formdef = FormDef()
+    formdef.name = 'bar'
+    formdef.fields = [
+        fields.BlockField(id='1', label='test', type='block:foobar', max_items=3),
+    ]
+    formdef.store()
+    formdef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get('/bar/')
+    resp.form['f1$element0$f1'].value = card.id
+    resp = resp.form.submit('f1$add_element')
+    resp.form['f1$element1$f1'].value = card2.id
+    resp = resp.form.submit('submit')  # -> validation page
+    assert resp.form['f1$element0$f1'].value == str(card.id)
+    assert resp.form['f1$element0$f1_label'].value == 'card plop'
+    assert resp.form['f1$element1$f1'].value == str(card2.id)
+    assert resp.form['f1$element1$f1_label'].value == 'card plop2'
+    resp = resp.form.submit('submit')  # -> final submit
+    resp = resp.follow()
+    assert '<div class="value">card plop</div>' in resp
+    assert '<div class="value">card plop2</div>' in resp
+
+    # check cards are links in backoffice
+    resp = app.get('/backoffice/management' + resp.request.path)
+    assert '<div class="value"><a href="http://example.net/backoffice/data/foo/%s/">card plop</a></div></div>' % card.id in resp
+    assert '<div class="value"><a href="http://example.net/backoffice/data/foo/%s/">card plop2</a></div></div>' % card2.id in resp
-- 
2.28.0

