From 8780d3169490feb5e255c4e869bb959387a7f8bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Mon, 18 Jul 2022 16:18:25 +0200
Subject: [PATCH 3/3] backoffice: add json import for cards (#60303)

---
 tests/api/test_carddef.py                     | 107 +++++++
 tests/backoffice_pages/test_carddata.py       | 264 +++++++++++++++++-
 wcs/api.py                                    |  43 ++-
 wcs/backoffice/data_management.py             | 131 +++++++--
 wcs/backoffice/management.py                  |  13 +-
 wcs/formdata.py                               |  56 +++-
 .../wcs/backoffice/card-data-import-form.html |  20 +-
 wcs/workflows.py                              |   1 +
 8 files changed, 563 insertions(+), 72 deletions(-)

diff --git a/tests/api/test_carddef.py b/tests/api/test_carddef.py
index 630d30586..9543f6c25 100644
--- a/tests/api/test_carddef.py
+++ b/tests/api/test_carddef.py
@@ -511,6 +511,113 @@ def test_cards_import_csv(pub, local_user, auth):
     assert resp.json['data']['creation_time'] <= resp.json['data']['completion_time']
 
 
+@pytest.mark.parametrize('auth', ['signature', 'http-basic'])
+def test_cards_import_json(pub, local_user, auth):
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+    local_user.roles = [role.id]
+    local_user.store()
+
+    ApiAccess.wipe()
+    access = ApiAccess()
+    access.name = 'test'
+    access.access_identifier = 'test'
+    access.access_key = '12345'
+    access.store()
+
+    app = get_app(pub)
+
+    if auth == 'http-basic':
+        access.roles = [role]
+        access.store()
+
+        def get_url(url, **kwargs):
+            app.set_authorization(('Basic', ('test', '12345')))
+            return app.get(url, **kwargs)
+
+        def put_url(url, *args, **kwargs):
+            app.set_authorization(('Basic', ('test', '12345')))
+            return app.put_json(url, *args, **kwargs)
+
+    else:
+
+        def get_url(url, **kwargs):
+            return app.get(
+                sign_uri(url, user=local_user, orig=access.access_identifier, key=access.access_key), **kwargs
+            )
+
+        def put_url(url, *args, **kwargs):
+            return app.put_json(
+                sign_uri(url, user=local_user, orig=access.access_identifier, key=access.access_key),
+                *args,
+                **kwargs,
+            )
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='0', label='foobar', varname='foo'),
+        fields.StringField(id='1', label='foobar2', varname='foo2'),
+    ]
+    carddef.workflow_roles = {'_viewer': role.id}
+    carddef.backoffice_submission_roles = [role.id]
+    carddef.digest_templates = {'default': 'bla {{ form_var_foo }} xxx'}
+    carddef.store()
+
+    carddef.data_class().wipe()
+
+    get_app(pub).get(sign_uri('/api/cards/test/import-json'), status=405)
+    get_app(pub).put(sign_uri('/api/cards/test/import-json'), status=403)
+    data = {
+        'data': [
+            {
+                'fields': {
+                    'foo': 'first entry',
+                    'foo2': 'plop',
+                }
+            },
+            {
+                'fields': {
+                    'foo': 'second entry',
+                    'foo2': 'plop',
+                }
+            },
+        ]
+    }
+    resp = put_url(
+        '/api/cards/test/import-json',
+        data,
+        headers={'content-type': 'application/json'},
+    )
+    assert resp.json == {'err': 0}
+    assert carddef.data_class().count() == 2
+    assert {x.data['0'] for x in carddef.data_class().select()} == {'first entry', 'second entry'}
+
+    # async mode
+    carddef.data_class().wipe()
+    assert carddef.data_class().count() == 0
+    resp = put_url(
+        '/api/cards/test/import-json?async=on',
+        data,
+        headers={'content-type': 'application/json'},
+    )
+    # afterjobs are not async in tests: job is already completed during request
+    assert carddef.data_class().count() == 2
+    assert {x.data['0'] for x in carddef.data_class().select()} == {'first entry', 'second entry'}
+    assert resp.json['err'] == 0
+    assert 'job' in resp.json['data']
+    job_id = resp.json['data']['job']['id']
+    assert AfterJob.get(job_id).status == 'completed'
+    # get job status from its api url
+    resp = get_url(resp.json['data']['job']['url'])
+    assert resp.json['err'] == 0
+    assert resp.json['data']['label'] == 'Importing data into cards'
+    assert resp.json['data']['status'] == 'completed'
+    assert resp.json['data']['creation_time'] <= resp.json['data']['completion_time']
+
+
 def test_cards_restricted_api(pub, local_user):
     pub.role_class.wipe()
     role = pub.role_class(name='test')
diff --git a/tests/backoffice_pages/test_carddata.py b/tests/backoffice_pages/test_carddata.py
index 65149c42e..a82de2900 100644
--- a/tests/backoffice_pages/test_carddata.py
+++ b/tests/backoffice_pages/test_carddata.py
@@ -1,4 +1,6 @@
+import base64
 import datetime
+import json
 import os
 import uuid
 
@@ -11,7 +13,7 @@ from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory
 from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
-from wcs.workflows import Workflow
+from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_superuser, create_user
@@ -372,24 +374,24 @@ def test_backoffice_cards_import_data_from_csv(pub):
     app = login(get_app(pub))
 
     resp = app.get(carddef.get_url())
-    assert 'Import data from a CSV file' not in resp.text
-    resp = app.get(carddef.get_url() + 'import-csv', status=403)
+    assert 'Import data from a file' not in resp.text
+    resp = app.get(carddef.get_url() + 'import-file', status=403)
 
     carddef.backoffice_submission_roles = user.roles
     carddef.store()
 
     resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
+    resp = resp.click('Import data from a file')
 
     assert 'Table, File are required but cannot be filled from CSV.' in resp
-    assert 'Download sample file for this card' not in resp
+    assert 'Download sample CSV file for this card' not in resp
     carddef.fields[0].required = False
     carddef.fields[7].required = False
     carddef.store()
 
     resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
-    sample_resp = resp.click('Download sample file for this card')
+    resp = resp.click('Import data from a file')
+    sample_resp = resp.click('Download sample CSV file for this card')
     today = datetime.date.today()
     assert sample_resp.text == (
         '"Table","Map","Test","Boolean","List","Date","File","Email","Long","List2","Items"\r\n'
@@ -481,7 +483,7 @@ def test_backoffice_cards_import_data_csv_user_support(pub):
         b'foobar,item2',
         b'foobar@mail.com,item2',
     ]
-    resp = app.get('/backoffice/data/test/import-csv')
+    resp = app.get('/backoffice/data/test/import-file')
     resp.forms[0]['file'] = Upload('test.csv', b'\n'.join(data), 'text/csv')
     resp = resp.forms[0].submit().follow()
     assert carddef.data_class().count() == 5
@@ -504,7 +506,7 @@ def test_backoffice_cards_import_data_csv_user_support(pub):
         b'foobar',
         b'foobar@mail.com',
     ]
-    resp = app.get('/backoffice/data/test/import-csv')
+    resp = app.get('/backoffice/data/test/import-file')
     resp.forms[0]['file'] = Upload('test.csv', b'\n'.join(data), 'text/csv')
     resp = resp.forms[0].submit().follow()
     assert carddef.data_class().count() == 4
@@ -529,7 +531,7 @@ def test_backoffice_cards_import_data_csv_invalid_columns(pub):
 
     app = login(get_app(pub))
     resp = app.get(carddef.get_url())
-    resp = resp.click('Import data from a CSV file')
+    resp = resp.click('Import data from a file')
 
     csv_data = '''String1,String2,Text
 1,2,3
@@ -551,6 +553,248 @@ def test_backoffice_cards_import_data_csv_invalid_columns(pub):
     assert '(line numbers 4, 5, 7, 8, 9 and more)' in resp.text
 
 
+def test_backoffice_cards_import_data_from_json(pub):
+    user = create_user(pub)
+
+    data_source = {
+        'type': 'formula',
+        'value': repr([{'id': '1', 'text': 'un', 'more': 'foo'}, {'id': '2', 'text': 'deux', 'more': 'bar'}]),
+    }
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+        fields.BoolField(id='2', label='Boolean', varname='bool'),
+        fields.ItemField(id='3', label='List', varname='item', data_source=data_source),
+        fields.FileField(id='4', label='File', varname='file'),
+        fields.DateField(id='5', label='Date', varname='date'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    data = {
+        'data': [
+            {
+                'fields': {
+                    'string': 'a string',
+                    'bool': True,
+                    'item': '1',
+                    'file': {
+                        'filename': 'test.txt',
+                        'content_type': 'application/pdf',
+                        'content': base64.encodebytes(b'%PDF-1.4 ...').decode(),
+                    },
+                    'date': '2022-07-19',
+                }
+            }
+        ]
+    }
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(data).encode(), 'application/json')
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+
+    carddata_export = carddef.data_class().select()[0].get_json_export_dict()
+    assert carddata_export['workflow']['status']['id'] == 'recorded'
+    assert carddata_export['fields'] == {
+        'string': 'a string',
+        'bool': True,
+        'item': 'un',
+        'item_raw': '1',
+        'item_structured': {'id': '1', 'more': 'foo', 'text': 'un'},
+        'file': {
+            'field_id': '4',
+            'filename': 'test.txt',
+            'content_type': 'application/pdf',
+            'content': base64.encodebytes(b'%PDF-1.4 ...').decode().strip(),
+            'url': 'http://example.net/api/cards/test/1/download?f=4',
+        },
+        'date': '2022-07-19',
+    }
+
+
+def test_backoffice_cards_import_data_with_no_varname_from_json(pub):
+    user = create_user(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    data = {
+        'data': [
+            {
+                'fields': {
+                    '_unnamed': {
+                        '1': 'a string',
+                    }
+                }
+            }
+        ]
+    }
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(data).encode(), 'application/json')
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+
+    carddata = carddef.data_class().select()[0]
+    assert carddata.data['1'] == 'a string'
+
+
+def test_backoffice_cards_import_status_from_json(pub):
+    user = create_user(pub)
+
+    workflow = CardDef.get_default_workflow()
+    workflow.id = None
+    st2 = workflow.add_status('status2')
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.workflow = workflow
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    data = {
+        'data': [
+            {
+                'fields': {
+                    'string': 'a string',
+                },
+                'workflow': {
+                    'status': {
+                        'id': 'wf-%s' % st2.id,
+                    }
+                },
+            }
+        ]
+    }
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(data).encode(), 'application/json')
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+
+    carddata = carddef.data_class().select()[0]
+    assert carddata.status == 'wf-%s' % st2.id
+
+
+def test_backoffice_cards_import_backoffice_fields_from_json(pub):
+    user = create_user(pub)
+
+    workflow = CardDef.get_default_workflow()
+    workflow.id = None
+    workflow.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(workflow)
+    workflow.backoffice_fields_formdef.fields = [
+        fields.StringField(id='bo1', label='bo field 1', varname='bo_data'),
+    ]
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.workflow = workflow
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    data = {
+        'data': [
+            {
+                'fields': {
+                    'string': 'a string',
+                },
+                'workflow': {
+                    'status': {
+                        'id': 'recorded',
+                    },
+                    'fields': {
+                        'bo_data': 'foo',
+                    },
+                },
+            }
+        ]
+    }
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(data).encode(), 'application/json')
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+
+    carddata = carddef.data_class().select()[0]
+    assert carddata.status == 'recorded'
+    assert carddata.data == {'1': 'a string', 'bo1': 'foo'}
+
+
+def test_backoffice_cards_import_user_from_json(pub):
+    user = create_user(pub)
+
+    user2 = pub.user_class(name='card import')
+    user2.email = 'card-import@example.org'
+    user2.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.backoffice_submission_roles = user.roles
+    carddef.user_support = 'optional'
+    carddef.store()
+    carddef.data_class().wipe()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    data = {
+        'data': [
+            {
+                'fields': {
+                    'string': 'a string',
+                },
+                'user': {
+                    'email': 'card-import@example.org',
+                },
+            }
+        ]
+    }
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(data).encode(), 'application/json')
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+
+    carddata = carddef.data_class().select()[0]
+    assert str(carddata.user_id) == str(user2.id)
+
+
 def test_backoffice_cards_wscall_failure_display(http_requests, pub):
     user = create_user(pub)
 
diff --git a/wcs/api.py b/wcs/api.py
index 778a80035..83fa771d8 100644
--- a/wcs/api.py
+++ b/wcs/api.py
@@ -74,6 +74,10 @@ def posted_json_data_to_formdata_data(formdef, data):
         if field.store_structured_value and structured in data:
             data['%s_structured' % field.id] = data.pop(structured)
 
+    # merge unnamed fields if they exist
+    if '_unnamed' in data:
+        data.update(data.pop('_unnamed'))
+
     # create a temporary formdata so datasources using previous fields in
     # parameters can find their values.
     transient_formdata = formdef.data_class()()
@@ -282,6 +286,7 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
     _q_exports = [  # restricted to API endpoints
         ('list', 'json'),
         ('import-csv', 'import_csv'),
+        ('import-json', 'import_json'),
         'geojson',
         'ods',
         ('@schema', 'schema'),
@@ -344,18 +349,7 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
         formdata.data = posted_json_data_to_formdata_data(self.formdef, data)
 
         if 'user' in json_input:
-            formdata_user = None
-            for name_id in json_input['user'].get('NameID') or []:
-                formdata_user = get_publisher().user_class.get_users_with_name_identifier(name_id)
-                if formdata_user:
-                    break
-            else:
-                if json_input['user'].get('email'):
-                    formdata_user = get_publisher().user_class.get_users_with_email(
-                        json_input['user'].get('email')
-                    )
-            if formdata_user:
-                formdata.user_id = formdata_user[0].id
+            formdata.set_user_from_json(json_input['user'])
         elif user and not user.is_api_user:
             formdata.user_id = user.id
 
@@ -377,6 +371,12 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
         )
 
     def import_csv(self):
+        return self.import_file('csv')
+
+    def import_json(self):
+        return self.import_file('json')
+
+    def import_file(self, file_format):
         if get_request().get_method() != 'PUT':
             raise MethodNotAllowedError(allowed_methods=['PUT'])
         get_request()._user = get_user_from_api_query_string()
@@ -386,7 +386,11 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
         afterjob = bool(get_request().form.get('async') == 'on')
         get_response().set_content_type('application/json')
         try:
-            job = self.import_csv_submit(get_request().stdin, afterjob=afterjob, api=True)
+            if file_format == 'csv':
+                content = get_request().stdin.read()
+                job = self.import_csv_submit(content, afterjob=afterjob, api=True)
+            elif file_format == 'json':
+                job = self.import_json_submit(get_request().json, afterjob=afterjob, api=True)
         except ValueError as e:
             return json.dumps({'err': 1, 'err_desc': str(e)})
         if job is None:
@@ -602,18 +606,7 @@ class ApiFormdefDirectory(Directory):
             formdata.backoffice_submission = True
         if not meta.get('backoffice-submission'):
             if 'user' in json_input:
-                formdata_user = None
-                for name_id in json_input['user'].get('NameID') or []:
-                    formdata_user = get_publisher().user_class.get_users_with_name_identifier(name_id)
-                    if formdata_user:
-                        break
-                else:
-                    if json_input['user'].get('email'):
-                        formdata_user = get_publisher().user_class.get_users_with_email(
-                            json_input['user'].get('email')
-                        )
-                if formdata_user:
-                    formdata.user_id = formdata_user[0].id
+                formdata.set_user_from_json(json_input['user'])
             elif user and not user.is_api_user:
                 formdata.user_id = user.id
 
diff --git a/wcs/backoffice/data_management.py b/wcs/backoffice/data_management.py
index 1ff46a9f0..280108ea0 100644
--- a/wcs/backoffice/data_management.py
+++ b/wcs/backoffice/data_management.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import copy
 import csv
 import datetime
 import io
@@ -25,8 +26,9 @@ from quixote.html import htmltext
 from wcs import fields
 from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory
+from wcs.workflows import ActionsTracingEvolutionPart
 
-from ..qommon import _, errors, template
+from ..qommon import _, errors, ngettext, template
 from ..qommon.afterjobs import AfterJob
 from ..qommon.backoffice.menu import html_top
 from ..qommon.form import FileWidget, Form
@@ -115,7 +117,7 @@ class CardPage(FormPage):
         ('export-spreadsheet', 'export_spreadsheet'),
         ('save-view', 'save_view'),
         ('delete-view', 'delete_view'),
-        ('import-csv', 'import_csv'),
+        ('import-file', 'import_file'),
         ('filter-options', 'filter_options'),
         ('data-sample-csv', 'data_sample_csv'),
     ]
@@ -156,9 +158,7 @@ class CardPage(FormPage):
     def get_formdata_sidebar_actions(self, qs=''):
         r = super().get_formdata_sidebar_actions(qs=qs)
         if self.formdef.can_user_add_cards(get_request().user):
-            r += htmltext('<li><a rel="popup" href="import-csv">%s</a></li>') % _(
-                'Import data from a CSV file'
-            )
+            r += htmltext('<li><a rel="popup" href="import-file">%s</a></li>') % _('Import data from a file')
         return r
 
     def data_sample_csv(self):
@@ -195,7 +195,7 @@ class CardPage(FormPage):
         )
         return output.getvalue()
 
-    def import_csv(self):
+    def import_file(self):
         if not self.formdef.can_user_add_cards(get_request().user):
             raise errors.AccessForbiddenError()
         context = {'required_fields': []}
@@ -208,30 +208,52 @@ class CardPage(FormPage):
             return redirect('.')
 
         if form.is_submitted() and not form.has_errors():
+            file_content = form.get_widget('file').parse().fp.read()
             try:
-                return self.import_csv_submit(form.get_widget('file').parse().fp)
-            except ValueError as e:
-                form.set_error('file', e)
+                json_content = json.loads(file_content)
+            except ValueError:
+                # not json -> CSV
+                try:
+                    return self.import_csv_submit(file_content)
+                except ValueError as e:
+                    form.set_error('file', e)
+            else:
+                try:
+                    return self.import_json_submit(json_content)
+                except ValueError as e:
+                    form.set_error('file', e)
 
-        get_response().breadcrumb.append(('import_csv', _('Import CSV')))
-        html_top('data_management', _('Import CSV'))
+        get_response().breadcrumb.append(('import-file', _('Import File')))
+        html_top('data_management', _('Import File'))
         context['html_form'] = form
+        context['impossible_csv_fields'] = self.get_csv_impossible_fields()
+
+        return template.QommonTemplateResponse(
+            templates=['wcs/backoffice/card-data-import-form.html'], context=context
+        )
 
+    def get_csv_impossible_fields(self):
+        impossible_fields = []
         for field in get_import_csv_fields(self.formdef):
             if not hasattr(field, 'required'):
                 continue
             if field.required and field.convert_value_from_str is None:
-                context['required_fields'].append(field.label)
+                impossible_fields.append(field.label)
+        return impossible_fields
 
-        return template.QommonTemplateResponse(
-            templates=['wcs/backoffice/card-data-import-form.html'], context=context
-        )
-
-    def import_csv_submit(self, fd, afterjob=True, api=False):
-        content = fd.read()
+    def import_csv_submit(self, content, afterjob=True, api=False):
         if b'\0' in content:
             raise ValueError(_('Invalid file format.'))
 
+        impossible_fields = self.get_csv_impossible_fields()
+        if impossible_fields:
+            error = ngettext(
+                '%s is required but cannot be filled from CSV.',
+                '%s are required but cannot be filled from CSV.',
+                len(impossible_fields),
+            ) % ', '.join(impossible_fields)
+            raise ValueError(error)
+
         for charset in ('utf-8', 'iso-8859-15'):
             try:
                 content = content.decode(charset)
@@ -280,6 +302,20 @@ class CardPage(FormPage):
         else:
             job.execute()
 
+    def import_json_submit(self, json_content, afterjob=True, api=False):
+        # basic check, looks like valid json card content?
+        if not isinstance(json_content, dict) or 'data' not in json_content:
+            raise ValueError(_('Invalid JSON file'))
+
+        job = ImportFromJsonAfterJob(carddef=self.formdef, json_content=json_content)
+        if afterjob:
+            get_response().add_after_job(job)
+            if api:
+                return job
+            return redirect(job.get_processing_url())
+        else:
+            job.execute()
+
     def _q_lookup(self, component):
 
         if not self.view:
@@ -408,3 +444,62 @@ class ImportFromCsvAfterJob(AfterJob):
 
     def done_button_attributes(self):
         return {'data-redirect-auto': 'true'}
+
+
+class ImportFromJsonAfterJob(AfterJob):
+    def __init__(self, carddef, json_content):
+        super().__init__(
+            label=_('Importing data into cards'),
+            carddef_class=carddef.__class__,
+            carddef_id=carddef.id,
+            json_content=json_content,
+        )
+
+    @property
+    def carddef(self):
+        return self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])
+
+    def execute(self):
+        json_content = self.kwargs['json_content']
+        from wcs.api import posted_json_data_to_formdata_data
+
+        for json_data in json_content['data']:
+            json_data = copy.deepcopy(json_data)
+            carddata = self.carddef.data_class()()
+            carddata.data = posted_json_data_to_formdata_data(self.carddef, json_data['fields'])
+            if 'user' in json_data:
+                carddata.set_user_from_json(json_data['user'])
+
+            carddata.just_created()
+
+            if 'workflow' not in json_data:
+                # perform as new
+                carddata.store()
+
+                get_publisher().substitutions.reset()
+                get_publisher().substitutions.feed(get_publisher())
+                get_publisher().substitutions.feed(carddata)
+                carddata.perform_workflow(event='json-import-created')
+            else:
+                if 'fields' in json_data['workflow']:
+                    backoffice_data_dict = posted_json_data_to_formdata_data(
+                        self.carddef, json_data['workflow']['fields']
+                    )
+                    carddata.data.update(backoffice_data_dict)
+
+                status = json_data['workflow'].get('real_status') or json_data['workflow'].get('status')
+                carddata.status = status.get('id')
+                carddata.evolution[-1].status = status
+                carddata.evolution[-1].add_part(ActionsTracingEvolutionPart('json-import-created', []))
+                carddata.store()
+
+            self.increment_count()
+
+    def done_action_url(self):
+        return self.carddef.get_url()
+
+    def done_action_label(self):
+        return _('Back to Listing')
+
+    def done_button_attributes(self):
+        return {'data-redirect-auto': 'true'}
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index 5cc136b71..efe2875a4 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -2417,6 +2417,7 @@ class FormPage(Directory):
                 include_evolution=True,
                 include_files=False,
                 include_roles=True,
+                include_unnamed_fields=False,
             )
         else:
             output = [
@@ -4208,7 +4209,15 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
         self.file_name = '%s.json' % formdef.url_name
 
     def create_json_export(
-        self, items, user, anonymise, digest_key, include_evolution, include_files, include_roles
+        self,
+        items,
+        user,
+        anonymise,
+        digest_key,
+        include_evolution,
+        include_files,
+        include_roles,
+        include_unnamed_fields,
     ):
         formdef = self.kwargs['formdef_class'].get(self.kwargs['formdef_id'])
         prefetched_users = None
@@ -4244,6 +4253,7 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
                 include_evolution=include_evolution,
                 include_files=include_files,
                 include_roles=include_roles,
+                include_unnamed_fields=include_unnamed_fields,
             )
             data.pop('digests')
             if digest_key:
@@ -4263,6 +4273,7 @@ class JsonFileExportAfterJob(CsvExportAfterJob):
                     include_evolution=False,
                     include_files=True,
                     include_roles=False,
+                    include_unnamed_fields=True,
                 )
             },
             indent=2,
diff --git a/wcs/formdata.py b/wcs/formdata.py
index 22fa1114a..ee734107a 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -390,6 +390,18 @@ class FormData(StorableObject):
 
     user = property(get_user, set_user)
 
+    def set_user_from_json(self, json_user):
+        formdata_user = None
+        for name_id in json_user.get('NameID') or []:
+            formdata_user = get_publisher().user_class.get_users_with_name_identifier(name_id)
+            if formdata_user:
+                break
+        else:
+            if json_user.get('email'):
+                formdata_user = get_publisher().user_class.get_users_with_email(json_user.get('email'))
+        if formdata_user:
+            self.user_id = formdata_user[0].id
+
     def get_user_label(self):
         user = self.user
         if user:
@@ -1263,13 +1275,15 @@ class FormData(StorableObject):
         return None
 
     @classmethod
-    def get_json_data_dict(cls, data, fields, formdata=None, include_files=True, anonymise=False):
+    def get_json_data_dict(
+        cls, data, fields, formdata=None, include_files=True, anonymise=False, include_unnamed_fields=False
+    ):
         new_data = {}
         seen = set()
         for field in fields:
             if anonymise and field.anonymise:
                 continue
-            if not field.varname:  # exports only named fields
+            if not field.varname and not include_unnamed_fields:
                 continue
             if field.varname in seen:
                 # skip fields with a varname that is used by another non-empty
@@ -1281,21 +1295,38 @@ class FormData(StorableObject):
                     value = field.get_json_value(value, formdata=formdata, include_file_content=include_files)
             else:
                 value = None
-            if value:
+
+            if value and field.varname:
                 seen.add(field.varname)
+
+            if not field.varname:
+                # include unnamed fields in a dedicated key
+                if '_unnamed' not in new_data:
+                    new_data['_unnamed'] = {}
+                store_dict = new_data['_unnamed']
+                store_key = str(field.id)
+            else:
+                store_dict = new_data
+                store_key = field.varname
+
             if field.store_display_value:
-                new_data[field.varname + '_raw'] = value
-                new_data[field.varname] = data.get('%s_display' % field.id)
+                store_dict[store_key + '_raw'] = value
+                store_dict[store_key] = data.get('%s_display' % field.id)
             else:
-                new_data[field.varname] = value
+                store_dict[store_key] = value
             if field.store_structured_value:
                 if data.get('%s_structured' % field.id):
-                    new_data[field.varname + '_structured'] = data.get('%s_structured' % field.id)
+                    store_dict[store_key + '_structured'] = data.get('%s_structured' % field.id)
         return new_data
 
-    def get_json_dict(self, fields, include_files=True, anonymise=False):
+    def get_json_dict(self, fields, include_files=True, anonymise=False, include_unnamed_fields=False):
         return self.get_json_data_dict(
-            self.data, fields, formdata=self, include_files=include_files, anonymise=anonymise
+            self.data,
+            fields,
+            formdata=self,
+            include_files=include_files,
+            anonymise=anonymise,
+            include_unnamed_fields=include_unnamed_fields,
         )
 
     def get_json_export_dict(
@@ -1308,6 +1339,7 @@ class FormData(StorableObject):
         prefetched_roles=None,
         include_evolution=True,
         include_roles=True,
+        include_unnamed_fields=False,
     ):
         data = {}
         data['id'] = str(self.id)
@@ -1333,7 +1365,10 @@ class FormData(StorableObject):
             data['user'] = user.get_json_export_dict()
 
         data['fields'] = self.get_json_dict(
-            self.formdef.fields, include_files=include_files, anonymise=anonymise
+            self.formdef.fields,
+            include_files=include_files,
+            anonymise=anonymise,
+            include_unnamed_fields=include_unnamed_fields,
         )
 
         data['workflow'] = {}
@@ -1351,6 +1386,7 @@ class FormData(StorableObject):
                 self.formdef.workflow.get_backoffice_fields(),
                 include_files=include_files,
                 anonymise=anonymise,
+                include_unnamed_fields=include_unnamed_fields,
             )
 
         if include_roles:
diff --git a/wcs/templates/wcs/backoffice/card-data-import-form.html b/wcs/templates/wcs/backoffice/card-data-import-form.html
index bbe89118d..78e6dbba2 100644
--- a/wcs/templates/wcs/backoffice/card-data-import-form.html
+++ b/wcs/templates/wcs/backoffice/card-data-import-form.html
@@ -3,25 +3,29 @@
 
 {% block appbar %}
 <div id="appbar">
-<h2>{% trans "Import CSV" %}</h2>
+<h2>{% trans "Import File" %}</h2>
 </div>
 {% endblock %}
 
 {% block content %}
 
-{% if required_fields %}
-<div class="errornotice">
+<p>{% trans "You can add data to this card by uploading a file." %}</p>
+
+{% if not impossible_csv_fields %}
+<p><a href="data-sample-csv">{% trans "Download sample CSV file for this card" %}</a></p>
+{% else %}
+<div class="warningnotice">
   <p>
-  {% blocktrans count fields=required_fields|length with labels=required_fields|join:", " %}
+  {% blocktrans count fields=impossible_csv_fields|length with labels=impossible_csv_fields|join:", " %}
   {{ labels }} is required but cannot be filled from CSV.
   {% plural %}
   {{ labels }} are required but cannot be filled from CSV.
   {% endblocktrans %}
+  {% trans "Only JSON files can be imported." %}
   </p>
 </div>
-{% else %}
-<p>{% trans "You can add data to this card by uploading a file." %}</p>
-<p><a href="data-sample-csv">{% trans "Download sample file for this card" %}</a></p>
-{{ html_form.render|safe }}
 {% endif %}
+
+{{ html_form.render|safe }}
+
 {% endblock %}
diff --git a/wcs/workflows.py b/wcs/workflows.py
index 274efaad2..a83a0caea 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -405,6 +405,7 @@ class ActionsTracingEvolutionPart(EvolutionPart):
             'global-action-timeout': _('Global action timeout'),
             'global-api-trigger': _('API Trigger'),
             'global-external-workflow': _('Trigger by external workflow'),
+            'json-import-created': _('Created (by JSON import)'),
             'timeout-jump': _('Timeout jump'),
             'workflow-created': _('Created (by workflow action)'),
             'workflow-form-submit': _('Action in workflow form'),
-- 
2.36.1

