From 1569fd8e3af3d4ef2894d34ba85c2882c66414e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Sun, 29 Nov 2020 17:50:37 +0100
Subject: [PATCH] general: allow assigning complex types from rendered
 templates (#41847)

---
 tests/test_workflows.py     | 196 ++++++++++++++++++++++++++++++++++--
 wcs/fields.py               |  14 ++-
 wcs/publisher.py            |  35 +++++++
 wcs/qommon/template.py      |   6 +-
 wcs/variables.py            |  11 +-
 wcs/wf/backoffice_fields.py |  20 +++-
 wcs/wf/create_formdata.py   |  20 +++-
 wcs/workflows.py            |   3 +-
 8 files changed, 279 insertions(+), 26 deletions(-)

diff --git a/tests/test_workflows.py b/tests/test_workflows.py
index 4aa156843..dca35a0ad 100644
--- a/tests/test_workflows.py
+++ b/tests/test_workflows.py
@@ -23,14 +23,16 @@ from quixote import cleanup, get_response
 from wcs.qommon.errors import ConnectionError
 from quixote.http_request import Upload as QuixoteUpload
 from wcs.qommon.http_request import HTTPRequest
+from wcs.qommon.form import PicklableUpload
 from wcs.qommon.form import *
 
+from wcs.blocks import BlockDef
 from wcs.formdef import FormDef
 from wcs.carddef import CardDef
 from wcs import sessions
 from wcs.fields import (StringField, DateField, MapField, FileField, ItemField,
         ItemsField, CommentField, EmailField, PageField, TitleField,
-        SubtitleField, TextField, BoolField, TableField)
+        SubtitleField, TextField, BoolField, TableField, BlockField)
 from wcs.formdata import Evolution
 from wcs.logged_errors import LoggedError
 from wcs.roles import Role
@@ -3880,6 +3882,7 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
     wf.backoffice_fields_formdef.fields = [
         FileField(id='bo1', label='1st backoffice field',
             type='file', varname='backoffice_file'),
+        StringField(id='bo2', label='2nd backoffice field', type='string'),
     ]
     st1 = wf.add_status('Status1')
     wf.store()
@@ -3940,7 +3943,50 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
     assert formdata.data['bo1'].content_type == 'application/vnd.oasis.opendocument.text'
     assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'template.odt'), 'rb').read()
 
-    # check striping metadata
+    # check with template string
+    formdata = formdef.data_class()()
+    formdata.data = {'00': upload}
+    formdata.just_created()
+    formdata.store()
+
+    two_pubs.substitutions.feed(formdata)
+    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file_raw}}'}]
+    item.perform(formdata)
+
+    assert formdata.data['bo1'].base_filename == 'test.jpeg'
+    assert formdata.data['bo1'].content_type == 'image/jpeg'
+    assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
+
+    # check with template string, without _raw
+    formdata = formdef.data_class()()
+    formdata.data = {'00': upload}
+    formdata.just_created()
+    formdata.store()
+
+    two_pubs.substitutions.feed(formdata)
+    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_file}}'}]
+    item.perform(formdata)
+
+    assert formdata.data['bo1'].base_filename == 'test.jpeg'
+    assert formdata.data['bo1'].content_type == 'image/jpeg'
+    assert formdata.data['bo1'].get_content() == open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb').read()
+
+    # check with a template string, into a string field
+    two_pubs.substitutions.feed(formdata)
+    item.fields = [{'field_id': 'bo2', 'value': '{{form_var_file}}'}]
+    item.perform(formdata)
+
+    assert formdata.data['bo2'] == 'test.jpeg'
+
+    # check with template string and missing file
+    formdata = formdef.data_class()()
+    formdata.data = {'00': None}
+    formdata.just_created()
+    formdata.store()
+
+    assert formdata.data.get('bo1') is None
+
+    # check stripping metadata
     two_pubs.substitutions.feed(formdata)
     item.fields = [{'field_id': 'bo1',
                     'value': '=utils.attachment(form_var_file_raw,' +
@@ -4028,7 +4074,7 @@ def test_set_backoffice_field_file(http_requests, two_pubs):
 
     hello_world = formdata.data['bo1']
     # check wrong value
-    for value in ('="HELLO"', 'BAD', '={}', '=[]'):
+    for value in ('="HELLO"', 'BAD'):
         formdata.data['bo1'] = hello_world
         formdata.store()
 
@@ -4339,6 +4385,44 @@ def test_set_backoffice_field_items(two_pubs):
     assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
     assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']
 
+    # from formdata field
+    formdef.fields = [
+            ItemsField(id='1', label='field', type='items', varname='items', data_source=datasource),
+    ]
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': ['a', 'c']}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
+    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
+    formdata.just_created()
+    formdata.store()
+    two_pubs.substitutions.feed(formdata)
+
+    item = SetBackofficeFieldsWorkflowStatusItem()
+    item.parent = st1
+    item.fields = [{'field_id': 'bo1', 'value': "=form_var_items_raw"}]
+    item.perform(formdata)
+
+    assert formdata.data['bo1'] == ['a', 'c']
+    assert formdata.data['bo1_display'] == 'aa, cc'
+    assert len(formdata.data['bo1_structured']) == 2
+    assert {'id': 'a', 'more': 'aaa', 'text': 'aa'} in formdata.data['bo1_structured']
+    assert {'id': 'c', 'more': 'ccc', 'text': 'cc'} in formdata.data['bo1_structured']
+
+    # with a template
+    formdata = formdef.data_class()()
+    formdata.data = {'1': ['a', 'c']}
+    formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
+    formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
+    formdata.just_created()
+    formdata.store()
+    two_pubs.substitutions.reset()
+    two_pubs.substitutions.feed(formdata)
+
+    item.fields = [{'field_id': 'bo1', 'value': "{{form_var_items_raw}}"}]
+    item.perform(formdata)
+
 
 def test_set_backoffice_field_date(two_pubs):
     Workflow.wipe()
@@ -4460,6 +4544,89 @@ def test_set_backoffice_field_boolean(two_pubs):
         formdata.store()
 
 
+def test_set_backoffice_field_block(two_pubs, blocks_feature):
+    BlockDef.wipe()
+    Workflow.wipe()
+    FormDef.wipe()
+    LoggedError.wipe()
+
+    block = BlockDef()
+    block.name = 'foobar'
+    block.digest_template = 'X{{foobar_var_foo}}Y'
+    block.fields = [
+        StringField(id='123', required=True, label='Test',
+            type='string', varname='foo'),
+        StringField(id='234', required=True, label='Test2',
+            type='string', varname='bar'),
+    ]
+    block.store()
+
+    wf = Workflow(name='xxx')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    st1 = wf.add_status('Status1')
+    wf.backoffice_fields_formdef.fields = [
+        BlockField(id='bo1', label='1st backoffice field', type='block:foobar'),
+        StringField(id='bo2', label='2nd backoffice field', type='string'),
+    ]
+    wf.store()
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        BlockField(id='1', label='test', type='block:foobar', max_items=3, varname='foo'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    # value from test_block_digest in tests/test_form_pages.py
+    formdata.data = {
+        '1': {
+            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
+            'schema': {'123': 'string', '234': 'string'}
+            },
+        '1_display': 'XfooY, Xfoo2Y',
+    }
+    formdata.just_created()
+    formdata.store()
+    get_publisher().substitutions.feed(formdata)
+
+    item = SetBackofficeFieldsWorkflowStatusItem()
+    item.parent = st1
+    item.fields = [{'field_id': 'bo1', 'value': '{{form_var_foo_raw}}'}]
+    item.perform(formdata)
+    formdata = formdef.data_class().get(formdata.id)
+    assert formdata.data['bo1'] == formdata.data['1']
+    assert formdata.data['bo1_display'] == formdata.data['1_display']
+
+    # without _raw suffix
+    formdata = formdef.data_class()()
+    # value from test_block_digest in tests/test_form_pages.py
+    formdata.data = {
+        '1': {
+            'data': [{'123': 'foo', '234': 'bar'}, {'123': 'foo2', '234': 'bar2'}],
+            'schema': {'123': 'string', '234': 'string'}
+            },
+        '1_display': 'XfooY, Xfoo2Y',
+    }
+    formdata.just_created()
+    formdata.store()
+    get_publisher().substitutions.reset()
+    get_publisher().substitutions.feed(formdata)
+
+    item = SetBackofficeFieldsWorkflowStatusItem()
+    item.parent = st1
+    item.fields = [
+        {'field_id': 'bo1', 'value': '{{form_var_foo}}'},
+        {'field_id': 'bo2', 'value': '{{form_var_foo}}'},
+    ]
+    item.perform(formdata)
+    formdata = formdef.data_class().get(formdata.id)
+    assert formdata.data['bo1'] == formdata.data['1']
+    assert formdata.data['bo1_display'] == formdata.data['1_display']
+    assert formdata.data['bo2'] == formdata.data['1_display']
+
+
 def test_set_backoffice_field_immediate_use(http_requests, two_pubs):
     Workflow.wipe()
     FormDef.wipe()
@@ -4902,7 +5069,8 @@ def test_create_carddata(pub):
         StringField(id='1', label='string'),
         ItemField(id='2', label='List', items=['item1', 'item2'],
                   varname='clist'),
-        DateField(id='3', label='Date', varname='cdate')
+        DateField(id='3', label='Date', varname='cdate'),
+        FileField(id='4', label='File', varname='cfile'),
     ]
     carddef.store()
 
@@ -4917,6 +5085,7 @@ def test_create_carddata(pub):
         Mapping(field_id='1', expression='=form_var_undefined'),
         Mapping(field_id='2', expression='{{ form_var_list }}'),
         Mapping(field_id='3', expression='{{ form_var_date }}'),
+        Mapping(field_id='4', expression='{{ form_var_file|default_if_none:"" }}'),
     ]
     create.parent = wf.possible_status[1]
     wf.possible_status[1].items.insert(0, create)
@@ -4925,9 +5094,9 @@ def test_create_carddata(pub):
     formdef = FormDef()
     formdef.name = 'source form'
     formdef.fields = [
-        ItemField(id='1', label='List', items=['item1', 'item2'],
-                  varname='list'),
-        DateField(id='2', label='Date', varname='date')
+        ItemField(id='1', label='List', items=['item1', 'item2'], varname='list'),
+        DateField(id='2', label='Date', varname='date'),
+        FileField(id='3', label='File', varname='file'),
     ]
     formdef.workflow_id = wf.id
     formdef.store()
@@ -4954,16 +5123,23 @@ def test_create_carddata(pub):
     formdata = formdef.data_class()()
     today = datetime.date.today()
 
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    with open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg'), 'rb') as jpg:
+        upload.receive([jpg.read()])
+
     formdata.data = {'1': 'item1',
                      '1_display': 'item1',
-                     '2': today}
+                     '2': today,
+                     '3': upload}
     formdata.just_created()
     formdata.perform_workflow()
 
     assert formdata.get_substitution_variables()['form_links_mycard_form_number'] == '1-2'
     carddata = carddef.data_class().get(id=2)
-    assert carddata.get_substitution_variables()['form_var_clist'] == 'item1'
-    assert carddata.get_substitution_variables()['form_var_cdate'] == today
+    assert carddata.data['2'] == 'item1'
+    assert carddata.data['2_display'] == 'item1'
+    assert carddata.data['3'] == today.timetuple()
+    assert carddata.data['4'].base_filename == 'test.jpeg'
 
     create.condition = {'type': 'python', 'value': '1 == 2'}
     wf.store()
diff --git a/wcs/fields.py b/wcs/fields.py
index 23c89a867..bf99c11c3 100644
--- a/wcs/fields.py
+++ b/wcs/fields.py
@@ -176,6 +176,7 @@ class Field(object):
     convert_value_from_str = None
     convert_value_to_str = None
     convert_value_from_anything = None
+    allow_complex = False
     display_locations = []
     prefill = None
     store_display_value = None
@@ -967,6 +968,7 @@ register_field_class(EmailField)
 class BoolField(WidgetField):
     key = 'bool'
     description = N_('Check Box (single choice)')
+    allow_complex = True
 
     widget_class = CheckboxWidget
     required = False
@@ -1062,6 +1064,8 @@ register_field_class(BoolField)
 class FileField(WidgetField):
     key = 'file'
     description = N_('File Upload')
+    allow_complex = True
+
     document_type = None
     max_file_size = None
     automatic_image_resize = False
@@ -1118,7 +1122,7 @@ class FileField(WidgetField):
 
     @classmethod
     def convert_value_from_anything(cls, value):
-        if value is None:
+        if not value:
             return None
         from wcs.variables import LazyFieldVarFile
         if isinstance(value, LazyFieldVarFile):
@@ -1435,6 +1439,7 @@ def item_items_stats(field, values):
 class ItemField(WidgetField):
     key = 'item'
     description = N_('List')
+    allow_complex = True
 
     items = []
     show_as_radio = None
@@ -1728,6 +1733,7 @@ register_field_class(ItemField)
 class ItemsField(WidgetField):
     key = 'items'
     description = N_('Multiple choice list')
+    allow_complex = True
 
     items = []
     max_choices = 0
@@ -2095,6 +2101,7 @@ register_field_class(PageField)
 class TableField(WidgetField):
     key = 'table'
     description = N_('Table')
+    allow_complex = True
 
     rows = None
     columns = None
@@ -2245,6 +2252,7 @@ register_field_class(TableField)
 class TableSelectField(TableField):
     key = 'table-select'
     description = N_('Table of Lists')
+    allow_complex = True
 
     items = None
 
@@ -2286,6 +2294,7 @@ register_field_class(TableSelectField)
 class TableRowsField(WidgetField):
     key = 'tablerows'
     description = N_('Table with rows')
+    allow_complex = True
 
     total_row = True
     columns = None
@@ -2529,6 +2538,7 @@ register_field_class(MapField)
 class RankedItemsField(WidgetField):
     key = 'ranked-items'
     description = N_('Ranked Items')
+    allow_complex = True
 
     items = []
     randomize_items = False
@@ -2680,6 +2690,8 @@ register_field_class(PasswordField)
 
 class BlockField(WidgetField):
     key = 'block'
+    allow_complex = True
+
     widget_class = BlockWidget
     max_items = 1
     extra_attributes = ['block', 'max_items', 'add_element_label', 'label_display']
diff --git a/wcs/publisher.py b/wcs/publisher.py
index e5e6c8069..d05163342 100644
--- a/wcs/publisher.py
+++ b/wcs/publisher.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+from contextlib import contextmanager
 import json
 import os
 import random
@@ -92,6 +93,8 @@ class WcsPublisher(StubWcsPublisher):
     tracking_code_class = TrackingCode
     unpickler_class = UnpicklerClass
 
+    complex_data_cache = None
+
     @classmethod
     def get_backoffice_module(cls):
         import backoffice
@@ -375,6 +378,38 @@ class WcsPublisher(StubWcsPublisher):
             from . import sql
             sql.cleanup_connection()
 
+    @contextmanager
+    def complex_data(self):
+        self.complex_data_cache = {}
+        try:
+            yield True
+        finally:
+            self.complex_data_cache = None
+
+    def cache_complex_data(self, value, str_value=None):
+        # Keep a temporary cache of assocations between a complex data value
+        # (value) and a string reprensentation (str(value) or the dedicated
+        # str_value parameter).
+        #
+        # It ensures string values are unique by appending a private unicode
+        # code point, that will be removed in wcs/qommon/template.py.
+
+        if self.complex_data_cache is None:
+            # it doesn't do anything unless initialized.
+            return str_value or value
+
+        if str_value is None:
+            str_value = str(value)
+        str_value += chr(0xE000 + len(self.complex_data_cache))
+        self.complex_data_cache[str_value] = value
+        return str_value
+
+    def get_cached_complex_data(self, value):
+        if not isinstance(value, str):
+            return value
+        return (self.complex_data_cache or {}).get(value)
+
+
 set_publisher_class(WcsPublisher)
 WcsPublisher.register_extra_dir(os.path.join(os.path.dirname(__file__), 'extra'))
 
diff --git a/wcs/qommon/template.py b/wcs/qommon/template.py
index 77060ed98..a07a0be9f 100644
--- a/wcs/qommon/template.py
+++ b/wcs/qommon/template.py
@@ -16,6 +16,7 @@
 
 import os
 import glob
+import re
 import xml.etree.ElementTree as ET
 
 import django.template
@@ -505,7 +506,10 @@ class Template(object):
                 raise TemplateError(_('failure to render Django template: %s'), e)
             else:
                 return self.value
-        return force_str(rendered)
+        rendered = str(rendered)
+        if context.get('allow_complex'):
+            return rendered
+        return re.sub(r'[\uE000-\uF8FF]', '', rendered)
 
     def ezt_render(self, context={}):
         fd = StringIO()
diff --git a/wcs/variables.py b/wcs/variables.py
index 0667d42a7..1f2d705b3 100644
--- a/wcs/variables.py
+++ b/wcs/variables.py
@@ -604,7 +604,8 @@ class LazyFieldVar(object):
     @property
     def raw(self):
         if self._field.store_display_value or self._field.key in ('file', 'date'):
-            return self._data.get(self._field.id)
+            raw_value = self._data.get(self._field.id)
+            return get_publisher().cache_complex_data(raw_value)
         raise AttributeError('raw')
 
     def get_value(self):
@@ -620,7 +621,7 @@ class LazyFieldVar(object):
     def __str__(self):
         value = self.get_value()
         if not isinstance(value, six.string_types):
-            value = str(value)
+            value = get_publisher().cache_complex_data(value)
         return force_str(value)
 
     def __nonzero__(self):
@@ -913,7 +914,11 @@ class LazyFieldVarBlock(LazyFieldVar):
 
     def get_value(self):
         # don't give access to underlying data dictionary.
-        return self._data.get('%s_display' % self._field.id, '---')
+        value = self._data.get(str(self._field.id))
+        str_value = self._data.get('%s_display' % self._field.id, '---')
+        if not value:
+            return str_value
+        return get_publisher().cache_complex_data(value, str_value)
 
     def __getitem__(self, key):
         try:
diff --git a/wcs/wf/backoffice_fields.py b/wcs/wf/backoffice_fields.py
index 82215d390..2b9ee7848 100644
--- a/wcs/wf/backoffice_fields.py
+++ b/wcs/wf/backoffice_fields.py
@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import xml.etree.ElementTree as ET
+from quixote import get_publisher
 from quixote.html import htmltext
 
 from ..qommon import _, N_
@@ -123,11 +124,20 @@ class SetBackofficeFieldsWorkflowStatusItem(WorkflowStatusItem):
                 # assign empty value as None, as that will work for all field types
                 new_value = None
             else:
-                try:
-                    new_value = self.compute(field['value'], raises=True,
-                                             formdata=formdata, status_item=self)
-                except:
-                    continue
+                with get_publisher().complex_data():
+                    try:
+                        new_value = self.compute(
+                                field['value'],
+                                raises=True,
+                                allow_complex=formdef_field.allow_complex,
+                                formdata=formdata,
+                                status_item=self)
+                    except Exception:
+                        continue
+                    if formdef_field.allow_complex:
+                        complex_value = get_publisher().get_cached_complex_data(new_value)
+                        if complex_value:
+                            new_value = complex_value
 
             if formdef_field.convert_value_from_anything:
                 try:
diff --git a/wcs/wf/create_formdata.py b/wcs/wf/create_formdata.py
index 8317104cb..99189f657 100644
--- a/wcs/wf/create_formdata.py
+++ b/wcs/wf/create_formdata.py
@@ -405,11 +405,21 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
             except KeyError:
                 missing_fields.append(mapping.field_id)
                 continue
-            try:
-                value = self.compute(mapping.expression, formdata=src, raises=True, status_item=self)
-            except Exception:
-                # already logged by self.compute
-                continue
+            with get_publisher().complex_data():
+                try:
+                    value = self.compute(
+                            mapping.expression,
+                            formdata=src,
+                            raises=True,
+                            allow_complex=dest_field.allow_complex,
+                            status_item=self)
+                except Exception:
+                    # already logged by self.compute
+                    continue
+                if dest_field.allow_complex:
+                    complex_value = get_publisher().get_cached_complex_data(value)
+                    if complex_value:
+                        value = complex_value
 
             try:
                 self._set_value(
diff --git a/wcs/workflows.py b/wcs/workflows.py
index 94cade90d..aa90b3bd4 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -1965,7 +1965,7 @@ class WorkflowStatusItem(XmlSerialisable):
         return {'type': expression_type, 'value': expression_value}
 
     @classmethod
-    def compute(cls, var, render=True, raises=False, context=None, formdata=None, status_item=None):
+    def compute(cls, var, render=True, raises=False, allow_complex=False, context=None, formdata=None, status_item=None):
         if not isinstance(var, six.string_types):
             return var
 
@@ -1993,6 +1993,7 @@ class WorkflowStatusItem(XmlSerialisable):
                                exception=exception)
 
         if expression['type'] == 'template':
+            vars['allow_complex'] = allow_complex
             try:
                 return Template(expression['value'], raises=raises, autoescape=False).render(vars)
             except TemplateError as e:
-- 
2.29.2

