From fcda46821d5461333d79f8bbc6cee079c54be0f2 Mon Sep 17 00:00:00 2001
From: Michael Bideau <mbideau@atreal.fr>
Date: Wed, 15 May 2019 14:44:18 +0200
Subject: [PATCH] workflow: added attachment support for RegisterComment    
 misc: added a function to create a dict from a dict and a key prefix   
 tests: updated cases in misc and workflow to tests above new features

---
 tests/test_misc.py         |  15 +++++
 tests/test_workflows.py    |  98 +++++++++++++++++++++++++++++-
 wcs/qommon/evalutils.py    |  26 ++++++++
 wcs/wf/register_comment.py |  28 ++++++++-
 wcs/workflows.py           | 145 +++++++++++++++++++++++++--------------------
 5 files changed, 244 insertions(+), 68 deletions(-)

diff --git a/tests/test_misc.py b/tests/test_misc.py
index 8e6e73a5..1e724fb1 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -440,3 +440,18 @@ M. Francis Kuntz
     assert not '<ul' in html
     assert 'arabic simple' in html
     assert 'M. Francis Kuntz' in html
+
+def test_dict_from_prefix():
+    hello_word_b64 = base64.encodestring('hello world')
+
+    d = evalutils.dict_from_prefix('var1', {})
+    assert d == {}
+
+    d = evalutils.dict_from_prefix('', {'k1':'v1'})
+    assert d == {'k1':'v1'}
+
+    d = evalutils.dict_from_prefix('k', {'k1':'v1', 'k2':'v2'})
+    assert d == {'1':'v1', '2':'v2'}
+
+    d = evalutils.dict_from_prefix('v', {'k1':'v1', 'k2':'v2'})
+    assert d == {}
diff --git a/tests/test_workflows.py b/tests/test_workflows.py
index 710f79d9..9f982784 100644
--- a/tests/test_workflows.py
+++ b/tests/test_workflows.py
@@ -37,7 +37,7 @@ from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
 from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
 from wcs.wf.timeout_jump import TimeoutWorkflowStatusItem
 from wcs.wf.profile import UpdateUserProfileStatusItem
-from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
+from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem, JournalEvolutionPart
 from wcs.wf.remove import RemoveWorkflowStatusItem
 from wcs.wf.roles import AddRoleWorkflowStatusItem, RemoveRoleWorkflowStatusItem
 from wcs.wf.wscall import WebserviceCallStatusItem
@@ -852,7 +852,7 @@ def test_register_comment_django_escaping(pub, emails):
     formdata.evolution[-1]._display_parts = None
     assert formdata.evolution[-1].display_parts()[-1] == '<div><p>hello</p></div>'
 
-def test_register_comment_attachment(pub):
+def test_register_comment_attachment_substitution(pub):
     pub.substitutions.feed(MockSubstitutionVariables())
 
     formdef = FormDef()
@@ -923,6 +923,100 @@ def test_register_comment_attachment(pub):
     assert url3 == url4
 
 
+def test_register_comment_attachment(pub):
+    wf = Workflow(name='comment with attachments')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', type='file', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    jpg = open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg')).read()
+    upload.receive([jpg])
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        FileField(id='1', label='File', type='file', varname='frontoffice_file'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': upload}
+    formdata.just_created()
+    formdata.store()
+
+    pub.substitutions.feed(formdata)
+
+    setbo = SetBackofficeFieldsWorkflowStatusItem()
+    setbo.parent = st1
+    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_frontoffice_file_raw'}]
+    setbo.perform(formdata)
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    comment_text = 'File is attached to the form history'
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = ['form_var_backoffice_file1_raw']
+    item.comment = comment_text
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 2
+    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[0].orig_filename == upload.orig_filename
+
+    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[1].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[1].view())
+    assert comment_view == '<p>%s</p>' % comment_text
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    formdata.evolution[-1].parts = []
+    formdata.store()
+
+    ws_response_varname = 'ws_response_afile'
+    wf_data = {
+        '%s_filename'     % ws_response_varname : 'hello.txt',
+        '%s_content_type' % ws_response_varname : 'text/plain',
+        '%s_b64_content'  % ws_response_varname : base64.encodestring('hello world')
+    }
+    formdata.update_workflow_data(wf_data)
+    formdata.store()
+    assert hasattr(formdata, 'workflow_data')
+    assert isinstance(formdata.workflow_data, dict)
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = ["utils.dict_from_prefix('%s_', locals())" % ws_response_varname]
+    item.comment = comment_text
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 2
+    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[0].orig_filename == 'hello.txt'
+
+    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[1].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[1].view())
+    assert comment_view == '<p>%s</p>' % comment_text
+
+
 def test_email(pub, emails):
     pub.substitutions.feed(MockSubstitutionVariables())
 
diff --git a/wcs/qommon/evalutils.py b/wcs/qommon/evalutils.py
index ae635fc3..9823a021 100644
--- a/wcs/qommon/evalutils.py
+++ b/wcs/qommon/evalutils.py
@@ -22,6 +22,7 @@ get_global_eval_dict.
 
 import datetime
 import time
+import re
 
 from .misc import get_as_datetime
 
@@ -136,3 +137,28 @@ def attachment(content, filename='', content_type='application/octet-stream'):
         'content_type': content_type,
         'b64_content': base64.b64encode(content),
     }
+
+def dict_from_prefix(prefix, in_dict):
+    '''Return a dict based on a dict filtered by a key prefix.
+
+       The prefix is removed from the key.
+
+       Intent: meant to help build a PicklableUpload from a set
+               of key/values stored in the workflow data.
+
+       Note: to use this function in a context of a Python
+             expression, you should pass the _wf_data_ using
+             the function locals()
+
+       Example: utils.dict_from_prefix('akey_', locals())
+         Where: the workflow data contains the key/values:
+                akey_filename     = <filename>
+                akey_content_type = <mime_type>
+                akey_b64_content  = <content base64 encoded>
+           And: it produces a dict like the key/values are:
+                filename     = wf_data['akey_filename']
+                content_type = wf_data['akey_content_type']
+                b64_content  = wf_data['akey_b64_content']
+    '''
+    return {re.sub(r'^%s' % prefix, '', k): v \
+            for k,v in in_dict.items() if k.startswith('%s' % prefix)}
diff --git a/wcs/wf/register_comment.py b/wcs/wf/register_comment.py
index e40eda4a..62f0889b 100644
--- a/wcs/wf/register_comment.py
+++ b/wcs/wf/register_comment.py
@@ -21,8 +21,10 @@ from qommon.form import *
 from qommon.template import TemplateError
 from qommon import get_logger
 
-from wcs.workflows import WorkflowStatusItem, register_item_class, template_on_formdata
+from wcs.workflows import (WorkflowStatusItem, register_item_class, template_on_formdata,
+                           AttachmentEvolutionPart)
 
+import sys
 
 class JournalEvolutionPart: #pylint: disable=C1001
     content = None
@@ -84,11 +86,33 @@ class RegisterCommenterWorkflowStatusItem(WorkflowStatusItem):
                 value=self.comment, cols=80, rows=10)
 
     def get_parameters(self):
-        return ('comment', 'condition')
+        return ('comment', 'attachments', 'condition')
+
+    def attach_uploads_to_formdata(self, formdata, uploads):
+        if not formdata.evolution[-1].parts:
+            formdata.evolution[-1].parts = []
+        for upload in uploads:
+            try:
+                # useless but required to restore upload.fp from serialized state, needed by 'AttachmentEvolutionPart.from_upload()
+                fp = upload.get_file_pointer()
+                formdata.evolution[-1].add_part(AttachmentEvolutionPart.from_upload(upload))
+            except:
+                get_publisher().notify_of_exception(sys.exc_info(),
+                                                    context='[comment/attachments]')
+            continue
 
     def perform(self, formdata):
         if not formdata.evolution:
             return
+
+        # process attachments first, they might be used in the comment
+        # (with substitution vars)
+        if self.attachments:
+            uploads = self.convert_attachments_to_uploads()
+            self.attach_uploads_to_formdata(formdata, uploads)
+            formdata.store()  # store and invalidate cache, so references can be used in the comment message.
+
+        # the comment can use attachments done above
         try:
             formdata.evolution[-1].add_part(JournalEvolutionPart(formdata, self.comment))
             formdata.store()
diff --git a/wcs/workflows.py b/wcs/workflows.py
index d734abbd..715da48e 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -45,6 +45,9 @@ from wcs.fields import FileField
 from wcs.formdef import FormDef
 from wcs.formdata import Evolution
 
+from wcs.qommon.form import PicklableUpload as PUpload
+from collections import OrderedDict
+
 if not __name__.startswith('wcs.') and not __name__ == "__main__":
     raise ImportError('Import of workflows module must be absolute (import wcs.workflows)')
 
@@ -1571,7 +1574,10 @@ class WorkflowStatusItem(XmlSerialisable):
     ok_in_global_action = True # means it can be used in a global action
     directory_name = None
     directory_class = None
-    support_substitution_variables = False
+
+    support_substitution_variables = True
+    attachments = None
+
 
     @classmethod
     def init(cls):
@@ -1651,6 +1657,23 @@ class WorkflowStatusItem(XmlSerialisable):
                      value=self.condition, size=40,
                      advanced=not(self.condition))
 
+        if 'attachments' in parameters:
+            attachments_options, attachments = self.get_attachments_options()
+            if len(attachments_options) > 1:
+                form.add(WidgetList, '%sattachments' % prefix, title=_('Attachments'),
+                         element_type=SingleSelectWidgetWithOther,
+                         value=attachments,
+                         add_element_label=_('Add attachment'),
+                         element_kwargs={'render_br': False, 'options': attachments_options})
+            else:
+                form.add(WidgetList, '%sattachments' % prefix,
+                         title=_('Attachments (Python expressions)'),
+                         element_type=StringWidget,
+                         value=attachments,
+                         add_element_label=_('Add attachment'),
+                         element_kwargs={'render_br': False, 'size': 50},
+                         advanced=not(bool(attachments)))
+
     def get_parameters(self):
         return ('condition',)
 
@@ -1890,6 +1913,62 @@ class WorkflowStatusItem(XmlSerialisable):
             del odict['parent']
         return odict
 
+    def attachments_init_with_xml(self, elem, charset, include_id=False):
+        if elem is None:
+            self.attachments = None
+        else:
+            self.attachments = [item.text.encode(charset) for item in elem.findall('attachment')]
+
+    def get_attachments_options(self):
+        attachments_options = [(None, '---', None)]
+        varnameless = []
+        for field in self.parent.parent.get_backoffice_fields():
+            if field.key != 'file':
+                continue
+            if field.varname:
+                codename = 'form_var_%s_raw' % field.varname
+            else:
+                codename = 'form_f%s' % field.id  # = form_fbo<n>
+                varnameless.append(codename)
+            attachments_options.append((codename, field.label, codename))
+        # filter: do not consider removed fields without varname
+        attachments = [attachment for attachment in self.attachments or []
+                       if ((not attachment.startswith('form_fbo')) or
+                           (attachment in varnameless))]
+        return attachments_options, attachments
+
+    def convert_attachments_to_uploads(self):
+        uploads = []
+
+        if self.attachments:
+            global_eval_dict = get_publisher().get_global_eval_dict()
+            local_eval_dict = get_publisher().substitutions.get_context_variables()
+            for attachment in self.attachments:
+                try:
+                    # execute any Python expression
+                    # and magically convert string like 'form_var_*_raw' to a PicklableUpload
+                    picklableupload = eval(attachment, global_eval_dict, local_eval_dict)
+                except:
+                    get_publisher().notify_of_exception(sys.exc_info(),
+                                                        context='[workflow/attachments]')
+                    continue
+
+                if not picklableupload:
+                    continue
+
+                try:
+                    # magically convert any value to a PicklableUpload
+                    # usualy a dict like one provided by qommon/evalutils:attachment()
+                    picklableupload = FileField.convert_value_from_anything(picklableupload)
+                except ValueError:
+                    get_publisher().notify_of_exception(sys.exc_info(),
+                                                        context='[workflow/attachments]')
+                    continue
+
+                uploads.append(picklableupload)
+
+        return uploads
+
 
 class WorkflowStatusJumpItem(WorkflowStatusItem):
     status = None
@@ -2230,13 +2309,11 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
     description = N_('Email')
     key = 'sendmail'
     category = 'interaction'
-    support_substitution_variables = True
 
     to = []
     subject = None
     body = None
     custom_from = None
-    attachments = None
 
     comment = None
 
@@ -2252,12 +2329,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
         return super(SendmailWorkflowStatusItem, self)._get_role_id_from_xml(
                 elem, charset, include_id=include_id)
 
-    def attachments_init_with_xml(self, elem, charset, include_id=False):
-        if elem is None:
-            self.attachments = None
-        else:
-            self.attachments = [item.text.encode(charset) for item in elem.findall('attachment')]
-
     def render_list_of_roles_or_emails(self, roles):
         t = []
         for r in roles:
@@ -2287,24 +2358,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
     def fill_admin_form(self, form):
         self.add_parameters_widgets(form, self.get_parameters())
 
-    def get_attachments_options(self):
-        attachments_options = [(None, '---', None)]
-        varnameless = []
-        for field in self.parent.parent.get_backoffice_fields():
-            if field.key != 'file':
-                continue
-            if field.varname:
-                codename = 'form_var_%s_raw' % field.varname
-            else:
-                codename = 'form_f%s' % field.id  # = form_fbo<n>
-                varnameless.append(codename)
-            attachments_options.append((codename, field.label, codename))
-        # filter: do not consider removed fields without varname
-        attachments = [attachment for attachment in self.attachments or []
-                       if ((not attachment.startswith('form_fbo')) or
-                           (attachment in varnameless))]
-        return attachments_options, attachments
-
     def add_parameters_widgets(self, form, parameters, prefix='', formdef=None):
         super(SendmailWorkflowStatusItem, self).add_parameters_widgets(
                 form, parameters, prefix=prefix, formdef=formdef)
@@ -2325,23 +2378,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
                      value=self.body, cols=80, rows=10,
                      validation_function=ComputedExpressionWidget.validate_template)
 
-        if 'attachments' in parameters:
-            attachments_options, attachments = self.get_attachments_options()
-            if len(attachments_options) > 1:
-                form.add(WidgetList, '%sattachments' % prefix, title=_('Attachments'),
-                         element_type=SingleSelectWidgetWithOther,
-                         value=attachments,
-                         add_element_label=_('Add attachment'),
-                         element_kwargs={'render_br': False, 'options': attachments_options})
-            else:
-                form.add(WidgetList, '%sattachments' % prefix,
-                         title=_('Attachments (Python expressions)'),
-                         element_type=StringWidget,
-                         value=attachments,
-                         add_element_label=_('Add attachment'),
-                         element_kwargs={'render_br': False, 'size': 50},
-                         advanced=not(bool(attachments)))
-
         if 'custom_from' in parameters:
             form.add(ComputedExpressionWidget, '%scustom_from' % prefix,
                      title=_('Custom From Address'), value=self.custom_from,
@@ -2429,26 +2465,7 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
         if self.custom_from:
             email_from = self.compute(self.custom_from)
 
-        attachments = []
-        if self.attachments:
-            global_eval_dict = get_publisher().get_global_eval_dict()
-            local_eval_dict = get_publisher().substitutions.get_context_variables()
-            for attachment in self.attachments:
-                try:
-                    picklableupload = eval(attachment, global_eval_dict, local_eval_dict)
-                except:
-                    get_publisher().notify_of_exception(sys.exc_info(),
-                                                        context='[Sendmail/attachments]')
-                    continue
-                if not picklableupload:
-                    continue
-                try:
-                    picklableupload = FileField.convert_value_from_anything(picklableupload)
-                except ValueError:
-                    get_publisher().notify_of_exception(sys.exc_info(),
-                                                        context='[Sendmail/attachments]')
-                    continue
-                attachments.append(picklableupload)
+        attachments = self.convert_attachments_to_uploads()
 
         if len(addresses) > 1:
             emails.email(mail_subject, mail_body, email_rcpt=None,
-- 
2.11.0

