From c8174a1a7c8022ebce3f2ec08a5b972ec10a90a7 Mon Sep 17 00:00:00 2001
From: Michael Bideau <mbideau@atreal.fr>
Date: Thu, 18 Apr 2019 11:15:03 +0000
Subject: [PATCH] workflow: added attachments for RegisterComment          
 (attachments methods added to class WorkflowStatusItem)

misc: updated evalutils.attachment() to handle content already encoded to base64
      and support varname parameter
---
 tests/test_misc.py         |  95 ++++++++++++++++++++++++++
 tests/test_workflows.py    | 106 +++++++++++++++++++++++++++-
 wcs/qommon/evalutils.py    |  13 ++--
 wcs/wf/register_comment.py |  28 +++++++-
 wcs/workflows.py           | 167 ++++++++++++++++++++++++++++-----------------
 5 files changed, 336 insertions(+), 73 deletions(-)

diff --git a/tests/test_misc.py b/tests/test_misc.py
index 8e6e73a5..28398d36 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -440,3 +440,98 @@ M. Francis Kuntz
     assert not '<ul' in html
     assert 'arabic simple' in html
     assert 'M. Francis Kuntz' in html
+
+def test_attachment():
+    hello_word_b64 = base64.encodestring('hello world')
+
+    d = evalutils.attachment(varname='var1')
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : '',
+        'content_type': 'application/octet-stream',
+        'b64_content' : ''
+    }
+
+    d = evalutils.attachment(
+        varname ='var1',
+        filename='hello.txt'
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : 'hello.txt',
+        'content_type': 'application/octet-stream',
+        'b64_content' : ''
+    }
+
+    d = evalutils.attachment(
+        varname     ='var1',
+        content_type='text/plain'
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : '',
+        'content_type': 'text/plain',
+        'b64_content' : ''
+    }
+
+    d = evalutils.attachment(
+        varname='var1',
+        content='hello world'
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : '',
+        'content_type': 'application/octet-stream',
+        'b64_content' : hello_word_b64
+    }
+
+    d = evalutils.attachment(
+        varname    ='var1',
+        b64_content=hello_word_b64
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : '',
+        'content_type': 'application/octet-stream',
+        'b64_content' : hello_word_b64
+    }
+
+    d = evalutils.attachment(
+        varname     ='var1',
+        filename    ='hello.txt',
+        content_type='text/plain',
+        content     ='hello world'
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : 'hello.txt',
+        'content_type': 'text/plain',
+        'b64_content' : hello_word_b64
+    }
+
+    d = evalutils.attachment(
+        varname     ='var1',
+        filename    ='hello.txt',
+        content_type='text/plain',
+        b64_content =hello_word_b64
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : 'hello.txt',
+        'content_type': 'text/plain',
+        'b64_content' : hello_word_b64
+    }
+
+    d = evalutils.attachment(
+        varname     ='var1',
+        filename    ='hello.txt',
+        content_type='text/plain',
+        content     ='hello world',
+        b64_content =hello_word_b64
+    )
+    assert d == {
+        'varname'     : 'var1',
+        'filename'    : 'hello.txt',
+        'content_type': 'text/plain',
+        'b64_content' : hello_word_b64
+    }
diff --git a/tests/test_workflows.py b/tests/test_workflows.py
index ec093fc4..5c19f7e5 100644
--- a/tests/test_workflows.py
+++ b/tests/test_workflows.py
@@ -37,7 +37,7 @@ from wcs.wf.form import FormWorkflowStatusItem, WorkflowFormFieldsFormDef
 from wcs.wf.jump import JumpWorkflowStatusItem, _apply_timeouts
 from wcs.wf.timeout_jump import TimeoutWorkflowStatusItem
 from wcs.wf.profile import UpdateUserProfileStatusItem
-from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
+from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem, JournalEvolutionPart
 from wcs.wf.remove import RemoveWorkflowStatusItem
 from wcs.wf.roles import AddRoleWorkflowStatusItem, RemoveRoleWorkflowStatusItem
 from wcs.wf.wscall import WebserviceCallStatusItem
@@ -851,7 +851,7 @@ def test_register_comment_django_escaping(pub, emails):
     formdata.evolution[-1]._display_parts = None
     assert formdata.evolution[-1].display_parts()[-1] == '<div><p>hello</p></div>'
 
-def test_register_comment_attachment(pub):
+def test_register_comment_attachment_substitution(pub):
     pub.substitutions.feed(MockSubstitutionVariables())
 
     formdef = FormDef()
@@ -922,6 +922,108 @@ def test_register_comment_attachment(pub):
     assert url3 == url4
 
 
+def test_register_comment_attachment(pub):
+    wf = Workflow(name='comment with attachments')
+    wf.backoffice_fields_formdef = WorkflowBackofficeFieldsFormDef(wf)
+    wf.backoffice_fields_formdef.fields = [
+        FileField(id='bo1', label='bo field 1', type='file', varname='backoffice_file1'),
+    ]
+    st1 = wf.add_status('Status1')
+    wf.store()
+
+    upload = PicklableUpload('test.jpeg', 'image/jpeg')
+    jpg = open(os.path.join(os.path.dirname(__file__), 'image-with-gps-data.jpeg')).read()
+    upload.receive([jpg])
+
+    formdef = FormDef()
+    formdef.name = 'baz'
+    formdef.fields = [
+        FileField(id='1', label='File', type='file', varname='frontoffice_file'),
+    ]
+    formdef.workflow_id = wf.id
+    formdef.store()
+
+    formdata = formdef.data_class()()
+    formdata.data = {'1': upload}
+    formdata.just_created()
+    formdata.store()
+
+    pub.substitutions.feed(formdata)
+
+    setbo = SetBackofficeFieldsWorkflowStatusItem()
+    setbo.parent = st1
+    setbo.fields = [{'field_id': 'bo1', 'value': '=form_var_frontoffice_file_raw'}]
+    setbo.perform(formdata)
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = ['form_var_backoffice_file1_raw']
+    item.comment = '<div>{{ attachments.%s.url }}</div>' % 'backoffice_file1'
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 2
+    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[0].varname == 'backoffice_file1'
+    assert formdata.evolution[-1].parts[0].orig_filename == upload.orig_filename
+
+    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[1].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[1].view())
+    assert len(comment_view) > len('<div></div>')
+    assert re.search('<div>[^<]+</div>', comment_view)
+
+    if os.path.exists(os.path.join(get_publisher().app_dir, 'attachments')):
+        shutil.rmtree(os.path.join(get_publisher().app_dir, 'attachments'))
+
+    formdata.evolution[-1].parts = []
+    formdata.store()
+
+    ws_response_varname = 'ws_response_afile'
+    wf_data = {
+        '%s_filename'     % ws_response_varname : 'hello.txt',
+        '%s_content_type' % ws_response_varname : 'text/plain',
+        '%s_b64_content'  % ws_response_varname : base64.encodestring('hello world')
+    }
+    formdata.update_workflow_data(wf_data)
+    formdata.store()
+    assert hasattr(formdata, 'workflow_data')
+    assert isinstance(formdata.workflow_data, dict)
+
+    item = RegisterCommenterWorkflowStatusItem()
+    item.attachments = [
+        "utils.attachment("
+            "varname='{0}',"
+            "filename={1}_filename,"
+            "b64_content ={1}_b64_content,"
+            "content_type={1}_content_type)"
+        "".format('afile', ws_response_varname)]
+    item.comment = '{{ attachments.%s.url }}' % 'afile'
+    item.perform(formdata)
+
+    assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments'))) == 1
+    for subdir in os.listdir(os.path.join(get_publisher().app_dir, 'attachments')):
+        assert len(subdir) == 4
+        assert len(os.listdir(os.path.join(get_publisher().app_dir, 'attachments', subdir))) == 1
+
+    assert len(formdata.evolution[-1].parts) == 2
+    assert isinstance(formdata.evolution[-1].parts[0], AttachmentEvolutionPart)
+    assert formdata.evolution[-1].parts[0].varname == 'afile'
+    assert formdata.evolution[-1].parts[0].orig_filename == 'hello.txt'
+
+    assert isinstance(formdata.evolution[-1].parts[1], JournalEvolutionPart)
+    assert len(formdata.evolution[-1].parts[1].content) > 0
+    comment_view = str(formdata.evolution[-1].parts[1].view())
+    assert len(comment_view) > len('<div></div>')
+    assert re.search('<div>[^<]+</div>', comment_view)
+
+
 def test_email(pub, emails):
     pub.substitutions.feed(MockSubstitutionVariables())
 
diff --git a/wcs/qommon/evalutils.py b/wcs/qommon/evalutils.py
index ae635fc3..51feccbc 100644
--- a/wcs/qommon/evalutils.py
+++ b/wcs/qommon/evalutils.py
@@ -127,12 +127,15 @@ def add_days(date, count):
     return make_date(date) + datetime.timedelta(days=count)
 
 
-def attachment(content, filename='', content_type='application/octet-stream'):
+def attachment(content='', filename='', content_type='application/octet-stream', b64_content='', varname=None):
     '''Serialize content as an attachment'''
     import base64
 
-    return {
-        'filename': filename,
-        'content_type': content_type,
-        'b64_content': base64.b64encode(content),
+    d = {
+        'filename'    : filename if filename else '',
+        'content_type': content_type if content_type else 'application/octet-stream',
+        'b64_content' : b64_content if b64_content else base64.encodestring(content)
     }
+    if varname:
+        d['varname'] = varname
+    return d
diff --git a/wcs/wf/register_comment.py b/wcs/wf/register_comment.py
index e40eda4a..ae3ec3c2 100644
--- a/wcs/wf/register_comment.py
+++ b/wcs/wf/register_comment.py
@@ -21,8 +21,10 @@ from qommon.form import *
 from qommon.template import TemplateError
 from qommon import get_logger
 
-from wcs.workflows import WorkflowStatusItem, register_item_class, template_on_formdata
+from wcs.workflows import (WorkflowStatusItem, register_item_class, template_on_formdata,
+                           AttachmentEvolutionPart)
 
+import sys
 
 class JournalEvolutionPart: #pylint: disable=C1001
     content = None
@@ -84,11 +86,33 @@ class RegisterCommenterWorkflowStatusItem(WorkflowStatusItem):
                 value=self.comment, cols=80, rows=10)
 
     def get_parameters(self):
-        return ('comment', 'condition')
+        return ('comment', 'attachments', 'condition')
+
+    def attach_uploads_to_formdata(self, formdata, uploads):
+        if not formdata.evolution[-1].parts:
+            formdata.evolution[-1].parts = []
+        for varname, upload in uploads.items():
+            try:
+                # useless but required to restore upload.fp from serialized state, needed by 'AttachmentEvolutionPart.from_upload()
+                fp = upload.get_file_pointer()
+                formdata.evolution[-1].add_part(AttachmentEvolutionPart.from_upload(upload, varname=varname))
+            except:
+                get_publisher().notify_of_exception(sys.exc_info(),
+                                                    context='[comment/attachments]')
+            continue
 
     def perform(self, formdata):
         if not formdata.evolution:
             return
+
+        # process attachments first, they might be used in the comment
+        # (with substitution vars)
+        if self.attachments:
+            uploads = self.convert_attachments_to_uploads()
+            self.attach_uploads_to_formdata(formdata, uploads)
+            formdata.store()  # store and invalidate cache, so references can be used in the comment message.
+
+        # the comment can use attachments done above
         try:
             formdata.evolution[-1].add_part(JournalEvolutionPart(formdata, self.comment))
             formdata.store()
diff --git a/wcs/workflows.py b/wcs/workflows.py
index 62f4bf46..5b759e7a 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -45,6 +45,9 @@ from wcs.fields import FileField
 from wcs.formdef import FormDef
 from wcs.formdata import Evolution
 
+from wcs.qommon.form import PicklableUpload as PUpload
+from collections import OrderedDict
+
 if not __name__.startswith('wcs.') and not __name__ == "__main__":
     raise ImportError('Import of workflows module must be absolute (import wcs.workflows)')
 
@@ -1523,7 +1526,10 @@ class WorkflowStatusItem(XmlSerialisable):
     ok_in_global_action = True # means it can be used in a global action
     directory_name = None
     directory_class = None
-    support_substitution_variables = False
+
+    support_substitution_variables = True
+    attachments = None
+
 
     @classmethod
     def init(cls):
@@ -1603,6 +1609,23 @@ class WorkflowStatusItem(XmlSerialisable):
                      value=self.condition, size=40,
                      advanced=not(self.condition))
 
+        if 'attachments' in parameters:
+            attachments_options, attachments = self.get_attachments_options()
+            if len(attachments_options) > 1:
+                form.add(WidgetList, '%sattachments' % prefix, title=_('Attachments'),
+                         element_type=SingleSelectWidgetWithOther,
+                         value=attachments,
+                         add_element_label=_('Add attachment'),
+                         element_kwargs={'render_br': False, 'options': attachments_options})
+            else:
+                form.add(WidgetList, '%sattachments' % prefix,
+                         title=_('Attachments (Python expressions)'),
+                         element_type=StringWidget,
+                         value=attachments,
+                         add_element_label=_('Add attachment'),
+                         element_kwargs={'render_br': False, 'size': 50},
+                         advanced=not(bool(attachments)))
+
     def get_parameters(self):
         return ('condition',)
 
@@ -1842,6 +1865,84 @@ class WorkflowStatusItem(XmlSerialisable):
             del odict['parent']
         return odict
 
+    def attachments_init_with_xml(self, elem, charset, include_id=False):
+        if elem is None:
+            self.attachments = None
+        else:
+            self.attachments = [item.text.encode(charset) for item in elem.findall('attachment')]
+
+    def get_attachments_options(self):
+        attachments_options = [(None, '---', None)]
+        varnameless = []
+        for field in self.parent.parent.get_backoffice_fields():
+            if field.key != 'file':
+                continue
+            if field.varname:
+                codename = 'form_var_%s_raw' % field.varname
+            else:
+                codename = 'form_f%s' % field.id  # = form_fbo<n>
+                varnameless.append(codename)
+            attachments_options.append((codename, field.label, codename))
+        # filter: do not consider removed fields without varname
+        attachments = [attachment for attachment in self.attachments or []
+                       if ((not attachment.startswith('form_fbo')) or
+                           (attachment in varnameless))]
+        return attachments_options, attachments
+
+    def get_attachment_varname(self, attachment, upload=None):
+        varname = str(attachment)
+        if isinstance(attachment, str) and re.match(r'form_var_', attachment):
+            varname = attachment
+            m = re.search(r'^form_var_(\w+)$', attachment)
+            if m:
+                varname = re.sub(r'_raw$', '', m.group(1)) if re.search(r'_raw$', m.group(1)) else m.group(1)
+        elif upload:
+            if isinstance(upload, PUpload):
+                varname = str(upload)
+            elif isinstance(upload, dict) and 'varname' in upload:
+                varname = upload['varname']
+            elif hasattr(upload, varname):
+                varname = upload.varname
+        return varname
+
+    def convert_attachments_to_uploads(self):
+        # forget previous attachment converted to uploads
+        # (file with same content and same varname, will not be duplicated
+        #  because the path of the file is made of the hash of the content)
+        uploads = OrderedDict()
+
+        if self.attachments:
+            global_eval_dict = get_publisher().get_global_eval_dict()
+            local_eval_dict = get_publisher().substitutions.get_context_variables()
+            for attachment in self.attachments:
+                try:
+                    # execute any Python expression
+                    # and magically convert string like 'form_var_*_raw' to a PicklableUpload
+                    picklableupload = eval(attachment, global_eval_dict, local_eval_dict)
+                except:
+                    get_publisher().notify_of_exception(sys.exc_info(),
+                                                        context='[workflow/attachments]')
+                    continue
+
+                if not picklableupload:
+                    continue
+
+                varname = self.get_attachment_varname(attachment, picklableupload)
+
+                try:
+                    # magically convert any value to a PicklableUpload
+                    # usualy a dict like one provided by qommon/evalutils:attachment()
+                    picklableupload = FileField.convert_value_from_anything(picklableupload)
+                except ValueError:
+                    get_publisher().notify_of_exception(sys.exc_info(),
+                                                        context='[workflow/attachments]')
+                    continue
+
+                # exsiting attachments upload will be replaced by the new one (last one wins)
+                uploads[varname] = picklableupload
+
+        return uploads
+
 
 class WorkflowStatusJumpItem(WorkflowStatusItem):
     status = None
@@ -2182,13 +2283,11 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
     description = N_('Email')
     key = 'sendmail'
     category = 'interaction'
-    support_substitution_variables = True
 
     to = []
     subject = None
     body = None
     custom_from = None
-    attachments = None
 
     comment = None
 
@@ -2204,12 +2303,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
         return super(SendmailWorkflowStatusItem, self)._get_role_id_from_xml(
                 elem, charset, include_id=include_id)
 
-    def attachments_init_with_xml(self, elem, charset, include_id=False):
-        if elem is None:
-            self.attachments = None
-        else:
-            self.attachments = [item.text.encode(charset) for item in elem.findall('attachment')]
-
     def render_list_of_roles_or_emails(self, roles):
         t = []
         for r in roles:
@@ -2239,24 +2332,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
     def fill_admin_form(self, form):
         self.add_parameters_widgets(form, self.get_parameters())
 
-    def get_attachments_options(self):
-        attachments_options = [(None, '---', None)]
-        varnameless = []
-        for field in self.parent.parent.get_backoffice_fields():
-            if field.key != 'file':
-                continue
-            if field.varname:
-                codename = 'form_var_%s_raw' % field.varname
-            else:
-                codename = 'form_f%s' % field.id  # = form_fbo<n>
-                varnameless.append(codename)
-            attachments_options.append((codename, field.label, codename))
-        # filter: do not consider removed fields without varname
-        attachments = [attachment for attachment in self.attachments or []
-                       if ((not attachment.startswith('form_fbo')) or
-                           (attachment in varnameless))]
-        return attachments_options, attachments
-
     def add_parameters_widgets(self, form, parameters, prefix='', formdef=None):
         super(SendmailWorkflowStatusItem, self).add_parameters_widgets(
                 form, parameters, prefix=prefix, formdef=formdef)
@@ -2277,23 +2352,6 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
                      value=self.body, cols=80, rows=10,
                      validation_function=ComputedExpressionWidget.validate_template)
 
-        if 'attachments' in parameters:
-            attachments_options, attachments = self.get_attachments_options()
-            if len(attachments_options) > 1:
-                form.add(WidgetList, '%sattachments' % prefix, title=_('Attachments'),
-                         element_type=SingleSelectWidgetWithOther,
-                         value=attachments,
-                         add_element_label=_('Add attachment'),
-                         element_kwargs={'render_br': False, 'options': attachments_options})
-            else:
-                form.add(WidgetList, '%sattachments' % prefix,
-                         title=_('Attachments (Python expressions)'),
-                         element_type=StringWidget,
-                         value=attachments,
-                         add_element_label=_('Add attachment'),
-                         element_kwargs={'render_br': False, 'size': 50},
-                         advanced=not(bool(attachments)))
-
         if 'custom_from' in parameters:
             form.add(ComputedExpressionWidget, '%scustom_from' % prefix,
                      title=_('Custom From Address'), value=self.custom_from,
@@ -2381,26 +2439,7 @@ class SendmailWorkflowStatusItem(WorkflowStatusItem):
         if self.custom_from:
             email_from = self.compute(self.custom_from)
 
-        attachments = []
-        if self.attachments:
-            global_eval_dict = get_publisher().get_global_eval_dict()
-            local_eval_dict = get_publisher().substitutions.get_context_variables()
-            for attachment in self.attachments:
-                try:
-                    picklableupload = eval(attachment, global_eval_dict, local_eval_dict)
-                except:
-                    get_publisher().notify_of_exception(sys.exc_info(),
-                                                        context='[Sendmail/attachments]')
-                    continue
-                if not picklableupload:
-                    continue
-                try:
-                    picklableupload = FileField.convert_value_from_anything(picklableupload)
-                except ValueError:
-                    get_publisher().notify_of_exception(sys.exc_info(),
-                                                        context='[Sendmail/attachments]')
-                    continue
-                attachments.append(picklableupload)
+        attachments = self.convert_attachments_to_uploads().values()
 
         if len(addresses) > 1:
             emails.email(mail_subject, mail_body, email_rcpt=None,
-- 
2.11.0

