From cbda20dbbf9a1d6e358ed7730838c2f32ada17c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= <fpeters@entrouvert.com>
Date: Tue, 1 Jun 2021 21:17:12 +0200
Subject: [PATCH 1/2] workflows: add actions tracing (#54497)

---
 tests/api/test_workflow.py        | 21 ++++++++++-----
 wcs/api.py                        |  6 ++---
 wcs/backoffice/data_management.py |  2 +-
 wcs/backoffice/submission.py      |  2 +-
 wcs/formdata.py                   | 12 ++++++---
 wcs/forms/root.py                 |  4 +--
 wcs/qommon/storage.py             |  1 +
 wcs/wf/create_formdata.py         |  2 +-
 wcs/wf/jump.py                    | 10 ++++---
 wcs/workflows.py                  | 43 +++++++++++++++++++++++++++----
 10 files changed, 75 insertions(+), 28 deletions(-)

diff --git a/tests/api/test_workflow.py b/tests/api/test_workflow.py
index 14eb7c85a..1300b354a 100644
--- a/tests/api/test_workflow.py
+++ b/tests/api/test_workflow.py
@@ -11,7 +11,7 @@ from wcs.formdef import FormDef
 from wcs.qommon.http_request import HTTPRequest
 from wcs.qommon.ident.password_accounts import PasswordAccount
 from wcs.wf.jump import JumpWorkflowStatusItem
-from wcs.wf.register_comment import RegisterCommenterWorkflowStatusItem
+from wcs.wf.register_comment import JournalEvolutionPart, RegisterCommenterWorkflowStatusItem
 from wcs.workflows import Workflow
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app
@@ -371,6 +371,13 @@ def test_workflow_trigger_http_auth_access(pub, local_user):
     assert formdef.data_class().get(formdata.id).evolution[-1].who is None
 
 
+def get_latest_comment(formdata):
+    for evolution in reversed(formdata.evolution):
+        for part in reversed(evolution.parts):
+            if isinstance(part, JournalEvolutionPart):
+                return part.content
+
+
 def test_workflow_global_webservice_trigger(pub, local_user):
     workflow = Workflow(name='test')
     workflow.add_status('Status1', 'st1')
@@ -406,12 +413,12 @@ def test_workflow_global_webservice_trigger(pub, local_user):
 
     # anonymous call
     get_app(pub).post(formdata.get_url() + 'hooks/plop/', status=200)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == 'HELLO WORLD'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == 'HELLO WORLD'
 
     add_to_journal.comment = 'HELLO WORLD 2'
     workflow.store()
     get_app(pub).post(formdata.get_api_url() + 'hooks/plop/', status=200)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == 'HELLO WORLD 2'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == 'HELLO WORLD 2'
 
     # call requiring user
     add_to_journal.comment = 'HELLO WORLD 3'
@@ -419,7 +426,7 @@ def test_workflow_global_webservice_trigger(pub, local_user):
     workflow.store()
     get_app(pub).post(formdata.get_api_url() + 'hooks/plop/', status=403)
     get_app(pub).post(sign_uri(formdata.get_api_url() + 'hooks/plop/'), status=200)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == 'HELLO WORLD 3'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == 'HELLO WORLD 3'
 
     # call requiring roles
     add_to_journal.comment = 'HELLO WORLD 4'
@@ -436,7 +443,7 @@ def test_workflow_global_webservice_trigger(pub, local_user):
     local_user.roles = [role.id]
     local_user.store()
     get_app(pub).post(sign_uri(formdata.get_api_url() + 'hooks/plop/', user=local_user), status=200)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == 'HELLO WORLD 4'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == 'HELLO WORLD 4'
 
     # call adding data
     add_to_journal.comment = 'HELLO {{plop_test}}'
@@ -445,7 +452,7 @@ def test_workflow_global_webservice_trigger(pub, local_user):
         sign_uri(formdata.get_api_url() + 'hooks/plop/', user=local_user), {'test': 'foobar'}, status=200
     )
     # (django templating make it turn into HTML)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == '<div>HELLO foobar</div>'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == '<div>HELLO foobar</div>'
 
     # call adding data but with no actions
     ac1.items = []
@@ -491,4 +498,4 @@ def test_workflow_global_webservice_trigger_no_trailing_slash(pub, local_user):
 
     # anonymous call
     get_app(pub).post(formdata.get_url() + 'hooks/plop', status=200)
-    assert formdef.data_class().get(formdata.id).evolution[-1].parts[-1].content == 'HELLO WORLD'
+    assert get_latest_comment(formdef.data_class().get(formdata.id)) == 'HELLO WORLD'
diff --git a/wcs/api.py b/wcs/api.py
index 30e1d2f89..43bfda231 100644
--- a/wcs/api.py
+++ b/wcs/api.py
@@ -165,7 +165,7 @@ class ApiFormdataPage(FormStatusPage):
 
             if item.status:
                 self.formdata.jump_status(item.status)
-                self.formdata.perform_workflow()
+                self.formdata.perform_workflow(event=('api-post-edit-action', item.id))
 
             return json.dumps({'err': 0, 'data': {'id': str(self.formdata.id)}})
 
@@ -324,7 +324,7 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
         formdata.store()
         formdata.just_created()
         formdata.store()
-        formdata.perform_workflow()
+        formdata.perform_workflow(event='api-created')
         formdata.store()
         return json.dumps(
             {
@@ -599,7 +599,7 @@ class ApiFormdefDirectory(Directory):
         else:
             formdata.just_created()
             formdata.store()
-            formdata.perform_workflow()
+            formdata.perform_workflow(event='api-created')
             formdata.store()
         return json.dumps(
             {
diff --git a/wcs/backoffice/data_management.py b/wcs/backoffice/data_management.py
index ae3cef0ed..bb412645a 100644
--- a/wcs/backoffice/data_management.py
+++ b/wcs/backoffice/data_management.py
@@ -370,7 +370,7 @@ class ImportFromCsvAfterJob(AfterJob):
             data_instance.data = item
             data_instance.just_created()
             data_instance.store()
-            data_instance.perform_workflow()
+            data_instance.perform_workflow(event='csv-import-created')
 
     def done_action_url(self):
         carddef = self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])
diff --git a/wcs/backoffice/submission.py b/wcs/backoffice/submission.py
index 41e71ae29..0575092f2 100644
--- a/wcs/backoffice/submission.py
+++ b/wcs/backoffice/submission.py
@@ -344,7 +344,7 @@ class FormFillPage(PublicFormFillPage):
         return self.redirect_after_submitted(form, filled)
 
     def redirect_after_submitted(self, form, filled):
-        url = filled.perform_workflow()
+        url = filled.perform_workflow(event='backoffice-created')
         if url:
             pass  # always redirect to an URL the workflow returned
         elif not self.formdef.is_of_concern_for_user(self.user, filled):
diff --git a/wcs/formdata.py b/wcs/formdata.py
index a738cd099..f1d585de1 100644
--- a/wcs/formdata.py
+++ b/wcs/formdata.py
@@ -516,13 +516,13 @@ class FormData(StorableObject):
             current_level = current_level - 100
         return levels[current_level]
 
-    def perform_workflow(self):
+    def perform_workflow(self, event=None):
         url = None
         get_publisher().substitutions.feed(self)
         wf_status = self.get_status()
         from wcs.workflows import perform_items
 
-        url = perform_items(wf_status.items, self)
+        url = perform_items(wf_status.items, self, event=event)
         return url
 
     def perform_global_action(self, action_id, user):
@@ -531,7 +531,7 @@ class FormData(StorableObject):
         for action in self.formdef.workflow.get_global_actions_for_user(formdata=self, user=user):
             if action.id != action_id:
                 continue
-            perform_items(action.items, self)
+            perform_items(action.items, self, event=('global-action', action.id))
             break
 
     def get_workflow_messages(self, position='top'):
@@ -636,6 +636,8 @@ class FormData(StorableObject):
             return None
 
     def jump_status(self, status_id, user_id=None):
+        from wcs.workflows import ActionsTracingEvolutionPart
+
         if status_id == '_previous':
             previous_status = self.pop_previous_marked_status()
             if not previous_status:
@@ -650,7 +652,9 @@ class FormData(StorableObject):
             self.status == status
             and self.evolution[-1].status == status
             and not self.evolution[-1].comment
-            and not self.evolution[-1].parts
+            and not [
+                x for x in self.evolution[-1].parts or [] if not isinstance(x, ActionsTracingEvolutionPart)
+            ]
         ):
             # if status do not change and last evolution is empty,
             # just update last jump time on last evolution, do not add one
diff --git a/wcs/forms/root.py b/wcs/forms/root.py
index fd965d052..709caa2c3 100644
--- a/wcs/forms/root.py
+++ b/wcs/forms/root.py
@@ -1430,7 +1430,7 @@ class FormPage(Directory, FormTemplateMixin):
         url = None
         if existing_formdata is None:
             self.clean_submission_context()
-            url = filled.perform_workflow()
+            url = filled.perform_workflow(event='frontoffice-created')
 
         if not filled.user_id:
             get_session().mark_anonymous_formdata(filled)
@@ -1483,7 +1483,7 @@ class FormPage(Directory, FormTemplateMixin):
                 wf_status = item.get_target_status(self.edited_data)
                 if wf_status:
                     self.edited_data.jump_status(wf_status[0].id, user_id=user_id)
-                    url = self.edited_data.perform_workflow()
+                    url = self.edited_data.perform_workflow(event=('edit-action', item.id))
                 else:
                     # add history entry
                     evo = Evolution()
diff --git a/wcs/qommon/storage.py b/wcs/qommon/storage.py
index 6892d0216..aa0fd7263 100644
--- a/wcs/qommon/storage.py
+++ b/wcs/qommon/storage.py
@@ -865,6 +865,7 @@ class StorableObject:
     def remove_self(self):
         assert not self.is_readonly()
         self.remove_object(self.id)
+        self.id = None
 
     def get_last_modification_info(self):
         if not get_publisher().snapshot_class:
diff --git a/wcs/wf/create_formdata.py b/wcs/wf/create_formdata.py
index 4658b134e..2b1bccfc6 100644
--- a/wcs/wf/create_formdata.py
+++ b/wcs/wf/create_formdata.py
@@ -507,7 +507,7 @@ class CreateFormdataWorkflowStatusItem(WorkflowStatusItem):
                 new_formdata.store()
                 if formdef.enable_tracking_codes:
                     code.formdata = new_formdata  # this will .store() the code
-                new_formdata.perform_workflow()
+                new_formdata.perform_workflow(event=('workflow-created', formdata.get_display_id()))
                 new_formdata.store()
 
             # update local object as it may have been modified by new_formdata
diff --git a/wcs/wf/jump.py b/wcs/wf/jump.py
index 98d3b4ddc..563e9dfd7 100644
--- a/wcs/wf/jump.py
+++ b/wcs/wf/jump.py
@@ -40,13 +40,13 @@ from ..qommon.template import Template
 JUMP_TIMEOUT_INTERVAL = max((60 // int(os.environ.get('WCS_JUMP_TIMEOUT_CHECKS', '3')), 1))
 
 
-def jump_and_perform(formdata, action, workflow_data=None):
+def jump_and_perform(formdata, action, workflow_data=None, event=None):
     action.handle_markers_stack(formdata)
     if workflow_data:
         formdata.update_workflow_data(workflow_data)
         formdata.store()
     formdata.jump_status(action.status)
-    url = formdata.perform_workflow()
+    url = formdata.perform_workflow(event=event)
     return url
 
 
@@ -89,7 +89,9 @@ class TriggerDirectory(Directory):
                     workflow_data = None
                     if hasattr(get_request(), '_json'):
                         workflow_data = get_request().json
-                    url = jump_and_perform(self.formdata, item, workflow_data=workflow_data)
+                    url = jump_and_perform(
+                        self.formdata, item, workflow_data=workflow_data, event=('api-trigger', item.trigger)
+                    )
                 else:
                     if get_request().is_json():
                         get_response().status_code = 403
@@ -354,7 +356,7 @@ def _apply_timeouts(publisher):
                     get_publisher().substitutions.feed(formdef)
                     get_publisher().substitutions.feed(formdata)
                     if jump_action.must_jump(formdata):
-                        jump_and_perform(formdata, jump_action)
+                        jump_and_perform(formdata, jump_action, event=('timeout-jump', jump_action.id))
                         break
 
 
diff --git a/wcs/workflows.py b/wcs/workflows.py
index d9a6a2260..ce90c6313 100644
--- a/wcs/workflows.py
+++ b/wcs/workflows.py
@@ -69,14 +69,18 @@ def lax_int(s):
         return -1
 
 
-def perform_items(items, formdata, depth=20):
+def perform_items(items, formdata, depth=20, event=None):
     if depth == 0:  # prevents infinite loops
         return
     url = None
     old_status = formdata.status
+    performed_actions = []
     for item in items:
+        if getattr(item.perform, 'empty', False):
+            continue
         if not item.check_condition(formdata):
             continue
+        performed_actions.append((datetime.datetime.now(), item.id))
         try:
             url = item.perform(formdata) or url
         except AbortActionException as e:
@@ -84,6 +88,13 @@ def perform_items(items, formdata, depth=20):
             break
         if formdata.status != old_status:
             break
+    if performed_actions:
+        latest_evolution = formdata.evolution[-1] if formdata.evolution else None
+        if latest_evolution:
+            latest_evolution.add_part(ActionsTracingEvolutionPart(event, performed_actions))
+        if formdata.id:
+            # don't save formdata it has been removed
+            formdata.store()
     if formdata.status != old_status:
         if not formdata.evolution:
             formdata.evolution = []
@@ -94,7 +105,7 @@ def perform_items(items, formdata, depth=20):
         formdata.store()
         # performs the items of the new status
         wf_status = formdata.get_status()
-        url = perform_items(wf_status.items, formdata, depth=depth - 1) or url
+        url = perform_items(wf_status.items, formdata, depth=depth - 1, event='continuation') or url
     if url:
         # hack around webtest as it checks type(url) is str and
         # this won't work on django safe strings (isinstance would work);
@@ -319,6 +330,17 @@ class AttachmentEvolutionPart(EvolutionPart):
         )
 
 
+class ActionsTracingEvolutionPart(EvolutionPart):
+    def __init__(self, event, actions):
+        if isinstance(event, tuple):
+            self.event = event[0]
+            self.event_args = event[1:]
+        else:
+            self.event = event
+            self.event_args = None
+        self.actions = actions
+
+
 class DuplicateGlobalActionNameError(Exception):
     pass
 
@@ -1458,7 +1480,11 @@ class WorkflowGlobalActionTimeoutTrigger(WorkflowGlobalActionTrigger):
                             continue
                         formdata.evolution[-1].add_part(WorkflowGlobalActionTimeoutTriggerMarker(trigger.id))
                         formdata.store()
-                        perform_items(action.items, formdata)
+                        perform_items(
+                            action.items,
+                            formdata,
+                            event=('global-action-timeout', (action.id, trigger.id)),
+                        )
                         break
 
 
@@ -1705,7 +1731,7 @@ class WorkflowStatus:
         # check for global actions
         for action in filled.formdef.workflow.get_global_actions_for_user(filled, user):
             if 'button-action-%s' % action.id in get_request().form:
-                url = perform_items(action.items, filled)
+                url = perform_items(action.items, filled, event=('global-action-button', action.id))
                 if url:
                     return url
                 return
@@ -1745,7 +1771,7 @@ class WorkflowStatus:
         if evo.status:
             filled.status = evo.status
         filled.store()
-        url = filled.perform_workflow()
+        url = filled.perform_workflow(event='workflow-form-submit')
         if url:
             return url
 
@@ -1907,6 +1933,12 @@ class WorkflowStatus:
         return '<%s %s %r>' % (self.__class__.__name__, self.id, self.name)
 
 
+def empty_mark(func):
+    # mark method as not executing anything
+    func.empty = True
+    return func
+
+
 class WorkflowStatusItem(XmlSerialisable):
     node_name = 'item'
     description = 'XX'
@@ -1959,6 +1991,7 @@ class WorkflowStatusItem(XmlSerialisable):
     def get_add_role_label(self):
         return self.parent.parent.get_add_role_label()
 
+    @empty_mark
     def perform(self, formdata):
         pass
 
-- 
2.32.0

