From eef3a1416a8400dee0c4308e27d63d947840bd83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Mon, 11 Oct 2021 17:59:15 +0200
Subject: [PATCH 2/2] misc: more logs for some cron jobs (#57604)

---
 wcs/formdef.py              | 16 +++---
 wcs/publisher.py            |  7 +--
 wcs/wf/aggregation_email.py | 97 +++++++++++++++++++------------------
 wcs/wf/jump.py              | 74 +++++++++++++++-------------
 4 files changed, 104 insertions(+), 90 deletions(-)

diff --git a/wcs/formdef.py b/wcs/formdef.py
index 26170b1c..934d0400 100644
--- a/wcs/formdef.py
+++ b/wcs/formdef.py
@@ -15,6 +15,7 @@
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
 import base64
+import contextlib
 import copy
 import datetime
 import glob
@@ -1767,16 +1768,19 @@ Status of the form just changed (from "{{ form_previous_status }}" to "{{ form_s
 Substitutions.register('form_name', category=_('Form'), comment=_('Form Name'))
 
 
-def clean_drafts(publisher):
+def clean_drafts(publisher, job=None):
     import wcs.qommon.storage as st
     from wcs.carddef import CardDef
 
     for formdef in FormDef.select() + CardDef.select():
-        removal_date = datetime.date.today() - datetime.timedelta(days=formdef.get_drafts_lifespan())
-        for formdata in formdef.data_class().select(
-            [st.Equal('status', 'draft'), st.Less('receipt_time', removal_date.timetuple())]
-        ):
-            formdata.remove_self()
+        with job.log_long_job(
+            '%s %s' % (formdef.xml_root_node, formdef.url_name)
+        ) if job else contextlib.ExitStack():
+            removal_date = datetime.date.today() - datetime.timedelta(days=formdef.get_drafts_lifespan())
+            for formdata in formdef.data_class().select(
+                [st.Equal('status', 'draft'), st.Less('receipt_time', removal_date.timetuple())]
+            ):
+                formdata.remove_self()
 
 
 def clean_unused_files(publisher):
diff --git a/wcs/publisher.py b/wcs/publisher.py
index 283d5046..be0ff8ff 100644
--- a/wcs/publisher.py
+++ b/wcs/publisher.py
@@ -22,7 +22,7 @@ import re
 import sys
 import traceback
 import zipfile
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
 
 from django.utils.encoding import force_text
 
@@ -427,11 +427,12 @@ class WcsPublisher(QommonPublisher):
             # Could also could happen on file descriptor exhaustion.
             pass
 
-    def apply_global_action_timeouts(self):
+    def apply_global_action_timeouts(self, job=None):
         from wcs.workflows import Workflow, WorkflowGlobalActionTimeoutTrigger
 
         for workflow in Workflow.select():
-            WorkflowGlobalActionTimeoutTrigger.apply(workflow)
+            with job.log_long_job('workflow %s' % workflow.url_name) if job else ExitStack():
+                WorkflowGlobalActionTimeoutTrigger.apply(workflow)
 
     def migrate_sql(self):
         from . import sql
diff --git a/wcs/wf/aggregation_email.py b/wcs/wf/aggregation_email.py
index bdf5a7bd..5ee9ca38 100644
--- a/wcs/wf/aggregation_email.py
+++ b/wcs/wf/aggregation_email.py
@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import contextlib
+
 from quixote import get_publisher
 
 from wcs.workflows import WorkflowStatusItem, register_item_class
@@ -98,7 +100,7 @@ def lax_int(s):
         return 10000
 
 
-def send_aggregation_emails(publisher):
+def send_aggregation_emails(publisher, job=None):
     from wcs.formdef import FormDef
 
     publisher.reload_cfg()
@@ -106,61 +108,64 @@ def send_aggregation_emails(publisher):
 
     cache = {}
     for aggregate_id in AggregationEmail.keys():
-        aggregate = AggregationEmail.get(aggregate_id)
-        aggregate.remove_self()
-
-        try:
-            role = get_publisher().role_class.get(aggregate_id)
-        except KeyError:
-            continue
-        if not role.get_emails():
-            continue
-        if not aggregate.items:
-            continue
-
-        last_formdef = None
-        body = []
-        for item in sorted(aggregate.items, key=lambda x: (lax_int(x['formdef']), lax_int(x['formdata']))):
-            formdef_id = item.get('formdef')
-            if formdef_id in cache:
-                formdef, formdata, workflow = cache[formdef_id]
-            else:
-                try:
-                    formdef = FormDef.get(formdef_id)
-                except KeyError:
-                    # formdef has been deleted after AggregationEmail creation
-                    continue
-                formdata = formdef.data_class()
-                workflow = formdef.workflow
-                cache[formdef_id] = (formdef, formdata, workflow)
+        with job.log_long_job('aggregation email %s' % aggregate_id) if job else contextlib.ExitStack():
+            aggregate = AggregationEmail.get(aggregate_id)
+            aggregate.remove_self()
 
             try:
-                data = formdata.get(item.get('formdata'))
+                role = get_publisher().role_class.get(aggregate_id)
             except KeyError:
                 continue
-            status = data.get_status()
-            url = item.get('formurl')
+            if not role.get_emails():
+                continue
+            if not aggregate.items:
+                continue
 
-            if last_formdef != formdef:
-                if last_formdef is not None:
-                    body.append('')  # blank line
-                last_formdef = formdef
-                body.append(formdef.name)
-                body.append('-' * len(formdef.name))
-                body.append('')
+            last_formdef = None
+            body = []
+            for item in sorted(
+                aggregate.items, key=lambda x: (lax_int(x['formdef']), lax_int(x['formdata']))
+            ):
+                formdef_id = item.get('formdef')
+                if formdef_id in cache:
+                    formdef, formdata, workflow = cache[formdef_id]
+                else:
+                    try:
+                        formdef = FormDef.get(formdef_id)
+                    except KeyError:
+                        # formdef has been deleted after AggregationEmail creation
+                        continue
+                    formdata = formdef.data_class()
+                    workflow = formdef.workflow
+                    cache[formdef_id] = (formdef, formdata, workflow)
 
-            body.append('- %sstatus (%s)' % (url, status.name))
+                try:
+                    data = formdata.get(item.get('formdata'))
+                except KeyError:
+                    continue
+                status = data.get_status()
+                url = item.get('formurl')
 
-        if not body:
-            continue
+                if last_formdef != formdef:
+                    if last_formdef is not None:
+                        body.append('')  # blank line
+                    last_formdef = formdef
+                    body.append(formdef.name)
+                    body.append('-' * len(formdef.name))
+                    body.append('')
+
+                body.append('- %sstatus (%s)' % (url, status.name))
+
+            if not body:
+                continue
 
-        body = '\n'.join(body)
+            body = '\n'.join(body)
 
-        mail_subject = _('New arrivals')
-        if site_name:
-            mail_subject += ' (%s)' % site_name
+            mail_subject = _('New arrivals')
+            if site_name:
+                mail_subject += ' (%s)' % site_name
 
-        emails.email(mail_subject, body, email_rcpt=role.get_emails())
+            emails.email(mail_subject, body, email_rcpt=role.get_emails())
 
 
 def register_cronjob():
diff --git a/wcs/wf/jump.py b/wcs/wf/jump.py
index dfb5caed..5d8c1de6 100644
--- a/wcs/wf/jump.py
+++ b/wcs/wf/jump.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, see <http://www.gnu.org/licenses/>.
 
+import contextlib
 import datetime
 import itertools
 import json
@@ -313,7 +314,7 @@ def workflows_with_timeout():
     return wfs_status
 
 
-def _apply_timeouts(publisher):
+def _apply_timeouts(publisher, job=None):
     '''Traverse all filled form and apply expired timeout jumps if needed'''
     from ..carddef import CardDef
     from ..formdef import FormDef
@@ -324,40 +325,43 @@ def _apply_timeouts(publisher):
         status_ids = wfs_status.get(str(formdef.workflow_id))
         if not status_ids:
             continue
-        formdata_class = formdef.data_class()
-        for status_id in status_ids:
-            if publisher.is_using_postgresql():
-                # get minimum delay for jumps in this status
-                delay = math.inf
-                for jump_action in wfs_status[str(formdef.workflow_id)][status_id]:
-                    if Template.is_template_string(jump_action.timeout):
-                        delay = 0
-                        break
-                    delay = min(delay, int(jump_action.timeout))
-                # limit delay to minimal delay
-                if delay < JUMP_TIMEOUT_INTERVAL * 60:
-                    delay = JUMP_TIMEOUT_INTERVAL * 60
-
-                criterias = [
-                    Equal('status', status_id),
-                    LessOrEqual(
-                        'last_update_time',
-                        (datetime.datetime.now() - datetime.timedelta(seconds=delay)).timetuple(),
-                    ),
-                ]
-                formdatas = formdata_class.select_iterator(criterias, ignore_errors=True)
-            else:
-                formdatas = formdata_class.get_with_indexed_value('status', status_id, ignore_errors=True)
-
-            for formdata in formdatas:
-                for jump_action in wfs_status[str(formdef.workflow_id)][formdata.status]:
-                    get_publisher().substitutions.reset()
-                    get_publisher().substitutions.feed(get_publisher())
-                    get_publisher().substitutions.feed(formdef)
-                    get_publisher().substitutions.feed(formdata)
-                    if jump_action.must_jump(formdata):
-                        jump_and_perform(formdata, jump_action, event=('timeout-jump', jump_action.id))
-                        break
+        with job.log_long_job(
+            '%s %s' % (formdef.xml_root_node, formdef.url_name)
+        ) if job else contextlib.ExitStack():
+            formdata_class = formdef.data_class()
+            for status_id in status_ids:
+                if publisher.is_using_postgresql():
+                    # get minimum delay for jumps in this status
+                    delay = math.inf
+                    for jump_action in wfs_status[str(formdef.workflow_id)][status_id]:
+                        if Template.is_template_string(jump_action.timeout):
+                            delay = 0
+                            break
+                        delay = min(delay, int(jump_action.timeout))
+                    # limit delay to minimal delay
+                    if delay < JUMP_TIMEOUT_INTERVAL * 60:
+                        delay = JUMP_TIMEOUT_INTERVAL * 60
+
+                    criterias = [
+                        Equal('status', status_id),
+                        LessOrEqual(
+                            'last_update_time',
+                            (datetime.datetime.now() - datetime.timedelta(seconds=delay)).timetuple(),
+                        ),
+                    ]
+                    formdatas = formdata_class.select_iterator(criterias, ignore_errors=True)
+                else:
+                    formdatas = formdata_class.get_with_indexed_value('status', status_id, ignore_errors=True)
+
+                for formdata in formdatas:
+                    for jump_action in wfs_status[str(formdef.workflow_id)][formdata.status]:
+                        get_publisher().substitutions.reset()
+                        get_publisher().substitutions.feed(get_publisher())
+                        get_publisher().substitutions.feed(formdef)
+                        get_publisher().substitutions.feed(formdata)
+                        if jump_action.must_jump(formdata):
+                            jump_and_perform(formdata, jump_action, event=('timeout-jump', jump_action.id))
+                            break
 
 
 def register_cronjob():
-- 
2.33.0

