From d9701615e84cfe3cb9328433577b681defea54e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Laur=C3=A9line=20Gu=C3=A9rin?= <zebuline@entrouvert.com>
Date: Thu, 12 Aug 2021 09:26:46 +0200
Subject: [PATCH] misc: display after job progression (#46672)

---
 tests/api/test_all.py                 |  2 +-
 tests/backoffice_pages/test_export.py |  2 +-
 wcs/admin/settings.py                 | 11 +++++++++++
 wcs/api.py                            |  2 +-
 wcs/backoffice/data_management.py     |  6 +++++-
 wcs/backoffice/management.py          | 13 ++++++++-----
 wcs/qommon/afterjobs.py               | 24 +++++++++++++++++++++---
 7 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/tests/api/test_all.py b/tests/api/test_all.py
index 4ac2f818..b7b8d0fd 100644
--- a/tests/api/test_all.py
+++ b/tests/api/test_all.py
@@ -246,6 +246,6 @@ def test_afterjobs_status(pub):
             'status': 'registered',
             'creation_time': job.creation_time,
             'completion_time': None,
-            'completion_status': None,
+            'completion_status': '',
         },
     }
diff --git a/tests/backoffice_pages/test_export.py b/tests/backoffice_pages/test_export.py
index 847451e2..23736d16 100644
--- a/tests/backoffice_pages/test_export.py
+++ b/tests/backoffice_pages/test_export.py
@@ -221,7 +221,7 @@ def test_backoffice_export_long_listings(pub, threshold):
 
     # check afterjob ajax call
     status_resp = app.get('/afterjobs/' + job_id)
-    assert status_resp.text == 'completed|completed'
+    assert status_resp.text == 'completed|completed 2/2 (100%)'
 
     # check error handling
     app.get('/afterjobs/whatever', status=404)
diff --git a/wcs/admin/settings.py b/wcs/admin/settings.py
index 132feff0..cb5e1004 100644
--- a/wcs/admin/settings.py
+++ b/wcs/admin/settings.py
@@ -1037,6 +1037,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                             if f in ('.indexes', '.max_id'):
                                 continue
                             z.write(os.path.join(path, f), os.path.join(d, f))
+                            job.increment_count()
                     if 'datasources' in self.dirs:
                         for ds in NamedDataSource.select():
                             if ds.external == 'agenda':
@@ -1047,6 +1048,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('datasources', str(ds.id)),
                                 ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'formdefs' in self.dirs:
                         for formdef in FormDef.select():
                             node = formdef.export_to_xml(include_id=True)
@@ -1055,6 +1057,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('formdefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'carddefs' in self.dirs:
                         for formdef in CardDef.select():
                             node = formdef.export_to_xml(include_id=True)
@@ -1063,6 +1066,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('carddefs_xml', str(formdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'workflows' in self.dirs:
                         for workflow in Workflow.select():
                             node = workflow.export_to_xml(include_id=True)
@@ -1071,6 +1075,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('workflows_xml', str(workflow.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'blockdefs' in self.dirs:
                         for blockdef in BlockDef.select():
                             node = blockdef.export_to_xml(include_id=True)
@@ -1079,6 +1084,7 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('blockdefs_xml', str(blockdef.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
                     if 'roles' in self.dirs:
                         for role in get_publisher().role_class.select():
                             node = role.export_to_xml(include_id=True)
@@ -1087,16 +1093,21 @@ class SettingsDirectory(QommonSettingsDirectory):
                                 os.path.join('roles_xml', str(role.id)),
                                 b'<?xml version="1.0"?>\n' + ET.tostring(node),
                             )
+                            job.increment_count()
 
                     if self.settings:
                         z.write(os.path.join(self.app_dir, 'config.pck'), 'config.pck')
+                        job.increment_count()
                         for f in os.listdir(self.app_dir):
                             if f.startswith('idp-') and os.path.splitext(f)[-1] in ('.pem', '.xml'):
                                 z.write(os.path.join(self.app_dir, f), f)
+                                job.increment_count()
                         if os.path.exists(os.path.join(self.app_dir, 'config')):
                             for f in os.listdir(os.path.join(self.app_dir, 'config')):
                                 z.write(os.path.join(self.app_dir, 'config', f), os.path.join('config', f))
+                                job.increment_count()
 
+                job.total_count = job.current_count
                 job.file_content = c.getvalue()
                 job.store()
 
diff --git a/wcs/api.py b/wcs/api.py
index 96b0e06b..8fd92438 100644
--- a/wcs/api.py
+++ b/wcs/api.py
@@ -1172,7 +1172,7 @@ class AfterJobDirectory(Directory):
                     'label': self.afterjob.label,
                     'creation_time': self.afterjob.creation_time,
                     'completion_time': self.afterjob.completion_time,
-                    'completion_status': self.afterjob.completion_status,
+                    'completion_status': self.afterjob.get_completion_status(),
                 },
             },
             cls=misc.JSONEncoder,
diff --git a/wcs/backoffice/data_management.py b/wcs/backoffice/data_management.py
index 5d346be6..c5cc30cc 100644
--- a/wcs/backoffice/data_management.py
+++ b/wcs/backoffice/data_management.py
@@ -363,7 +363,10 @@ class ImportFromCsvAfterJob(AfterJob):
     def execute(self):
         self.carddef = self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])
         carddata_class = self.carddef.data_class()
-        for item in self.kwargs['data_lines']:
+        self.total_count = len(self.kwargs['data_lines'])
+        self.store()
+
+        for i, item in enumerate(self.kwargs['data_lines']):
             data_instance = carddata_class()
             user_value = item.pop('_user', None)
             data_instance.user = self.user_lookup(user_value)
@@ -371,6 +374,7 @@ class ImportFromCsvAfterJob(AfterJob):
             data_instance.just_created()
             data_instance.store()
             data_instance.perform_workflow(event='csv-import-created')
+            self.increment_count()
 
     def done_action_url(self):
         carddef = self.kwargs['carddef_class'].get(self.kwargs['carddef_id'])
diff --git a/wcs/backoffice/management.py b/wcs/backoffice/management.py
index 71f7227d..4ccf42d1 100644
--- a/wcs/backoffice/management.py
+++ b/wcs/backoffice/management.py
@@ -3894,7 +3894,7 @@ class MassActionAfterJob(AfterJob):
             return
 
         formdatas = formdef.data_class().get_ids(item_ids, order_by='receipt_time')
-        self.completion_status = f'0/{len(formdatas)}'
+        self.total_count = len(formdatas)
         self.store()
 
         publisher = get_publisher()
@@ -3910,7 +3910,7 @@ class MassActionAfterJob(AfterJob):
                 {
                     'oldest_form': oldest_lazy_form,
                     'mass_action_index': i,
-                    'mass_action_length': len(formdatas),
+                    'mass_action_length': self.total_count,
                 }
             )
             if getattr(action['action'], 'status_action', False):
@@ -3921,8 +3921,7 @@ class MassActionAfterJob(AfterJob):
             else:
                 # global action
                 formdata.perform_global_action(action['action'].id, user)
-            self.completion_status = f'{i + 1}/{len(formdatas)}'
-            self.store()
+            self.increment_count()
 
     def done_action_url(self):
         return self.kwargs['return_url']
@@ -3970,6 +3969,8 @@ class CsvExportAfterJob(AfterJob):
         items, total_count = FormDefUI(formdef).get_listing_items(
             fields, selected_filter, user=user, query=query, criterias=criterias, order_by=order_by
         )
+        self.total_count = total_count
+        self.store()
 
         return self.create_export(formdef, fields, items, total_count)
 
@@ -3980,8 +3981,9 @@ class CsvExportAfterJob(AfterJob):
         if not self.kwargs.get('skip_header_line'):
             csv_output.writerow(self.csv_tuple_heading(fields))
 
-        for filled in items:
+        for i, filled in enumerate(items):
             csv_output.writerow(tuple(x['value'] for x in self.get_spreadsheet_line(fields, filled)))
+            self.increment_count()
 
         self.file_content = output.getvalue()
         self.content_type = 'text/csv'
@@ -4025,6 +4027,7 @@ class OdsExportAfterJob(CsvExportAfterJob):
                     data_field=item['field'],
                     native_value=item['native_value'],
                 )
+            self.increment_count()
 
         output = io.BytesIO()
         workbook.save(output)
diff --git a/wcs/qommon/afterjobs.py b/wcs/qommon/afterjobs.py
index 0c4701ab..10a1f74e 100644
--- a/wcs/qommon/afterjobs.py
+++ b/wcs/qommon/afterjobs.py
@@ -35,9 +35,10 @@ class AfterJobStatusDirectory(Directory):
 
         response = get_response()
         response.set_content_type('text/plain')
-        if not job.completion_status:
+        completion_status = job.get_completion_status()
+        if not completion_status:
             return job.status + '|' + str(_(job.status))
-        return job.status + '|' + str(_(job.status)) + ' ' + job.completion_status
+        return job.status + '|' + str(_(job.status)) + ' ' + str(completion_status)
 
 
 class AfterJob(StorableObject):
@@ -48,7 +49,8 @@ class AfterJob(StorableObject):
     status = None
     creation_time = None
     completion_time = None
-    completion_status = None
+    current_count = None
+    total_count = None
 
     execute = None
 
@@ -73,6 +75,22 @@ class AfterJob(StorableObject):
     def done_action_attributes(self):
         return self.done_button_attributes_arg
 
+    def increment_count(self):
+        self.current_count = (self.current_count or 0) + 1
+        self.store()
+
+    def get_completion_status(self):
+        current_count = self.current_count or 0
+
+        if not current_count:
+            return ''
+
+        if not self.total_count:
+            return _(f'{current_count} (unknown total)')
+
+        percent = int(current_count * 100 / self.total_count)
+        return _(f'{current_count}/{self.total_count} ({percent}%)')
+
     def run(self, spool=False):
         if self.completion_time:
             return
-- 
2.30.2

