From 30ee5474273650a46acdf6edda7612abff02a273 Mon Sep 17 00:00:00 2001
From: Valentin Deniaud <vdeniaud@entrouvert.com>
Date: Wed, 21 Apr 2021 12:05:28 +0200
Subject: [PATCH] manager: display progress while importing users (#50163)

---
 src/authentic2/csv_import.py                  | 15 +++++---
 .../authentic2/manager/user_import.html       | 34 ++++++++++++-------
 .../manager/user_import_report_row.html       | 12 +++++++
 src/authentic2/manager/user_import.py         | 17 ++++++++--
 src/authentic2/manager/user_views.py          |  6 +++-
 tests/test_user_manager.py                    |  5 +++
 6 files changed, 70 insertions(+), 19 deletions(-)
 create mode 100644 src/authentic2/manager/templates/authentic2/manager/user_import_report_row.html

diff --git a/src/authentic2/csv_import.py b/src/authentic2/csv_import.py
index 04c62147..56ece005 100644
--- a/src/authentic2/csv_import.py
+++ b/src/authentic2/csv_import.py
@@ -363,7 +363,7 @@ class UserCsvImporter(object):
             line_error = LineError.from_error(line_error)
         self.errors.append(line_error)
 
-    def run(self, fd_or_str, encoding, ou=None, simulate=False):
+    def run(self, fd_or_str, encoding, ou=None, simulate=False, progress_callback=None):
         self.ou = ou or get_default_ou()
         self.errors = []
         self._missing_roles = set()
@@ -379,7 +379,9 @@ class UserCsvImporter(object):
 
             try:
                 with atomic():
-                    for row in self.rows:
+                    for i, row in enumerate(self.rows):
+                        if progress_callback:
+                            progress_callback(_('importing'), i, len(self.rows))
                         try:
                             if not self.do_import_row(row, unique_map):
                                 self.rows_with_errors += 1
@@ -393,7 +395,10 @@ class UserCsvImporter(object):
             except Simulate:
                 pass
 
-        for action in [parse_csv, self.parse_header_row, self.parse_rows, do_import]:
+        def parse_rows():
+            self.parse_rows(progress_callback)
+
+        for action in [parse_csv, self.parse_header_row, parse_rows, do_import]:
             action()
             if self.errors:
                 break
@@ -514,13 +519,15 @@ class UserCsvImporter(object):
             except (AttributeError, TypeError, KeyError):
                 self.add_error(LineError('unknown-flag', _('unknown flag "%s"'), line=1, column=column))
 
-    def parse_rows(self):
+    def parse_rows(self, progress_callback=None):
         base_form_class = ImportUserForm
         if SOURCE_NAME in self.headers_by_name:
             base_form_class = ImportUserFormWithExternalId
         form_class = modelform_factory(User, fields=self.headers_by_name.keys(), form=base_form_class)
         rows = self.rows = []
         for i, row in enumerate(self.csv_importer.rows[1:]):
+            if progress_callback:
+                progress_callback(_('parsing'), i, len(self.csv_importer.rows))
             csv_row = self.parse_row(form_class, row, line=i + 2)
             self.has_errors = self.has_errors or not (csv_row.is_valid)
             rows.append(csv_row)
diff --git a/src/authentic2/manager/templates/authentic2/manager/user_import.html b/src/authentic2/manager/templates/authentic2/manager/user_import.html
index 80b9b2d7..822c4c84 100644
--- a/src/authentic2/manager/templates/authentic2/manager/user_import.html
+++ b/src/authentic2/manager/templates/authentic2/manager/user_import.html
@@ -56,19 +56,29 @@
     </thead>
     <tbody>
       {% for report in reports %}
-        <tr data-uuid="{{ report.uuid }}">
-          <td class="created">
-            {% if report.state != report.STATE_RUNNING %}
-              <a href="{% url "a2-manager-users-import-report" import_uuid=user_import.uuid report_uuid=report.uuid %}">{{ report.created }}</a>
-            {% else %}
-              {{ report.created }}
-            {% endif %}
-          </td>
-          <td class="state">{{ report.state_display }}</td>
-          <td class="applied">{% if not report.simulate %}<span class="icon-check"></span>{% endif %}</td>
-          <td class="delete-action">{% if report.simulate %}<form method="post" id="delete-form-{{ report.uuid }}">{% csrf_token %}<button name="delete" value="{{ report.uuid }}">{% trans "Delete" %}</button></form>{% endif %}</td>
-        </tr>
+      <tr data-uuid="{{ report.uuid }}" data-url="{% url "a2-manager-users-import-report" import_uuid=user_import.uuid report_uuid=report.uuid %}">
+        {% include "authentic2/manager/user_import_report_row.html" %}
+      </tr>
       {% endfor %}
     </tbody>
   </table>
+  <script>
+  function autorefresh(el) {
+    $.ajax({
+      url: $(el).data('url'),
+      success: function(html) {
+        $(el).html(html);
+      },
+    });
+  }
+
+  function scheduleRefreshes() {
+    $('tbody tr').each(function() {
+      if($(this).children('td.state.running').length > 0)
+        setInterval(autorefresh, 2000, this);
+    });
+  }
+
+  $(document).ready(scheduleRefreshes);
+  </script>
 {% endblock %}
diff --git a/src/authentic2/manager/templates/authentic2/manager/user_import_report_row.html b/src/authentic2/manager/templates/authentic2/manager/user_import_report_row.html
new file mode 100644
index 00000000..86f5653e
--- /dev/null
+++ b/src/authentic2/manager/templates/authentic2/manager/user_import_report_row.html
@@ -0,0 +1,12 @@
+{% load i18n %}
+
+<td class="created">
+  {% if report.state != report.STATE_RUNNING %}
+    <a href="{% url "a2-manager-users-import-report" import_uuid=user_import.uuid report_uuid=report.uuid %}">{{ report.created }}</a>
+  {% else %}
+    {{ report.created }}
+  {% endif %}
+</td>
+<td class="state{% if report.state == report.STATE_RUNNING %} running{% endif %}">{{ report.state_display }}</td>
+<td class="applied">{% if not report.simulate %}<span class="icon-check"></span>{% endif %}</td>
+<td class="delete-action">{% if report.simulate %}<form method="post" id="delete-form-{{ report.uuid }}">{% csrf_token %}<button name="delete" value="{{ report.uuid }}">{% trans "Delete" %}</button></form>{% endif %}</td>
diff --git a/src/authentic2/manager/user_import.py b/src/authentic2/manager/user_import.py
index a8d53636..dba91e49 100644
--- a/src/authentic2/manager/user_import.py
+++ b/src/authentic2/manager/user_import.py
@@ -166,7 +166,10 @@ class Report(object):
     @property
     def state_display(self):
         state = self.data['state']
-        return self.STATES.get(state, state)
+        state_display = self.STATES.get(state, state)
+        if state == self.STATE_RUNNING and 'progress' in self.data:
+            state_display = '%s (%s)' % (state_display, self.data['progress'])
+        return state_display
 
     @property
     @contextlib.contextmanager
@@ -205,6 +208,12 @@ class Report(object):
             else:
                 yield None
 
+        def callback(status, line, total):
+            if total < 1 or not self.exists():
+                return
+            with self.data_update as data:
+                data['progress'] = '%s, %d%%' % (status, round((line / total) * 100))
+
         def thread_worker():
             from authentic2.csv_import import UserCsvImporter
 
@@ -218,7 +227,11 @@ class Report(object):
                 try:
                     with publik_provisionning():
                         importer.run(
-                            fd, encoding=self.data['encoding'], ou=self.data['ou'], simulate=simulate
+                            fd,
+                            encoding=self.data['encoding'],
+                            ou=self.data['ou'],
+                            simulate=simulate,
+                            progress_callback=callback,
                         )
                 except Exception as e:
                     logger.exception('error during report %s:%s run', self.user_import.uuid, self.uuid)
diff --git a/src/authentic2/manager/user_views.py b/src/authentic2/manager/user_views.py
index 35d6c796..cf0ba456 100644
--- a/src/authentic2/manager/user_views.py
+++ b/src/authentic2/manager/user_views.py
@@ -902,7 +902,6 @@ class UserImportReportView(MediaMixin, PermissionMixin, TemplateView):
     form_class = UserEditImportForm
     permissions = ['custom_user.admin_user']
     permissions_global = True
-    template_name = 'authentic2/manager/user_import_report.html'
 
     def dispatch(self, request, import_uuid, report_uuid):
         from authentic2.manager.user_import import UserImport
@@ -926,6 +925,11 @@ class UserImportReportView(MediaMixin, PermissionMixin, TemplateView):
             ctx['report_title'] = _('Execution')
         return ctx
 
+    def get_template_names(self):
+        if self.request.is_ajax():
+            return ['authentic2/manager/user_import_report_row.html']
+        return ['authentic2/manager/user_import_report.html']
+
 
 user_import_report = UserImportReportView.as_view()
 
diff --git a/tests/test_user_manager.py b/tests/test_user_manager.py
index bcd28ce3..58783d26 100644
--- a/tests/test_user_manager.py
+++ b/tests/test_user_manager.py
@@ -500,6 +500,11 @@ x,x,x,x'''.encode(
 
     response = response.follow()
 
+    report_url = response.pyquery('tr[data-uuid="%s"]' % uuid).attr('data-url')
+    ajax_resp = app.get(report_url, xhr=True)
+    assert len(ajax_resp.pyquery('td')) == 4
+    assert 'body' not in ajax_resp
+
     def assert_timeout(duration, wait_function):
         start = time.time()
         while True:
-- 
2.20.1