From 0fb14859c18c9d2b21a7d38aadf40b215ec5baff Mon Sep 17 00:00:00 2001 From: Valentin Deniaud Date: Tue, 9 Mar 2021 17:27:01 +0100 Subject: [PATCH 3/3] manager: export users asynchronously (#43153) --- .../authentic2/manager/css/indicator.gif | Bin 0 -> 1553 bytes .../static/authentic2/manager/css/style.css | 5 ++ .../authentic2/manager/user_export.html | 45 +++++++++++ src/authentic2/manager/urls.py | 4 + src/authentic2/manager/user_export.py | 71 ++++++++++++++++++ src/authentic2/manager/user_views.py | 63 +++++++++++++++- src/authentic2/utils/__init__.py | 4 +- src/authentic2/utils/spooler.py | 37 +++++++++ tests/test_user_manager.py | 52 +++++++++++++ 9 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 src/authentic2/manager/static/authentic2/manager/css/indicator.gif create mode 100644 src/authentic2/manager/templates/authentic2/manager/user_export.html create mode 100644 src/authentic2/utils/spooler.py diff --git a/src/authentic2/manager/static/authentic2/manager/css/indicator.gif b/src/authentic2/manager/static/authentic2/manager/css/indicator.gif new file mode 100644 index 0000000000000000000000000000000000000000..085ccaecaf5fa5c34bc14cd2c2ed5cbbd8e25dcb GIT binary patch literal 1553 zcma)+TTl~c6vwlh>nb99Af5rT)t{mCEg5urg=A(g z{C|6SPb~9Xage|wB`SrZk2FOMYM!buln2sX?5Y+T78iB(Zu9cS7|LZyZ++}u$^oi1 z_j@S}bW9OzU2R+RMy&~OT>X-oZ98$jq#ogNfJ!BM-42wHGZk*6s2KD}U*IA%epmxb zm}|6BK9YoIF;*xSL!+z@<64lB7->LTW2Vi4ostCA(z&2XniwNIv}fFo-`MbG;)u4G z^p@F!)|9HhZprHd_vXjDoxs6WkK-6P0@lfxnGT>*p(QHoUV=u1FAqb@b%*W=a3{`LsH5k^AvQNL>6fPpy#oU(&MuH(*aEX4b35*} zn4n7)`I2U%=+Z=?BVZQ?vjQFW4gD@~XSOO6b{qu81`4&LFuU2(ilxW+1|ZkNMnWe79C$gs zWT?Ele|HR{JGPe)5BTW>0Ey?-Ls6S#GoV0tbt6ku7B&*0 z;i9QM$W1Rj*rRIdceL)rAOSl+sDe3LkB87<%){;ZdHp6|SNlopDXRx< zxBDF9-lTo&v`8$humFygUij@qgT=Qzhj8{ym2-{Xciwqq_Xwk%=O3B-MNAL_6e`3U zyxwmXex4`g0^1RYw~Dth3av3Dl^AAlpO3mG!nLr#&ZZ7c_wUboI+deC+&%TFjK2Lm z!Y&f1h|T_On%RCV&=4bx`!>(YezqGVhl&QpED?N6GV)HmzJ9&rh$x*i?*@o9#6QI< z5ZI_MRX;0+pY8$`j)eF#TlUyG(eE%E7S!rj;mj^M5vhUicPm zVWQ2z+imFyg}SRABmOBY_@osR!>7Ov!ioK`NB6_Rv}7Ud?35ed5Sb@?yND?kv~RCa wqs^a3Sh>&&L4)!LKI?D2&k@))k(LESaga|C278ChSzn3NWVkcuNoY&{0f?~U_5c6? literal 0 HcmV?d00001 diff --git a/src/authentic2/manager/static/authentic2/manager/css/style.css b/src/authentic2/manager/static/authentic2/manager/css/style.css index 98a01015..d41fa464 100644 --- a/src/authentic2/manager/static/authentic2/manager/css/style.css +++ b/src/authentic2/manager/static/authentic2/manager/css/style.css @@ -264,3 +264,8 @@ form .widget span.select2-container { .journal-list--timestamp-column { white-space: pre; } + +span.activity { + background: url(indicator.gif) no-repeat top right; + padding-right: 30px; +} diff --git a/src/authentic2/manager/templates/authentic2/manager/user_export.html b/src/authentic2/manager/templates/authentic2/manager/user_export.html new file mode 100644 index 00000000..f66ebb71 --- /dev/null +++ b/src/authentic2/manager/templates/authentic2/manager/user_export.html @@ -0,0 +1,45 @@ +{% extends "authentic2/manager/base.html" %} +{% load i18n gadjo staticfiles %} + +{% block page-title %}{{ block.super }} - {% trans "User export" %}{% endblock %} + +{% block appbar %} +

Users export

+{% endblock %} + +{% block breadcrumb %} + {{ block.super }} + {% trans 'Users' %} + +{% endblock %} + +{% block main %} +
+
+

{% trans "Preparing CSV export file..." %}

+ {% trans "Progress:" %} 0% +
+
+

{% trans "Export completed." %}

+

{% trans "Download CSV" %}

+
+
+ +{% endblock %} diff --git a/src/authentic2/manager/urls.py b/src/authentic2/manager/urls.py index 244f242d..ea4113c7 100644 --- a/src/authentic2/manager/urls.py +++ b/src/authentic2/manager/urls.py @@ -38,6 +38,10 @@ urlpatterns = required( url(r'^users/$', user_views.users, name='a2-manager-users'), url(r'^users/export/(?Pcsv)/$', user_views.users_export, name='a2-manager-users-export'), + url(r'^users/export-progress/(?P[a-z0-9-]+)/$', + user_views.users_export_progress, name='a2-manager-users-export-progress'), + url(r'^users/export/(?P[a-z0-9-]+)/$', + user_views.users_export_file, name='a2-manager-users-export-file'), url(r'^users/add/$', user_views.user_add_default_ou, name='a2-manager-user-add-default-ou'), url(r'^users/add/choose-ou/$', user_views.user_add_choose_ou, diff --git a/src/authentic2/manager/user_export.py b/src/authentic2/manager/user_export.py index c364f212..baeb04c7 100644 --- a/src/authentic2/manager/user_export.py +++ b/src/authentic2/manager/user_export.py @@ -16,14 +16,19 @@ import collections import datetime +import os +import pickle +import uuid import tablib from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType +from django.core.files.storage import default_storage from authentic2.manager.resources import UserResource from authentic2.models import Attribute, AttributeValue +from authentic2.utils import batch_queryset def iso(rec): @@ -78,3 +83,69 @@ def get_user_dataset(qs): dataset.append(create_record(user)) return dataset + + +class UserExport(object): + def __init__(self, uuid): + self.uuid = uuid + self.path = os.path.join(self.base_path(), self.uuid) + self.export_path = os.path.join(self.path, 'export.csv') + self.progress_path = os.path.join(self.path, 'progress') + + @classmethod + def base_path(self): + path = default_storage.path('user_exports') + if not os.path.exists(path): + os.makedirs(path) + return path + + @property + def exists(self): + return os.path.exists(self.path) + + @classmethod + def new(cls): + export = cls(str(uuid.uuid4())) + os.makedirs(export.path) + return export + + @property + def csv(self): + return open(self.export_path, 'r') + + def set_export_content(self, content): + with open(self.export_path, 'w') as f: + f.write(content) + + @property + def progress(self): + progress = 0 + if os.path.exists(self.progress_path): + with open(self.progress_path, 'r') as f: + progress = f.read() + return int(progress) if progress else 0 + + def set_progress(self, progress): + with open(self.progress_path, 'w') as f: + f.write(str(progress)) + + +def export_users_to_file(uuid, query): + export = UserExport(uuid) + qs = get_user_model().objects.all() + qs.query = pickle.loads(query.encode('latin-1')) + count = qs.count() or 1 + + def callback(progress): + export.set_progress(round(progress / count * 100)) + + qs = batch_queryset(qs, progress_callback=callback) + dataset = get_user_dataset(qs) + + if hasattr(dataset, 'csv'): + # compatiblity for tablib < 0.11 + csv = dataset.csv + else: + csv = dataset.export('csv') + export.set_export_content(csv) + export.set_progress(100) diff --git a/src/authentic2/manager/user_views.py b/src/authentic2/manager/user_views.py index 24fd6684..db3c5c10 100644 --- a/src/authentic2/manager/user_views.py +++ b/src/authentic2/manager/user_views.py @@ -17,8 +17,10 @@ import base64 import collections import operator +import pickle +import sys -from django.db import models, transaction +from django.db import models, transaction, connection from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _, pgettext_lazy, ugettext from django.utils.html import format_html @@ -28,10 +30,10 @@ from django.core.mail import EmailMultiAlternatives from django.template import loader from django.contrib.auth import get_user_model, REDIRECT_FIELD_NAME from django.contrib import messages -from django.views.generic import FormView, TemplateView, DetailView +from django.views.generic import FormView, TemplateView, DetailView, View from django.views.generic.edit import BaseFormView from django.views.generic.detail import SingleObjectMixin -from django.http import Http404, FileResponse, HttpResponseRedirect +from django.http import Http404, FileResponse, HttpResponseRedirect, HttpResponse from django.shortcuts import get_object_or_404 from authentic2.models import Attribute, PasswordReset @@ -55,7 +57,7 @@ from .forms import (UserSearchForm, UserAddForm, UserEditForm, from .resources import UserResource from .utils import get_ou_count, has_show_username from .journal_views import BaseJournalView -from .user_export import get_user_dataset +from .user_export import get_user_dataset, UserExport from . import app_settings User = get_user_model() @@ -481,6 +483,20 @@ class UsersExportView(ExportMixin, UsersView): return self._dataset.csv return self._dataset.export('csv') + def get(self, request, *args, **kwargs): + if 'uwsgi' in sys.modules: + from authentic2.utils.spooler import export_users + + export = UserExport.new() + query = pickle.dumps(self.get_table_data().query) + # workaround django-uwsgi encoding issue https://github.com/unbit/django-uwsgi/issues/10 + query = query.decode('latin1').encode('utf-8') + tenant = getattr(connection, 'tenant', None) + export_users.spool(domain=getattr(tenant, 'domain_url', None), uuid=export.uuid, query=query) + return redirect(request, 'a2-manager-users-export-progress', kwargs={'uuid': export.uuid}) + else: + return super().get(request, *args, **kwargs) + def get_dataset(self): self._dataset = get_user_dataset(self.get_data()) return self @@ -489,6 +505,45 @@ class UsersExportView(ExportMixin, UsersView): users_export = UsersExportView.as_view() +class UsersExportFileView(ExportMixin, PermissionMixin, View): + permissions = ['custom_user.view_user'] + + def get(self, request, *args, **kwargs): + self.export = UserExport(kwargs.get('uuid')) + if not self.export.exists: + raise Http404() + return super().get(request, *args, format='csv', **kwargs) + + def get_dataset(self): + return self.export + + +users_export_file = UsersExportFileView.as_view() + + +class UsersExportProgressView(MediaMixin, TemplateView): + template_name = 'authentic2/manager/user_export.html' + + def get(self, request, *args, **kwargs): + self.uuid = kwargs.get('uuid') + export = UserExport(self.uuid) + if not export.exists: + raise Http404() + + if request.is_ajax(): + return HttpResponse(export.progress) + + return super().get(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + ctx['uuid'] = self.uuid + return ctx + + +users_export_progress = UsersExportProgressView.as_view() + + class UserChangePasswordView(BaseEditView): template_name = 'authentic2/manager/form.html' model = get_user_model() diff --git a/src/authentic2/utils/__init__.py b/src/authentic2/utils/__init__.py index cf8a81d7..1eba362f 100644 --- a/src/authentic2/utils/__init__.py +++ b/src/authentic2/utils/__init__.py @@ -879,12 +879,14 @@ def batch(iterable, size): yield chain([batchiter.next()], batchiter) -def batch_queryset(qs, size=1000): +def batch_queryset(qs, size=1000, progress_callback=None): '''Batch prefetched potentially very large queryset, it's a middle ground between using .iterator() which cannot be prefetched and prefetching a full table, which can take a larte place in memory. ''' for i in count(0): + if progress_callback: + progress_callback(i * size) chunk = qs[i * size:(i + 1) * size] if not chunk: break diff --git a/src/authentic2/utils/spooler.py b/src/authentic2/utils/spooler.py new file mode 100644 index 00000000..e0e5baf9 --- /dev/null +++ b/src/authentic2/utils/spooler.py @@ -0,0 +1,37 @@ +# authentic2 - versatile identity manager +# Copyright (C) 2010-2021 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from uwsgidecorators import spool + +from django.db import connection + +from authentic2.manager.user_export import export_users_to_file + + +def set_connection(domain): + from hobo.multitenant.middleware import TenantMiddleware + + tenant = TenantMiddleware.get_tenant_by_hostname(domain) + connection.set_tenant(tenant) + + +@spool +def export_users(args): + if args.get('domain'): + # multitenant installation + set_connection(args['domain']) + + export_users_to_file(args['uuid'], args['query']) diff --git a/tests/test_user_manager.py b/tests/test_user_manager.py index d2f25b4f..6c6bae47 100644 --- a/tests/test_user_manager.py +++ b/tests/test_user_manager.py @@ -19,7 +19,9 @@ from __future__ import unicode_literals import csv import datetime +import mock import re +import sys import time from urllib.parse import urlparse @@ -53,6 +55,29 @@ from .utils import login, get_link_from_mail, logout OU = get_ou_model() +@pytest.fixture +def mock_uwsgi(): + from authentic2.manager.user_export import export_users_to_file + + def run_now(f): + def spool(**kwargs): + for k, v in kwargs.items(): + # workaround django-uwsgi encoding issue https://github.com/unbit/django-uwsgi/issues/10 + if isinstance(v, bytes): + kwargs[k] = v.decode('utf-8') + f(kwargs) + + f.spool = spool + return f + + sys.modules['uwsgi'] = mock.MagicMock() + uwsgidecorators = mock.MagicMock() + uwsgidecorators.spool = run_now + with mock.patch.dict('sys.modules', uwsgidecorators=uwsgidecorators): + yield + del sys.modules['uwsgi'] + + def visible_users(response): return set(elt.text for elt in response.pyquery('td.username')) @@ -423,6 +448,33 @@ def test_user_table(app, admin, user_ou1, ou1): assert response.pyquery('td.username') +def test_export_csv_async(settings, app, superuser, mock_uwsgi): + users = [User(username='user%s' % i) for i in range(10)] + User.objects.bulk_create(users) + + resp = login(app, superuser, reverse('a2-manager-users')) + resp = resp.click('CSV') + url = resp.url + resp = resp.follow() + assert 'Preparing CSV export file...' in resp.text + assert '0' in resp.text + + # spooler mock is in fact synchronous, csv file is already there + resp = resp.click('Download CSV') + table = list(csv.reader(resp.text.splitlines())) + assert len(table) == User.objects.count() + 1 + + # ajax call returns 100% progress + resp = app.get(url, xhr=True) + assert resp.text == '100' + + resp = app.get('/manage/users/?search-text=user1') + resp = resp.click('CSV').follow() + resp = resp.click('Download CSV') + table = list(csv.reader(resp.text.splitlines())) + assert len(table) == 3 # user1 and superuser match + + @pytest.mark.parametrize('encoding', ['utf-8-sig', 'cp1252', 'iso-8859-15']) def test_user_import(encoding, transactional_db, app, admin, ou1, admin_ou1): Attribute.objects.create(name='phone', kind='phone_number', label='Numéro de téléphone') -- 2.20.1