From 355e2c432f37aeee30becb662ed94c5b0b7b28b8 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Sat, 15 Jun 2019 15:59:31 +0200 Subject: [PATCH 3/3] manager: add user import views (#32833) --- debian/control | 3 +- setup.py | 1 + src/authentic2/manager/forms.py | 74 ++++++ .../authentic2/manager/css/user_import.css | 51 ++++ .../authentic2/manager/user_import.html | 67 ++++++ .../manager/user_import_report.html | 126 ++++++++++ .../authentic2/manager/user_imports.html | 47 ++++ .../templates/authentic2/manager/users.html | 5 + src/authentic2/manager/urls.py | 8 + src/authentic2/manager/user_import.py | 227 ++++++++++++++++++ src/authentic2/manager/user_views.py | 132 +++++++++- tests/test_manager_user_import.py | 90 +++++++ 12 files changed, 826 insertions(+), 5 deletions(-) create mode 100644 src/authentic2/manager/static/authentic2/manager/css/user_import.css create mode 100644 src/authentic2/manager/templates/authentic2/manager/user_import.html create mode 100644 src/authentic2/manager/templates/authentic2/manager/user_import_report.html create mode 100644 src/authentic2/manager/templates/authentic2/manager/user_imports.html create mode 100644 src/authentic2/manager/user_import.py create mode 100644 tests/test_manager_user_import.py diff --git a/debian/control b/debian/control index 0cfdd507..4b6753ae 100644 --- a/debian/control +++ b/debian/control @@ -31,7 +31,8 @@ Depends: ${misc:Depends}, ${python:Depends}, python-pil, python-tablib, python-chardet, - python-attr + python-attr, + python-atomicwrites Breaks: python-authentic2-auth-fc (<< 0.26) Replaces: python-authentic2-auth-fc (<< 0.26) Provides: ${python:Provides}, python-authentic2-auth-fc diff --git a/setup.py b/setup.py index 43193a43..e16946e7 100755 --- a/setup.py +++ b/setup.py @@ -142,6 +142,7 @@ setup(name="authentic2", 'tablib', 'chardet', 'attrs', + 'atomicwrites', ], zip_safe=False, classifiers=[ diff --git a/src/authentic2/manager/forms.py b/src/authentic2/manager/forms.py index 3f851ce7..b618d071 100644 --- a/src/authentic2/manager/forms.py +++ b/src/authentic2/manager/forms.py @@ -45,6 +45,7 @@ from authentic2 import app_settings as a2_app_settings from . import fields, app_settings, utils User = get_user_model() +OU = get_ou_model() logger = logging.getLogger(__name__) @@ -715,3 +716,76 @@ class UserChangeEmailForm(CssClass, FormWithRequest, forms.ModelForm): class SiteImportForm(forms.Form): site_json = forms.FileField( label=_('Site Export File')) + + +ENCODINGS = [ + ('utf-8', _('UTF-8')), + ('cp1252', _('CP-1252 - Windows France')), + ('iso-8859-1', _('Latin 1')), + ('iso-8859-15', _('Latin 15')), + ('detect', _('Detect encoding')), +] + + +class UserImportForm(forms.Form): + import_file = forms.FileField( + label=_('Import file'), + help_text=_('A CSV file')) + encoding = forms.ChoiceField( + label=_('Encoding'), + choices=ENCODINGS) + ou = forms.ModelChoiceField( + label=_('Organizational Unit'), + queryset=OU.objects.all()) + + +class UserNewImportForm(UserImportForm): + def clean(self): + from authentic2.csv_import import CsvImporter + + import_file = self.cleaned_data['import_file'] + encoding = self.cleaned_data['encoding'] + # force seek(0) + import_file.open() + importer = CsvImporter() + if not importer.run(import_file, encoding): + raise forms.ValidationError(importer.error.description or importer.error.code) + self.cleaned_data['rows_count'] = len(importer.rows) + + def save(self): + from . import user_import + + import_file = self.cleaned_data['import_file'] + import_file.open() + new_import = user_import.UserImport.new( + import_file=import_file, + encoding=self.cleaned_data['encoding']) + with new_import.meta_update as meta: + meta['filename'] = import_file.name + meta['ou'] = self.cleaned_data['ou'] + meta['rows_count'] = self.cleaned_data['rows_count'] + return new_import + + +class UserEditImportForm(UserImportForm): + def __init__(self, *args, **kwargs): + self.user_import = kwargs.pop('user_import') + initial = kwargs.setdefault('initial', {}) + initial['encoding'] = self.user_import.meta['encoding'] + initial['ou'] = self.user_import.meta['ou'] + super(UserEditImportForm, self).__init__(*args, **kwargs) + del self.fields['import_file'] + + def clean(self): + from authentic2.csv_import import CsvImporter + encoding = self.cleaned_data['encoding'] + with self.user_import.import_file as fd: + importer = CsvImporter(fd, encoding) + if not importer.run(): + raise forms.ValidationError(importer.error.description or importer.error.code) + self.cleaned_data['rows_count'] = len(importer.rows) + + def save(self): + with self.user_import.meta_update as meta: + meta['ou'] = self.cleaned_data['ou'] + meta['encoding'] = self.cleaned_data['encoding'] diff --git a/src/authentic2/manager/static/authentic2/manager/css/user_import.css b/src/authentic2/manager/static/authentic2/manager/css/user_import.css new file mode 100644 index 00000000..23b6ed20 --- /dev/null +++ b/src/authentic2/manager/static/authentic2/manager/css/user_import.css @@ -0,0 +1,51 @@ +#import-report-table tr.row-valid td, .legend-row-valid { + background-color: #d5f5e3 ; +} + +#import-report-table tr.row-invalid td, .legend-row-invalid { + background-color: #ff4408; +} + +#import-report-table tr td.cell-action-updated, .legend-cell-action-updated { + background-color: #abebc6; +} + +#import-report-table tr td.cell-errors, .legend-cell-errors { + background-color: #cd6155; +} +.header-flag-key::after { + content: "\f084"; /* fa-key */ + font-family: FontAwesome; + padding-left: 1ex; +} + +.header-flag-unique::after, +.header-flag-globally-unique::after +{ + content: "\f0cd"; /* fa-underline */ + font-family: FontAwesome; + padding-left: 1ex; +} + +.header-flag-create::after { + content: "\f0fe"; /* fa-plus-square */ + font-family: FontAwesome; + padding-left: 1ex; +} + +.header-flag-update::after { + content: "\f040"; /* fa-pencil */ + font-family: FontAwesome; + padding-left: 1ex; +} + +.header-flag-verified::after { + content: "\f023"; /* fa-lock */ + font-family: FontAwesome; + padding-left: 1ex; +} + +span.icon-check::after { + content: "\f00c"; + font-family: FontAwesome; +} diff --git a/src/authentic2/manager/templates/authentic2/manager/user_import.html b/src/authentic2/manager/templates/authentic2/manager/user_import.html new file mode 100644 index 00000000..8a64f507 --- /dev/null +++ b/src/authentic2/manager/templates/authentic2/manager/user_import.html @@ -0,0 +1,67 @@ +{% extends "authentic2/manager/base.html" %} +{% load i18n gadjo staticfiles %} + +{% block page-title %}{{ block.super }} - {% trans "Import Users" %}{% endblock %} + +{% block css %} +{{ block.super }} + +{% endblock %} + +{% block breadcrumb %} + {{ block.super }} + {% trans 'Users' %} + {% trans 'Import' %} + {% trans "User Import" %} {{ user_import.created }} +{% endblock %} + +{% block sidebar %} + +{% endblock %} + +{% block main %} +

{% trans "User Import" %} - {{ user_import.created }} - {{ user_import.user }}

+

{% trans "Rows count:" %} {{ user_import.rows_count }}

+

{% trans "Reports" %}

+ + + + + + + + + + + {% for report in reports %} + + + + + + + {% endfor %} + +
{% trans "Creation date" %}{% trans "State" %}{% trans "Imported" %}
{% if report.state != 'running' %}{{ report.created }}{% else %}{{ report.created }}{% endif %}{{ report.state }} {% if report.state == 'error' %}"{{ report.exception }}"{% endif %}{% if not report.simulate %}{% endif %}{% if report.simulate %}
{% csrf_token %}
{% endif %}
+{% endblock %} diff --git a/src/authentic2/manager/templates/authentic2/manager/user_import_report.html b/src/authentic2/manager/templates/authentic2/manager/user_import_report.html new file mode 100644 index 00000000..685317c2 --- /dev/null +++ b/src/authentic2/manager/templates/authentic2/manager/user_import_report.html @@ -0,0 +1,126 @@ +{% extends "authentic2/manager/base.html" %} +{% load i18n gadjo staticfiles %} + +{% block page-title %}{{ block.super }} - {% trans "Import Users" %}{% endblock %} + +{% block css %} +{{ block.super }} + +{% endblock %} + +{% block breadcrumb %} + {{ block.super }} + {% trans 'Users' %} + {% trans 'Import' %} + {% trans "User Import" %} {{ user_import.created }} + {{ report_title }} {{ report.created }} +{% endblock %} + +{% block sidebar %} + +{% endblock %} + +{% block main %} +

{{ report_title }} - {{ report.created }} - {{ report.state }}

+ {% if report.exception %} +

{% trans "Exception:" %} {{ report.exception}}

+ {% endif %} + {% if report.importer %} + {% with importer=report.importer %} + {% if importer.errors %} +

{% trans "Errors" %}

+ + {% endif %} + {% if importer.rows %} + + + + + {% for header in importer.headers %} + + {% endfor %} + + + + + {% for row in importer.rows %} + + + {% for cell in row %} + + {% endfor %} + + + {% endfor %} + +
{% trans "Line" %} + {{ header.name }} + {% for flag in header.flags %} + + {% endfor %} + {% trans "Action" %}
{{ row.line }} + {{ cell.value }} + {% firstof row.action "-" %}
+ {% else %} +

{% trans "No row analysed." %}

+ {% endif %} + {% endwith %} + {% endif %} +{% endblock %} diff --git a/src/authentic2/manager/templates/authentic2/manager/user_imports.html b/src/authentic2/manager/templates/authentic2/manager/user_imports.html new file mode 100644 index 00000000..446ee6d7 --- /dev/null +++ b/src/authentic2/manager/templates/authentic2/manager/user_imports.html @@ -0,0 +1,47 @@ +{% extends "authentic2/manager/base.html" %} +{% load i18n gadjo staticfiles %} + +{% block page-title %}{{ block.super }} - {% trans "Import Users" %}{% endblock %} + +{% block breadcrumb %} + {{ block.super }} + {% trans 'Users' %} + {% trans 'Import Users' %} +{% endblock %} + +{% block sidebar %} + +{% endblock %} + +{% block content %} +

{% trans "Imports" %}

+ + + + + + + + + + + + {% for import in imports %} + + + + + + + + {% endfor %} + +
{% trans "Filename" %}{% trans "Creation date" %}{% trans "By" %}{% trans "Rows" %}
{{ import.filename }}{{ import.created }}{{ import.user }}{{ import.rows_count }}
{% csrf_token %}
+{% endblock %} diff --git a/src/authentic2/manager/templates/authentic2/manager/users.html b/src/authentic2/manager/templates/authentic2/manager/users.html index 34489519..f07c975e 100644 --- a/src/authentic2/manager/templates/authentic2/manager/users.html +++ b/src/authentic2/manager/templates/authentic2/manager/users.html @@ -6,6 +6,7 @@ {% block appbar %} {{ block.super }} + {% if add_ou %} {% endif %} + + {% endblock %} {% block breadcrumb %} diff --git a/src/authentic2/manager/urls.py b/src/authentic2/manager/urls.py index 57b95cb9..d64892bd 100644 --- a/src/authentic2/manager/urls.py +++ b/src/authentic2/manager/urls.py @@ -39,6 +39,14 @@ urlpatterns = required( user_views.users_export, name='a2-manager-users-export'), url(r'^users/add/$', user_views.user_add_default_ou, name='a2-manager-user-add-default-ou'), + url(r'^users/import/$', + user_views.user_imports, name='a2-manager-users-imports'), + url(r'^users/import/(?P[a-z0-9]+)/download/(?P.*)$', + user_views.user_import, name='a2-manager-users-import-download'), + url(r'^users/import/(?P[a-z0-9]+)/$', + user_views.user_import, name='a2-manager-users-import'), + url(r'^users/import/(?P[a-z0-9]+)/(?P[a-z0-9]+)/$', + user_views.user_import_report, name='a2-manager-users-import-report'), url(r'^users/(?P\d+)/add/$', user_views.user_add, name='a2-manager-user-add'), url(r'^users/(?P\d+)/$', user_views.user_detail, diff --git a/src/authentic2/manager/user_import.py b/src/authentic2/manager/user_import.py new file mode 100644 index 00000000..0f94ea45 --- /dev/null +++ b/src/authentic2/manager/user_import.py @@ -0,0 +1,227 @@ +# authentic2 - versatile identity manager +# Copyright (C) 2010-2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import unicode_literals + +import base64 +import contextlib +import datetime +import logging +import os +import pickle +import shutil +import uuid +import threading + +from atomicwrites import atomic_write + +from django.core.files.storage import default_storage +from django.utils import six +from django.utils.functional import cached_property +from django.utils.timezone import utc + +logger = logging.getLogger(__name__) + + +def new_id(): + return (base64.b32encode(uuid.uuid4().get_bytes()) + .strip('=') + .lower() + .decode('ascii')) + + +class UserImport(object): + def __init__(self, uuid): + self.uuid = uuid + self.path = os.path.join(self.base_path(), self.uuid) + self.import_path = os.path.join(self.path, 'content') + self.meta_path = os.path.join(self.path, 'meta.pck') + + def exists(self): + return os.path.exists(self.import_path) and os.path.exists(self.meta_path) + + @cached_property + def created(self): + return datetime.datetime.fromtimestamp(os.path.getctime(self.path), utc) + + @property + def import_file(self): + return open(self.import_path, 'rb') + + @cached_property + def meta(self): + meta = {} + if os.path.exists(self.meta_path): + with open(self.meta_path) as fd: + meta = pickle.load(fd) + return meta + + @property + @contextlib.contextmanager + def meta_update(self): + try: + yield self.meta + finally: + with atomic_write(self.meta_path, overwrite=True) as fd: + pickle.dump(self.meta, fd) + + @classmethod + def base_path(self): + path = default_storage.path('user_imports') + if not os.path.exists(path): + os.makedirs(path) + return path + + @classmethod + def new(cls, import_file, encoding): + o = cls(new_id()) + os.makedirs(o.path) + with open(o.import_path, 'wb') as fd: + import_file.seek(0) + fd.write(import_file.read()) + with o.meta_update as meta: + meta['encoding'] = encoding + return o + + @classmethod + def all(cls): + for subpath in os.listdir(cls.base_path()): + user_import = UserImport(subpath) + if user_import.exists(): + yield user_import + + @property + def reports(self): + return Reports(self) + + def __getattr__(self, name): + try: + return self.meta[name] + except KeyError: + raise AttributeError(name) + + def delete(self): + if self.exists(): + shutil.rmtree(self.path) + + +class Report(object): + def __init__(self, user_import, uuid): + self.user_import = user_import + self.uuid = uuid + self.path = os.path.join(self.user_import.path, '%s%s' % (Reports.PREFIX, uuid)) + + @cached_property + def created(self): + return datetime.datetime.fromtimestamp(os.path.getctime(self.path), utc) + + @cached_property + def data(self): + data = {} + if os.path.exists(self.path): + with open(self.path) as fd: + data = pickle.load(fd) + return data + + @property + @contextlib.contextmanager + def data_update(self): + try: + yield self.data + finally: + with atomic_write(self.path, overwrite=True) as fd: + pickle.dump(self.data, fd) + + @classmethod + def new(cls, user_import): + report = cls(user_import, new_id()) + with report.data_update as data: + data['encoding'] = user_import.meta['encoding'] + data['ou'] = user_import.meta.get('ou') + data['state'] = 'waiting' + return report + + def run(self, start=True, simulate=False): + assert self.data.get('state') == 'waiting' + + with self.data_update as data: + data['simulate'] = simulate + + def target(): + from authentic2.csv_import import UserCsvImporter + + with self.user_import.import_file as fd: + importer = UserCsvImporter() + try: + importer.run(fd, + encoding=self.data['encoding'], + ou=self.data['ou'], + simulate=simulate) + except Exception as e: + logger.exception('error during report %s:%s run', self.user_import.uuid, self.uuid) + state = 'error' + try: + exception = six.text_type(e) + except Exception: + exception = repr(repr(e)) + else: + exception = None + state = 'finished' + + with self.data_update as data: + data['state'] = state + data['exception'] = exception + data['importer'] = importer + t = threading.Thread(target=target) + with self.data_update as data: + data['state'] = 'running' + if start: + t.start() + return t + + def __getattr__(self, name): + try: + return self.data[name] + except KeyError: + raise AttributeError(name) + + def exists(self): + return os.path.exists(self.path) + + def delete(self): + if self.simulate and self.exists(): + os.unlink(self.path) + + +class Reports(object): + PREFIX = 'report-' + + def __init__(self, user_import): + self.user_import = user_import + + def __getitem__(self, uuid): + report = Report(self.user_import, uuid) + if not report.exists(): + raise KeyError + return report + + def __iter__(self): + for name in os.listdir(self.user_import.path): + if name.startswith(self.PREFIX): + try: + yield self[name[len(self.PREFIX):]] + except KeyError: + pass diff --git a/src/authentic2/manager/user_views.py b/src/authentic2/manager/user_views.py index 768d2bfa..002fc553 100644 --- a/src/authentic2/manager/user_views.py +++ b/src/authentic2/manager/user_views.py @@ -16,6 +16,7 @@ import datetime import collections +import operator from django.db import models from django.utils.translation import ugettext_lazy as _, ugettext @@ -26,6 +27,8 @@ from django.core.urlresolvers import reverse from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.contrib import messages +from django.views.generic import FormView, TemplateView +from django.http import Http404, FileResponse import tablib @@ -36,12 +39,15 @@ from authentic2 import hooks from django_rbac.utils import get_role_model, get_role_parenting_model, get_ou_model -from .views import BaseTableView, BaseAddView, \ - BaseEditView, ActionMixin, OtherActionsMixin, Action, ExportMixin, \ - BaseSubTableView, HideOUColumnMixin, BaseDeleteView, BaseDetailView +from .views import (BaseTableView, BaseAddView, BaseEditView, ActionMixin, + OtherActionsMixin, Action, ExportMixin, BaseSubTableView, + HideOUColumnMixin, BaseDeleteView, BaseDetailView, + PermissionMixin) from .tables import UserTable, UserRolesTable, OuUserRolesTable from .forms import (UserSearchForm, UserAddForm, UserEditForm, - UserChangePasswordForm, ChooseUserRoleForm, UserRoleSearchForm, UserChangeEmailForm) + UserChangePasswordForm, ChooseUserRoleForm, + UserRoleSearchForm, UserChangeEmailForm, UserNewImportForm, + UserEditImportForm) from .resources import UserResource from .utils import get_ou_count, has_show_username from . import app_settings @@ -618,3 +624,121 @@ class UserDeleteView(BaseDeleteView): user_delete = UserDeleteView.as_view() + + +class UserImportsView(PermissionMixin, FormView): + form_class = UserNewImportForm + template_name = 'authentic2/manager/user_imports.html' + + def post(self, request, *args, **kwargs): + from . import user_import + + if 'delete' in request.POST: + uuid = request.POST['delete'] + user_import.UserImport(uuid).delete() + return redirect(self.request, 'a2-manager-users-imports') + return super(UserImportsView, self).post(request, *args, **kwargs) + + def form_valid(self, form): + user_import = form.save() + with user_import.meta_update as meta: + meta['user'] = self.request.user.get_full_name() + meta['user_pk'] = self.request.user.pk + return redirect(self.request, 'a2-manager-users-import', kwargs={'uuid': user_import.uuid}) + + def get_context_data(self, **kwargs): + from . import user_import + + ctx = super(UserImportsView, self).get_context_data() + ctx['imports'] = sorted(user_import.UserImport.all(), key=operator.attrgetter('created'), reverse=True) + return ctx + +user_imports = UserImportsView.as_view() + + +class UserImportView(PermissionMixin, FormView): + form_class = UserEditImportForm + permissions = ['custom_user.admin_user'] + template_name = 'authentic2/manager/user_import.html' + + def dispatch(self, request, uuid, **kwargs): + from user_import import UserImport + self.user_import = UserImport(uuid) + if not self.user_import.exists(): + raise Http404 + return super(UserImportView, self).dispatch(request, uuid, **kwargs) + + def get(self, request, uuid, filename=None): + if filename: + return FileResponse(self.user_import.import_file, content_type='text/csv') + return super(UserImportView, self).get(request, uuid=uuid, filename=filename) + + def get_form_kwargs(self): + kwargs = super(UserImportView, self).get_form_kwargs() + kwargs['user_import'] = self.user_import + return kwargs + + def post(self, request, *args, **kwargs): + from . import user_import + + if 'delete' in request.POST: + uuid = request.POST['delete'] + try: + report = self.user_import.reports[uuid] + except KeyError: + pass + else: + report.delete() + return redirect(request, 'a2-manager-users-import', kwargs={'uuid': self.user_import.uuid}) + + simulate = 'simulate' in request.POST + execute = 'execute' in request.POST + if simulate or execute: + report = user_import.Report.new(self.user_import) + report.run(simulate=simulate) + return redirect(request, 'a2-manager-users-import', kwargs={'uuid': self.user_import.uuid}) + return super(UserImportView, self).post(request, *args, **kwargs) + + def form_valid(self, form): + form.save() + return super(UserImportView, self).form_valid(form) + + def get_success_url(self): + return reverse('a2-manager-users-import', kwargs={'uuid': self.user_import.uuid}) + + def get_context_data(self, **kwargs): + ctx = super(UserImportView, self).get_context_data() + ctx['user_import'] = self.user_import + ctx['reports'] = sorted(self.user_import.reports, key=operator.attrgetter('created'), reverse=True) + return ctx + +user_import = UserImportView.as_view() + + +class UserImportReportView(PermissionMixin, TemplateView): + form_class = UserEditImportForm + permissions = ['custom_user.admin_user'] + template_name = 'authentic2/manager/user_import_report.html' + + def dispatch(self, request, import_uuid, report_uuid): + from user_import import UserImport + self.user_import = UserImport(import_uuid) + if not self.user_import.exists(): + raise Http404 + try: + self.report = self.user_import.reports[report_uuid] + except KeyError: + raise Http404 + return super(UserImportReportView, self).dispatch(request, import_uuid, report_uuid) + + def get_context_data(self, **kwargs): + ctx = super(UserImportReportView, self).get_context_data() + ctx['user_import'] = self.user_import + ctx['report'] = self.report + if self.report.simulate: + ctx['report_title'] = _('Simulation') + else: + ctx['report_title'] = _('Execution') + return ctx + +user_import_report = UserImportReportView.as_view() diff --git a/tests/test_manager_user_import.py b/tests/test_manager_user_import.py new file mode 100644 index 00000000..aee07a66 --- /dev/null +++ b/tests/test_manager_user_import.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# authentic2 - versatile identity manager +# Copyright (C) 2010-2019 Entr'ouvert +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from __future__ import unicode_literals + +import io +import operator + +import pytest + +from authentic2.manager.user_import import UserImport, Report +from authentic2.models import Attribute + + +@pytest.fixture +def profile(transactional_db): + Attribute.objects.create(name='phone', kind='phone_number', label='Numéro de téléphone') + + +def test_user_import(media, transactional_db, profile): + content = '''email key verified,first_name,last_name,phone no-create +tnoel@entrouvert.com,Thomas,Noël,1234 +fpeters@entrouvert.com,Frédéric,Péters,5678 +x,x,x,x''' + fd = io.BytesIO(content.encode('utf-8')) + + assert len(list(UserImport.all())) == 0 + + UserImport.new(fd, encoding='utf-8') + UserImport.new(fd, encoding='utf-8') + + assert len(list(UserImport.all())) == 2 + for user_import in UserImport.all(): + with user_import.import_file as fd: + assert fd.read() == content.encode('utf-8') + + for user_import in UserImport.all(): + report = Report.new(user_import) + assert user_import.reports[report.uuid].exists() + assert user_import.reports[report.uuid].data['encoding'] == 'utf-8' + assert user_import.reports[report.uuid].data['state'] == 'waiting' + + t = report.run(start=False) + + assert user_import.reports[report.uuid].data['state'] == 'running' + + t.start() + t.join() + + assert user_import.reports[report.uuid].data['state'] == 'finished' + assert user_import.reports[report.uuid].data['importer'] + assert not user_import.reports[report.uuid].data['importer'].errors + + for user_import in UserImport.all(): + reports = list(user_import.reports) + assert len(reports) == 1 + assert reports[0].created + importer = reports[0].data['importer'] + assert importer.rows[0].is_valid + assert importer.rows[1].is_valid + assert not importer.rows[2].is_valid + + user_imports = sorted(UserImport.all(), key=operator.attrgetter('created')) + user_import1 = user_imports[0] + report1 = list(user_import1.reports)[0] + importer = report1.data['importer'] + assert all(row.action == 'create' for row in importer.rows[:2]) + assert all(cell.action == 'updated' for row in importer.rows[:2] for cell in row.cells[:3]) + assert all(cell.action == 'nothing' for row in importer.rows[:2] for cell in row.cells[3:]) + + user_import2 = user_imports[1] + report2 = list(user_import2.reports)[0] + importer = report2.data['importer'] + assert all(row.action == 'update' for row in importer.rows[:2]) + assert all(cell.action == 'nothing' for row in importer.rows[:2] for cell in row.cells[:3]) + assert all(cell.action == 'updated' for row in importer.rows[:2] for cell in row.cells[3:]) -- 2.20.1