From f23c7a0f347c4db210d80ceb56bbf75ee69ab448 Mon Sep 17 00:00:00 2001
From: Paul Marillonnet <pmarillonnet@entrouvert.com>
Date: Tue, 7 May 2019 10:02:35 +0200
Subject: [PATCH] import users from csv file (#32833)

---
 src/authentic2/app_settings.py                |  1 +
 src/authentic2/data_transfer.py               | 49 +++++++++++
 src/authentic2/manager/forms.py               | 13 +++
 .../templates/authentic2/manager/users.html   | 14 +++
 .../authentic2/manager/users_csv_import.html  | 34 ++++++++
 src/authentic2/manager/urls.py                |  2 +
 src/authentic2/manager/user_views.py          | 57 ++++++++++++-
 src/authentic2/manager/utils.py               |  6 ++
 tests/test_data_transfer.py                   | 85 ++++++++++++++++++-
 tests/test_manager.py                         | 43 ++++++++++
 tests/users_csv_import_testfile.csv           |  3 +
 tests/users_csv_import_testfile2.csv          |  3 +
 tests/users_csv_import_testfile3.csv          |  3 +
 13 files changed, 309 insertions(+), 4 deletions(-)
 create mode 100644 src/authentic2/manager/templates/authentic2/manager/users_csv_import.html
 create mode 100644 tests/users_csv_import_testfile.csv
 create mode 100644 tests/users_csv_import_testfile2.csv
 create mode 100644 tests/users_csv_import_testfile3.csv

diff --git a/src/authentic2/app_settings.py b/src/authentic2/app_settings.py
index d21323a0..bae735ce 100644
--- a/src/authentic2/app_settings.py
+++ b/src/authentic2/app_settings.py
@@ -209,6 +209,7 @@ default_settings = dict(
     A2_ACCOUNTS_URL=Setting(default=None, definition='IdP has no account page, redirect to this one.'),
     A2_CACHE_ENABLED=Setting(default=True, definition='Disable all cache decorators for testing purpose.'),
     A2_ACCEPT_EMAIL_AUTHENTICATION=Setting(default=True, definition='Enable authentication by email'),
+    A2_USERS_CSV_IMPORT_APP_ID_LABEL=Setting(default='app_id', definition='Label of the applicative identifier used when importing users from csv'),
 
 )
 
diff --git a/src/authentic2/data_transfer.py b/src/authentic2/data_transfer.py
index 6f5a5d77..eac8dc36 100644
--- a/src/authentic2/data_transfer.py
+++ b/src/authentic2/data_transfer.py
@@ -1,9 +1,18 @@
+import csv
+import logging
+
+from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
+from django.db import transaction
 
 from django_rbac.models import Operation
 from django_rbac.utils import (
     get_ou_model, get_role_model, get_role_parenting_model, get_permission_model)
 from authentic2.a2_rbac.models import RoleAttribute
+from authentic2 import app_settings
+from authentic2.models import UserExternalId, Attribute
+
+logger = logging.getLogger(__name__)
 
 
 def update_model(obj, d):
@@ -318,3 +327,43 @@ def import_site(json_d, import_context=None):
                     import_context.role_delete_orphans))
 
     return result
+
+
+def create_user_from_data(user_data):
+    User = get_user_model()
+    user = User.objects.create()
+    for name, value in user_data.items():
+        if hasattr(user, name):
+            setattr(user, name, value)
+        elif hasattr(user.attributes, name):
+            attribute = Attribute.objects.get(name=name)
+            deserialize = attribute.get_kind()['deserialize']
+            try:
+                value = deserialize(value)
+            except:
+                logger.error(
+                    'invalid value %s for attribute %s' % (value, name))
+                continue
+            attribute.set_value(owner=user, value=value)
+    user.save()
+    return user
+
+
+def import_users_from_csv(csv_file, ou):
+    reader = csv.reader(csv_file) # XXX dialect
+    rows = list(reader)
+    app_id_label = app_settings.A2_USERS_CSV_IMPORT_APP_ID_LABEL
+
+    rows[0].index(app_id_label)
+
+    with transaction.atomic():
+        User = get_user_model()
+        for i in range(1,len(rows)):
+            user_data = dict(zip(rows[0], rows[i]))
+            user_data.update({'ou': ou})
+            app_id = user_data.pop(app_id_label)
+            user = create_user_from_data(user_data)
+            user_external_id = UserExternalId.objects.create(
+                user=user, external_id=app_id, source='import')
+
+    return len(rows)-1
diff --git a/src/authentic2/manager/forms.py b/src/authentic2/manager/forms.py
index 1708290f..c6d610e0 100644
--- a/src/authentic2/manager/forms.py
+++ b/src/authentic2/manager/forms.py
@@ -694,3 +694,16 @@ class UserChangeEmailForm(CssClass, FormWithRequest, forms.ModelForm):
 
     class Meta:
         fields = ()
+
+
+class UsersCsvImportForm(forms.Form):
+    csv_file = forms.FileField(
+        label=_('CSV file'),
+        help_text=_('CSV file for the import.')
+    )
+    ou = forms.ChoiceField(
+        label=_('Organizational unit'),
+        help_text=_('The newly created users will be registered to the chosen OU.'),
+        choices=utils.get_ou_choices(),
+        initial=get_default_ou().uuid
+    )
diff --git a/src/authentic2/manager/templates/authentic2/manager/users.html b/src/authentic2/manager/templates/authentic2/manager/users.html
index 34489519..a7de73b2 100644
--- a/src/authentic2/manager/templates/authentic2/manager/users.html
+++ b/src/authentic2/manager/templates/authentic2/manager/users.html
@@ -20,6 +20,20 @@
          {% trans "Add user" %}
      </a>
    {% endif %}
+   {% if import_users %}
+     <a
+        href="{% url "a2-manager-users-import" %}"
+        id="import-users-btn">
+         {% trans "Import users from csv" %}
+     </a>
+   {% else %}
+     <a
+        href="#"
+        class="disabled"
+        id="import-users-btn">
+         {% trans "Import users from csv" %}
+     </a>
+   {% endif %}
   </span>
 {% endblock %}
 
diff --git a/src/authentic2/manager/templates/authentic2/manager/users_csv_import.html b/src/authentic2/manager/templates/authentic2/manager/users_csv_import.html
new file mode 100644
index 00000000..d58850c3
--- /dev/null
+++ b/src/authentic2/manager/templates/authentic2/manager/users_csv_import.html
@@ -0,0 +1,34 @@
+{% extends "authentic2/manager/form.html" %}
+{% load i18n %}
+
+{% block page-title %}
+  {{ block.super }} - {{ view.title }}
+{% endblock %}
+
+{% block breadcrumb %}
+  {{ block.super }}
+  <a href="..">{% trans "Users" %}</a>
+  <a href="">{{ view.title }}</a>
+{% endblock %}
+
+{% block content %}
+  {% trans "Comma-separated value file defining optional standard user attributes as well as the extended profile attributes below:" %}
+  <ul>
+    <li><b>{{ app_id_label }}</b> ({% trans "mandatory" %})</li>
+  {% for attr in attributes %}
+    <li>{{ attr.name }}</li>
+  {% endfor %}
+  </ul>
+
+  <form enctype="multipart/form-data" method="post">
+
+    {% csrf_token %}
+    {{ form.as_p }}
+    {% if form.instance and form.instance.id %}
+      <button class="submit-button">{% trans "Submit" %}</button>
+      <button class="cancel-button" name="cancel" formnovalidate>{% trans "Cancel" %}</button>
+    {% else %}
+      <button class="submit-button">{% trans "Create" %}</button>
+    {% endif %}
+  </form>
+{% endblock %}
diff --git a/src/authentic2/manager/urls.py b/src/authentic2/manager/urls.py
index c0646d5d..3175ed01 100644
--- a/src/authentic2/manager/urls.py
+++ b/src/authentic2/manager/urls.py
@@ -21,6 +21,8 @@ urlpatterns = required(
         url(r'^users/$', user_views.users, name='a2-manager-users'),
         url(r'^users/export/(?P<format>csv)/$',
             user_views.users_export, name='a2-manager-users-export'),
+        url(r'^users/import/$',
+            user_views.users_csv_import, name='a2-manager-users-import'),
         url(r'^users/add/$', user_views.user_add_default_ou,
             name='a2-manager-user-add-default-ou'),
         url(r'^users/(?P<ou_pk>\d+)/add/$', user_views.user_add,
diff --git a/src/authentic2/manager/user_views.py b/src/authentic2/manager/user_views.py
index 711b1ea6..c9333af4 100644
--- a/src/authentic2/manager/user_views.py
+++ b/src/authentic2/manager/user_views.py
@@ -15,7 +15,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.contrib import messages
 from django.http import HttpResponseRedirect, QueryDict
 from django.views.generic.detail import SingleObjectMixin
-from django.views.generic import View
+from django.views.generic import View, FormView
 
 from import_export.fields import Field
 import tablib
@@ -24,7 +24,7 @@ from authentic2.constants import SWITCH_USER_SESSION_KEY
 from authentic2.models import Attribute, AttributeValue, PasswordReset
 from authentic2.utils import switch_user, send_password_reset_mail, redirect, select_next_url
 from authentic2.a2_rbac.utils import get_default_ou
-from authentic2 import hooks
+from authentic2 import hooks, app_settings as a2_app_settings
 from django_rbac.utils import get_role_model, get_role_parenting_model, get_ou_model
 
 
@@ -33,7 +33,8 @@ from .views import BaseTableView, BaseAddView, \
     BaseSubTableView, HideOUColumnMixin, BaseDeleteView, BaseDetailView
 from .tables import UserTable, UserRolesTable, OuUserRolesTable
 from .forms import (UserSearchForm, UserAddForm, UserEditForm,
-    UserChangePasswordForm, ChooseUserRoleForm, UserRoleSearchForm, UserChangeEmailForm)
+    UserChangePasswordForm, ChooseUserRoleForm, UserRoleSearchForm, UserChangeEmailForm,
+    UsersCsvImportForm)
 from .resources import UserResource
 from .utils import get_ou_count
 from . import app_settings
@@ -88,6 +89,8 @@ class UsersView(HideOUColumnMixin, BaseTableView):
             ou = self.search_form.cleaned_data.get('ou')
         if ou and self.request.user.has_ou_perm('custom_user.add_user', ou):
             ctx['add_ou'] = ou
+        if self.request.user.has_perm('custom_user.add_user'):
+            ctx['import_users'] = True
         return ctx
 
 
@@ -603,3 +606,51 @@ class UserDeleteView(BaseDeleteView):
 
 
 user_delete = UserDeleteView.as_view()
+
+
+class UsersCsvImportView(FormView):
+    form_class = UsersCsvImportForm
+    template_name = 'authentic2/manager/users_csv_import.html'
+    title = _('CSV import')
+    success_url = '..'
+    permissions = ['custom_user.add_user']
+
+    def get_context_data(self, **kwargs):
+        from authentic2.models import Attribute
+
+        ctx = super(UsersCsvImportView, self).get_context_data()
+        ctx.update({
+            'attributes': Attribute.objects.all(),
+            'app_id_label': a2_app_settings.A2_USERS_CSV_IMPORT_APP_ID_LABEL
+        })
+        return ctx
+
+    def form_valid(self, form):
+        from authentic2.data_transfer import import_users_from_csv
+
+        ou_uuid = form.cleaned_data.get('ou')
+        ou = get_ou_model().objects.get(uuid=ou_uuid) or get_default_ou()
+        csv_file = form.cleaned_data.get('csv_file')
+
+        if not csv_file:
+            messages.warning(
+                self.request,
+                _('No import performed. Please provide a csv file.')
+            )
+        else:
+            try:
+                nb_created = import_users_from_csv(csv_file, ou)
+            except ValueError as e:
+                messages.error(
+                    self.request,
+                    _('Value error while importing the users: %s') % e
+                )
+            else:
+                messages.info(
+                    self.request,
+                    _('Created %s users.') % nb_created
+                )
+        return super(UsersCsvImportView, self).form_valid(form)
+
+
+users_csv_import = UsersCsvImportView.as_view()
diff --git a/src/authentic2/manager/utils.py b/src/authentic2/manager/utils.py
index 619b5e7c..a84fa850 100644
--- a/src/authentic2/manager/utils.py
+++ b/src/authentic2/manager/utils.py
@@ -26,3 +26,9 @@ def label_from_user(user):
 @GlobalCache(timeout=10)
 def get_ou_count():
     return get_ou_model().objects.count()
+
+
+@GlobalCache(timeout=10)
+def get_ou_choices():
+    Ou = get_ou_model()
+    return [(ou.uuid, ou.name) for ou in Ou.objects.all()]
diff --git a/tests/test_data_transfer.py b/tests/test_data_transfer.py
index 779396ef..1d92cb36 100644
--- a/tests/test_data_transfer.py
+++ b/tests/test_data_transfer.py
@@ -1,5 +1,7 @@
+from django.contrib.auth import get_user_model
 from django_rbac.utils import get_role_model, get_ou_model
 import pytest
+import csv
 
 from authentic2.a2_rbac.models import RoleParenting
 from authentic2.data_transfer import (
@@ -12,7 +14,9 @@ from authentic2.data_transfer import (
     ImportContext,
     RoleDeserializer,
     search_role,
-    import_ou)
+    import_ou,
+    import_users_from_csv)
+from authentic2.models import UserExternalId, Attribute
 from authentic2.utils import get_hex_uuid
 
 
@@ -523,3 +527,82 @@ def test_export_site(db):
     d = export_site(ExportContext(export_ous=False))
     assert 'ous' not in d
 
+
+def test_import_users_from_csv(db):
+    ou = OU.objects.create(name='ou')
+    User = get_user_model()
+    assert not len(User.objects.all())
+
+    with open('tests/users_csv_import_testfile.csv', 'r') as csv_file:
+        assert import_users_from_csv(csv_file, ou) == 2
+
+    with open('tests/users_csv_import_testfile.csv', 'r') as csv_file:
+        reader = csv.reader(csv_file)
+        rows = list(reader)
+        assert len(User.objects.all()) == len(rows)-1
+        for i in range(2):
+            user = User.objects.all()[i]
+            assert user.first_name == rows[i+1][1].decode('utf-8')
+            assert user.last_name == rows[i+1][2].decode('utf-8')
+            assert user.email == rows[i+1][3].decode('utf-8')
+            assert user.ou == ou
+            external_id = UserExternalId.objects.get(user=user)
+            assert external_id.source == 'import'
+            assert external_id.external_id == rows[i+1][0].decode('utf-8')
+
+
+def test_import_users_from_csv_profile_attributes(db):
+    Attribute.objects.create(
+        label='Nickname',
+        name='nickname',
+        kind='string',
+        user_visible=True,
+        user_editable=True
+    )
+    Attribute.objects.create(
+        label='Is a spy',
+        name='is_a_spy',
+        kind='boolean',
+        user_visible=True,
+        user_editable=True
+    )
+    Attribute.objects.create(
+        label='Enrolled on',
+        name='enrolled_on',
+        kind='date',
+        user_visible=True,
+        user_editable=True
+    )
+    Attribute.objects.create(
+        label='Secret phone number',
+        name='secret_phone_number',
+        kind='phone_number',
+        user_visible=True,
+        user_editable=True
+    )
+    Attribute.objects.create(
+        label='Headquarters postcode',
+        name='headquarters_postcode',
+        kind='fr_postcode',
+        user_visible=True,
+        user_editable=True
+    )
+
+    ou = OU.objects.create(name='ou')
+    User = get_user_model()
+    assert not len(User.objects.all())
+
+    with open('tests/users_csv_import_testfile2.csv', 'r') as csv_file:
+        assert import_users_from_csv(csv_file, ou) == 2
+
+    with open('tests/users_csv_import_testfile2.csv', 'r') as csv_file:
+        reader = csv.reader(csv_file)
+        rows = list(reader)
+        assert len(User.objects.all()) == len(rows)-1
+        for i in range(2):
+            user = User.objects.all()[i]
+            assert user.attributes.nickname == rows[i+1][3].decode('utf-8')
+            assert user.attributes.is_a_spy == bool(int(rows[i+1][4].decode('utf-8')))
+            assert user.attributes.enrolled_on.isoformat() == rows[i+1][5].decode('utf-8')
+            assert user.attributes.secret_phone_number == rows[i+1][6].decode('utf-8')
+            assert user.attributes.headquarters_postcode == rows[i+1][7].decode('utf-8')
diff --git a/tests/test_manager.py b/tests/test_manager.py
index d8ae1b19..53902534 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -8,7 +8,9 @@ from django.core import mail
 
 from webtest import Upload
 
+from authentic2 import app_settings
 from authentic2.a2_rbac.utils import get_default_ou
+from authentic2.models import Attribute
 
 from django_rbac.utils import get_ou_model, get_role_model
 from django.contrib.auth import get_user_model
@@ -889,3 +891,44 @@ def test_manager_edit_user_email_verified(app, simple_user, superuser_or_admin):
 
     user = User.objects.get(id=simple_user.id)
     assert not user.email_verified
+
+
+def test_manager_import_users_from_csv(admin, app, db, ou1, ou2):
+    Attribute.objects.create(
+        label='Nickname',
+        name='nickname',
+        kind='string',
+        user_visible=True,
+        user_editable=True
+    )
+    Attribute.objects.create(
+        label='Freemason',
+        name='freemason',
+        kind='boolean',
+        user_visible=True,
+        user_editable=True
+    )
+
+    User = get_user_model()
+    nb_initial = len(User.objects.all())
+
+    login(app, admin, '/manage/')
+    url = u'/manage/users/import/'
+
+    ous = [ou1, ou2, get_default_ou()]
+
+    response = app.get(url)
+    assert 'nickname' in response.text
+    assert 'freemason' in response.text
+    assert app_settings.A2_USERS_CSV_IMPORT_APP_ID_LABEL in response.text
+    for ou in ous:
+        assert ou.uuid in response.text
+
+    form = response.form
+    form.set('csv_file', Upload('tests/users_csv_import_testfile3.csv'))
+    form.set('ou', ou1.uuid)
+    response = form.submit().follow()
+
+    assert len(User.objects.all()) == nb_initial+2
+    for last_name in ('Dupont', 'Durant'):
+        assert User.objects.get(last_name=last_name).ou == ou1
diff --git a/tests/users_csv_import_testfile.csv b/tests/users_csv_import_testfile.csv
new file mode 100644
index 00000000..6594e984
--- /dev/null
+++ b/tests/users_csv_import_testfile.csv
@@ -0,0 +1,3 @@
+app_id,first_name,last_name,email
+123,Hervé,Dupont,foo@bar.null
+456,René,Durant,boo@far.null
diff --git a/tests/users_csv_import_testfile2.csv b/tests/users_csv_import_testfile2.csv
new file mode 100644
index 00000000..97adaa39
--- /dev/null
+++ b/tests/users_csv_import_testfile2.csv
@@ -0,0 +1,3 @@
+app_id,first_name,last_name,nickname,is_a_spy,enrolled_on,secret_phone_number,headquarters_postcode
+123,Hervé,Dupont,Herbie,1,2006-06-06,+12345,75001
+456,René,Durant,Ronnie,0,2007-07-07,+12356,13001
diff --git a/tests/users_csv_import_testfile3.csv b/tests/users_csv_import_testfile3.csv
new file mode 100644
index 00000000..762bed5a
--- /dev/null
+++ b/tests/users_csv_import_testfile3.csv
@@ -0,0 +1,3 @@
+app_id,first_name,last_name,nickname,freemason
+123,Hervé,Dupont,Herv,0
+456,René,Durant,RR,1
-- 
2.20.1

