From afa9c737fc754c38dd2afbd1b0fc15d150f1b29a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Tue, 26 Jan 2016 14:49:37 +0100 Subject: [PATCH] bdd user migration from csv (#9717) --- mandayejs/applications.py | 25 +++++-- .../management/commands/migrate-ldap-users.py | 49 ------------- .../mandaye/management/commands/migrate-users.py | 80 ++++++++++++++++++++++ tests/csv_users.csv | 4 ++ tests/ldap_users.txt | 9 +-- tests/tests.py | 60 +++++++++++++--- 6 files changed, 157 insertions(+), 70 deletions(-) delete mode 100644 mandayejs/mandaye/management/commands/migrate-ldap-users.py create mode 100644 mandayejs/mandaye/management/commands/migrate-users.py create mode 100644 tests/csv_users.csv diff --git a/mandayejs/applications.py b/mandayejs/applications.py index ca84f44..c1d31b0 100644 --- a/mandayejs/applications.py +++ b/mandayejs/applications.py @@ -62,11 +62,6 @@ class AppSettingsMeta(type): dct['SITE_AUTH_CHECKER'] = os.path.join(settings.STATIC_ROOT, dct['SITE_AUTH_CHECKER']) - # Global js scripts for the current app - if dct.get('SITE_APP_SCRIPTS',None): - dct['SITE_APP_SCRIPTS'] = [ - script for script in dct['SITE_APP_SCRIPTS'] - ] # Default form submit element if not dct.get('SITE_FORM_SUBMIT_ELEMENT', None): dct['SITE_FORM_SUBMIT_ELEMENT'] = 'input[type=submit], button' @@ -77,6 +72,26 @@ class AppSettingsMeta(type): class AppSettings(object): __metaclass__ = AppSettingsMeta +# Test App Settings +class Test(AppSettings): + SITE_LOGIN_PATH = '/' + SITE_LOCATORS = [ + { + 'id': '#login', + 'label': 'Login', + 'name': 'login', + 'kind': 'string', + }, + { + 'id': '#password', + 'label': 'Password', + 'name': 'password', + 'kind': 'password' + } + ] + SITE_AUTH_CHECKER = 'js/test/auth.checker.js' + SITE_AUTH_COOKIE_KEYS = [ 'test'] + SITE_FORCE_REDIRECT_URL = '/whatever' # Duonet App Settings class Duonet(AppSettings): diff --git a/mandayejs/mandaye/management/commands/migrate-ldap-users.py b/mandayejs/mandaye/management/commands/migrate-ldap-users.py deleted file mode 100644 index afbd057..0000000 --- a/mandayejs/mandaye/management/commands/migrate-ldap-users.py +++ /dev/null @@ -1,49 +0,0 @@ -from __future__ import absolute_import - -import json -import ldif -import logging - -from django.core.management.base import BaseCommand, CommandError -from django.db import IntegrityError -from django.contrib.auth import get_user_model - -from mandayejs.mandaye.models import UserCredentials - -logger = logging.getLogger(__name__) - -class Command(BaseCommand): - args = '' - help = 'Migrate users from ldap' - - def handle(self, *args, **kwargs): - - if len(args) < 1: - self.stdout.write('An input file is required') - return False - for ldif_file in args: - data = self.get_ldif_data(ldif_file) - data = [ d[1] for d in data] - self.migrate(data) - - def get_ldif_data(self, filename): - with open(filename, 'rb') as fd: - return ldif.ParseLDIF(fd) - - def migrate(self, parsed_data): - User = get_user_model() - for data in parsed_data: - data = { k : ''.join(v) for k,v in data.items()} - try: - user, created = User.objects.get_or_create(username=data.get('idpUniqueID'), - last_name=data.get('spLogin')) - user.save() - uc = UserCredentials(user=user, locators=json.loads(data.get('spPostValues'))) - uc.decrypt() - uc.save() - self.stdout.write('{idpUniqueID} imported'.format(**data)) - logger.debug('{idpUniqueID} imported'.format(**data)) - except (IntegrityError,) as e: - logger.debug(e) - continue - diff --git a/mandayejs/mandaye/management/commands/migrate-users.py b/mandayejs/mandaye/management/commands/migrate-users.py new file mode 100644 index 0000000..92922f8 --- /dev/null +++ b/mandayejs/mandaye/management/commands/migrate-users.py @@ -0,0 +1,80 @@ +from __future__ import absolute_import + +import os +import csv +import json +import ldif +import logging +from optparse import make_option + +from django.core.management.base import BaseCommand, CommandError +from django.db import IntegrityError +from django.contrib.auth.models import User + +from mandayejs.mandaye.models import UserCredentials + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + args = '' + help = 'Migrate users from ldif file or csv file' + + option_list = BaseCommand.option_list +( + make_option( + '--ldap', + action='store_true', + default=False, + help='Migrate users from a ldap dump file' + ), + make_option( + '--csv', + action='store_true', + default=False, + help='Migrate users from a csv file' + ) + ) + + def handle(self, *args, **kwargs): + if len(args) < 1: + self.stdout.write('An input file is required') + return False + + if kwargs.get('csv'): + for filename in args: + data = self.get_csv_data(filename) + self.migrate(data) + else: + for filename in args: + data = self.get_ldif_data(filename) + data = [ d[1] for d in data] + data = [ + { k : ''.join(v) for k,v in d.items()} for d in data + ] + self.migrate(data) + + def get_ldif_data(self, filename): + with open(filename, 'rb') as fd: + return ldif.ParseLDIF(fd) + + def get_csv_data(self, filename): + with open(filename, 'rb') as fd: + fieldnames = ['idpUniqueID', 'spPostValues'] + reader = csv.DictReader(fd, delimiter=';', quotechar='|', fieldnames=fieldnames) + return list(reader) + + + def migrate(self, parsed_data): + for data in parsed_data: + try: + user, created = User.objects.get_or_create(username=data.get('idpUniqueID'), + last_name=data.get('spLogin','')) + user.save() + uc = UserCredentials(user=user, locators=json.loads(data.get('spPostValues'))) + uc.decrypt() + uc.save() + logger.debug('{idpUniqueID} imported'.format(**data)) + except (IntegrityError,) as e: + logger.debug(e) + continue + diff --git a/tests/csv_users.csv b/tests/csv_users.csv new file mode 100644 index 0000000..52e4a41 --- /dev/null +++ b/tests/csv_users.csv @@ -0,0 +1,4 @@ +_288CE3839831B75D8ADD6524329236B7;{"username": "csv1", "password": "cIrWyg=="} +_8C36D00EE18E5702A51813413097028F;{"username": "csv2", "password": "cIrWyQ=="} +_45EB3616CEE069A42AFD23D7039E6FBD;{"username": "csv3", "password": "cIrWyA=="} +_45EB3616CEJ069A42AFD23D70S9E6FBD;{"username": "csv4", "password": "cIrWzw=="} diff --git a/tests/ldap_users.txt b/tests/ldap_users.txt index d47f678..a02af17 100644 --- a/tests/ldap_users.txt +++ b/tests/ldap_users.txt @@ -11,8 +11,7 @@ uniqueID: 928438 entryUUID: 061ece3e-f3ac-1033-91e7-8926ba287cf0 creatorsName: cn=admin,dc=entrouvert,dc=org createTimestamp: 20141029113944Z -spPostValues: {"txtCode": "Y4HL6cbGxRsoHQU97VzXgkqEdA==", "txtNomFoyer": "ldap_user1", "t - xtDateNaissance": "23/04/1991"} +spPostValues: {"password": "Y4HL6cbGxRsoHQU97VzXgkqEdA==", "login": "ldap_user1"} entryCSN: 20141029133206.125440Z#000000#001#000000 modifiersName: cn=admin,dc=entrouvert,dc=org modifyTimestamp: 20141029133206Z @@ -30,8 +29,7 @@ uniqueID: 434365 entryUUID: 061fe3c8-f3ac-1033-91ea-8926ba287cf0 creatorsName: cn=admin,dc=entrouvert,dc=org createTimestamp: 20141029113944Z -spPostValues: {"txtCode": "Y4HL6cbGxRsoHQU97VzXgkqEdw==", "txtNomFoyer": "ldap_user2", "t - xtDateNaissance": "23/04/1991"} +spPostValues: {"password": "Y4HL6cbGxRsoHQU97VzXgkqEdw==", "login": "ldap_user2"} entryCSN: 20141029133206.144171Z#000000#001#000000 modifiersName: cn=admin,dc=entrouvert,dc=org modifyTimestamp: 20141029133206Z @@ -49,8 +47,7 @@ uniqueID: 1179584 entryUUID: 0620818e-f3ac-1033-91ec-8926ba287cf0 creatorsName: cn=admin,dc=entrouvert,dc=org createTimestamp: 20141029113944Z -spPostValues: {"txtCode": "Y4HL6cbGxRsoHQU97VzXgkqEdg==", "txtNomFoyer": "ldap_user3", - "txtDateNaissance": "19/08/1953"} +spPostValues: {"password": "Y4HL6cbGxRsoHQU97VzXgkqEdg==", "login": "ldap_user3"} entryCSN: 20141029133206.156443Z#000000#001#000000 modifiersName: cn=admin,dc=entrouvert,dc=org modifyTimestamp: 20141029133206Z diff --git a/tests/tests.py b/tests/tests.py index dfc61fd..ba3e123 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,8 +8,10 @@ from mandayejs.mandaye.models import UserCredentials pytestmark = pytest.mark.django_db -# Encryption/Decryption +settings.SITE_APP = 'mandayejs.applications.Test' +settings.SECRET_KEY = 'od5cei4aeveel8dui4lei2ou9ahsei2A' +# ENCRYPTION/DECRYPTION def create_user(**kwargs): password = kwargs.pop('password', None) or kwargs.get('username') user, created = User.objects.get_or_create(**kwargs) @@ -44,21 +46,59 @@ def test_encryption(credentials): assert decrypted.get('password') == 'john password' -# Migration -def test_migrate_users_command(): +# MIGRATION COMMAND +def cmd(*args, **kwargs): + cmd = type( + 'Cmd', + (object,), + { + 'name': 'migrate-users', + 'args': args, + 'opts': kwargs + } + ) + return cmd - args = ['tests/ldap_users.txt',] - opts = {} - call_command('migrate-ldap-users', *args, **opts) +@pytest.fixture +def command_ldap(): + return cmd( + 'tests/ldap_users.txt', + ldap= True + ) + +@pytest.fixture +def command_csv(): + return cmd( + 'tests/csv_users.csv', + csv= True + ) + +@pytest.fixture(params=['command_ldap', 'command_csv']) +def command(request, command_ldap, command_csv): + return locals().get(request.param) - credentials = UserCredentials.objects.filter(user__last_name__in=[ +def test_command_migrate_users(command): + call_command(command.name, *command.args, **command.opts) + if command.opts.get('ldap'): + credentials = UserCredentials.objects.filter(user__last_name__in=[ + 'ldap_user1', + 'ldap_user2', + 'ldap_user3' + ]) + + assert len(credentials) == 3 + + for cred in credentials: + assert cred.to_login_info(decrypt=True)['#password'] == 'password_{}'.format(cred.user.last_name) + else: + credentials = UserCredentials.objects.all().exclude(user__last_name__in=[ 'ldap_user1', 'ldap_user2', 'ldap_user3' ]) - assert len(credentials) == 3 + assert len(credentials) == 4 - for cred in credentials: - assert cred.to_login_info(decrypt=True)['#txtCode'] == 'password_{}'.format(cred.user.last_name) + for cred in credentials: + assert cred.to_login_info(decrypt=True)['#password'] == cred.to_login_info()['#username'] -- 2.7.0.rc3