From 758c52fbad6872023f4805ae78f4715d76ca54ad Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Fri, 4 Aug 2017 12:03:34 +0200 Subject: [PATCH] add roles import/export command (#16514) --- src/authentic2/a2_rbac/models.py | 36 ++- src/authentic2/management/commands/export-roles.py | 26 ++ src/authentic2/management/commands/import-roles.py | 27 ++ src/authentic2/models.py | 4 +- src/authentic2/utils.py | 133 ++++++++- tests/test_import_export.py | 304 +++++++++++++++++++++ 6 files changed, 525 insertions(+), 5 deletions(-) create mode 100644 src/authentic2/management/commands/export-roles.py create mode 100644 src/authentic2/management/commands/import-roles.py create mode 100644 tests/test_import_export.py diff --git a/src/authentic2/a2_rbac/models.py b/src/authentic2/a2_rbac/models.py index 42cd2a95..c46a12b9 100644 --- a/src/authentic2/a2_rbac/models.py +++ b/src/authentic2/a2_rbac/models.py @@ -10,6 +10,8 @@ from django_rbac.models import (RoleAbstractBase, PermissionAbstractBase, CHANGE_OP, Operation) from django_rbac import utils as rbac_utils +from authentic2.utils import ImportExportError, ImportExportMixin + try: from django.contrib.contenttypes.fields import GenericForeignKey, \ GenericRelation @@ -21,7 +23,7 @@ except ImportError: from . import managers, fields -class OrganizationalUnit(OrganizationalUnitAbstractBase): +class OrganizationalUnit(OrganizationalUnitAbstractBase, ImportExportMixin): username_is_unique = models.BooleanField( blank=True, default=False, @@ -91,7 +93,7 @@ class Permission(PermissionAbstractBase): object_id_field='admin_scope_id') -class Role(RoleAbstractBase): +class Role(RoleAbstractBase, ImportExportMixin): admin_scope_ct = models.ForeignKey( to='contenttypes.ContentType', null=True, @@ -195,6 +197,36 @@ class Role(RoleAbstractBase): 'ou__slug': self.ou.slug if self.ou else None, } + @classmethod + def import_json(cls, data): + role = super(Role, cls).import_json(data) + # set attributes + attributes_json = data.pop('attributes', {}) + for attr in attributes_json: + attribute, created = RoleAttribute.objects.get_or_create( + role=role, name=attr['name'], kind=attr['kind'], defaults={ + 'value': attr['value']}) + if created: + attribute.value = attr['value'] + attribute.save() + return role + + def export_json(self): + data = super(Role, self).export_json() + attributes = [] + for attr in self.attributes.all(): + attributes.append({ + 'name': attr.name, 'kind': attr.kind, + 'value': attr.value}) + data['attributes'] = attributes + + parents = [] + for parent in self.parents(include_self=False): + parents.append({ + 'uuid': parent.uuid, 'slug': parent.slug, 'name': parent.name}) + data['parents'] = parents + return data + class RoleParenting(RoleParentingAbstractBase): class Meta(RoleParentingAbstractBase.Meta): diff --git a/src/authentic2/management/commands/export-roles.py b/src/authentic2/management/commands/export-roles.py new file mode 100644 index 00000000..8389bd95 --- /dev/null +++ b/src/authentic2/management/commands/export-roles.py @@ -0,0 +1,26 @@ +import json +from optparse import make_option +import sys + +from django.core.management import BaseCommand + +from authentic2.utils import export_roles + + +class Command(BaseCommand): + help = 'Export roles as json' + + args = '' + + option_list = BaseCommand.option_list + ( + make_option('--ou', dest='ou', default=None, type=str, + help='restrict to the organizational unit slug'),) + + def handle(self, *args, **options): + if len(args) < 1: + output = sys.stdout + else: + output = open(args[0], 'w') + + data = export_roles(ou_slug=options['ou']) + json.dump(data, output, encoding='utf-8', indent=4) diff --git a/src/authentic2/management/commands/import-roles.py b/src/authentic2/management/commands/import-roles.py new file mode 100644 index 00000000..3a58e95d --- /dev/null +++ b/src/authentic2/management/commands/import-roles.py @@ -0,0 +1,27 @@ +import json +from optparse import make_option + +from django.core.management.base import BaseCommand, CommandError + +from authentic2.utils import import_roles, ImportExportError + + +class Command(BaseCommand): + help = 'Import roles from json file' + + args = '' + + option_list = BaseCommand.option_list + ( + make_option('--ou', dest='ou', default=None, type=str, + help='restrict to the organizational unit slug'), + make_option('--stop-on-absent-parent', action='store_true', dest='stop_absent_parent', default=False, + help='stop if parent is absent') + ) + + def handle(self, *args, **options): + if args: + fd = open(args[0]) + try: + import_roles(json.load(fd), **options) + except(ImportExportError,) as exc: + raise CommandError(exc.message) diff --git a/src/authentic2/models.py b/src/authentic2/models.py index 40afd94e..90cbe22b 100644 --- a/src/authentic2/models.py +++ b/src/authentic2/models.py @@ -23,7 +23,7 @@ except ImportError: from django.contrib.contenttypes.models import ContentType from . import managers -from .utils import ServiceAccessDenied +from .utils import ServiceAccessDenied, ImportExportMixin class DeletedUser(models.Model): @@ -310,7 +310,7 @@ class PasswordReset(models.Model): return unicode(self.user) -class Service(models.Model): +class Service(models.Model, ImportExportMixin): name = models.CharField( verbose_name=_('name'), max_length=128) diff --git a/src/authentic2/utils.py b/src/authentic2/utils.py index b85339b5..54ec3acd 100644 --- a/src/authentic2/utils.py +++ b/src/authentic2/utils.py @@ -16,8 +16,10 @@ from importlib import import_module import django from django.conf import settings +from django.db import transaction +from django.db import models from django.http import HttpResponseRedirect, HttpResponse -from django.core.exceptions import ImproperlyConfigured, PermissionDenied +from django.core.exceptions import ImproperlyConfigured, PermissionDenied, FieldError from django.http.request import QueryDict from django.contrib.auth import (REDIRECT_FIELD_NAME, login as auth_login, SESSION_KEY, HASH_SESSION_KEY, BACKEND_SESSION_KEY, authenticate, @@ -41,6 +43,8 @@ from django.utils.http import urlsafe_base64_encode from django.utils.encoding import force_bytes from django.shortcuts import render +from jsonfield import JSONField + try: from django.core.exceptions import FieldDoesNotExist @@ -54,6 +58,17 @@ from authentic2.saml.saml2utils import filter_attribute_private_key, \ from . import plugins, app_settings, constants +IMPORT_EXPORT_FIELDS = ( + models.TextField, models.CharField, models.SlugField, + models.URLField, models.BooleanField, models.IntegerField, + models.CommaSeparatedIntegerField, models.EmailField, models.NullBooleanField, + models.IntegerField, models.PositiveIntegerField, JSONField) + + +class ImportExportError(Exception): + pass + + class CleanLogMessage(logging.Filter): def filter(self, record): record.msg = filter_attribute_private_key(record.msg) @@ -966,3 +981,119 @@ def simulate_authentication(request, user, method, user = copy.deepcopy(user) user.backend = backend return login(request, user, method, **kwargs) + + +class ImportExportMixin(object): + + @classmethod + def get_model_fields(cls): + return cls._meta.get_concrete_fields_with_model() + + @classmethod + def get_model_required_fields(cls): + return [field for field, _ in cls.get_model_fields() if not field.null and field.name != 'id'] + + @classmethod + def get_instance(cls, data): + fields = cls.get_model_required_fields() + kwargs = {field.name: data[field.name] for field in fields} + instance, created = cls.objects.get_or_create(**kwargs) + return instance + + @classmethod + def import_json(cls, data): + instance = cls.get_instance(data) + for field, model in cls._meta.get_concrete_fields_with_model(): + if field.name == 'id': + continue + value = data[field.name] + if isinstance(field, IMPORT_EXPORT_FIELDS): + setattr(instance, field.attname, value) + elif isinstance(field, models.ForeignKey): + if value: + related_model = field.rel.to + related_instance = instance.get_object_by_main_attr(related_model, **value) + if not related_instance: + raise ImportExportError( + '%s %s related object %s %s does not exist.' % ( + cls.__name__, instance, related_model.__name__, value.values())) + value = related_instance + setattr(instance, field.name, value) + else: + raise Exception('export_json: field %s of ressource class %s is unsupported' % ( + field, self.__class__)) + instance.save() + return instance + + def export_json(self): + data = {} + for field, model in self.get_model_fields(): + if field.name == 'id': + continue + value = getattr(self, field.attname) + if isinstance(field, IMPORT_EXPORT_FIELDS): + data[field.name] = value + elif isinstance(field, models.ForeignKey): + if value: + value = getattr(self, field.name) + data[field.name] = value.export_json() + else: + data[field.name] = None + else: + raise Exception('export_json: field %s of ressource class %s is unsupported' % ( + field, self.__class__)) + return data + + def get_object_by_main_attr(self, model, **kwargs): + """Sequentially try to get an object by uuid, slug, then name. + Creates one if is True + """ + uuid = kwargs.pop('uuid', None) + slug = kwargs.pop('slug', None) + name = kwargs.pop('name', None) + try: + return model.objects.get(uuid=uuid) + except (model.DoesNotExist, FieldError): + try: + return model.objects.get(slug=slug) + except (model.DoesNotExist, FieldError): + try: + return model.objects.get(name=name) + except (model.DoesNotExist, FieldError): + pass + return None + + +def export_roles(ou_slug=None): + from django_rbac.utils import get_role_model + Role = get_role_model() + filters = {'slug__startswith': '_'} + if ou_slug: + roles = Role.objects.filter(ou__slug=ou_slug).exclude(**filters) + else: + roles = Role.objects.exclude(**filters) + return [role.export_json() for role in roles] + + +def import_roles(data, **options): + from django_rbac.utils import get_role_model + + Role = get_role_model() + ou_slug = options.pop('ou') + + if ou_slug: + data = [datum for datum in data if datum['ou']['slug'] == ou_slug] + with transaction.atomic(): + for role_json in data: + Role.import_json(role_json) + + # once all roles are created, set their relationships + for role_json in data: + role = Role.objects.get(uuid=role_json['uuid']) + for parent_json in role_json.get('parents', []): + parent = role.get_object_by_main_attr(Role, **parent_json) + if parent: + role.add_parent(parent) + else: + raise ImportExportError( + 'Parent role (%(uuid)s, %(slug)s, %(name)s does not exist.' % parent_json) diff --git a/tests/test_import_export.py b/tests/test_import_export.py new file mode 100644 index 00000000..ca7e6dd3 --- /dev/null +++ b/tests/test_import_export.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- + +import json + +import pytest + +from django.core.management import call_command, CommandError +from django.contrib.auth import get_user_model +from django_rbac.utils import get_role_model, get_ou_model + +from authentic2.models import Attribute, UserExternalId, Service +from authentic2.a2_rbac.models import RoleAttribute + +pytestmark = pytest.mark.django_db + + +def create_user(**kwargs): + User = get_user_model() + Role = get_role_model() + + attributes = kwargs.pop('attributes') + roles = kwargs.pop('roles') + password = kwargs.pop('password', None) or kwargs['username'] + external_ids = kwargs.pop('external_ids', []) + + user = User.objects.create(**kwargs) + + if password: + user.set_password(password) + user.save() + + for key in attributes: + Attribute.objects.get(name=key).set_value(user, attributes[key]) + + for role in roles: + user.roles.add(Role.objects.get(slug=role)) + + for eid in external_ids: + UserExternalId.objects.create(user=user, source=eid['source'], + external_id=eid['external_id']) + + return user + + +# FIXTURES + +@pytest.fixture(scope='session') +def tmp_export_dir(tmpdir_factory): + return tmpdir_factory.mktemp('export') + + +@pytest.fixture +def city_ou(db): + OU = get_ou_model() + + return OU.objects.create( + uuid='6b0d73622a694ca7a0440cbc809487f7', + name='Narnia', slug='city') + + +@pytest.fixture +def weird_ou(db): + OU = get_ou_model() + + return OU.objects.create( + uuid='aca57fbf08e1406c86900c86b335d004', + slug='weird', + name='Weird' + ) + + +@pytest.fixture +def city_roles(db, city_ou, weird_ou): + + Role = get_role_model() + + city_service = Service.objects.create(name='f.i.b', slug='fib', ou=city_ou) + + emails = { + 'city-agents': json.dumps(['smith@city.fan', 'coolsen@city.fan', 'fury@city.fan']), + 'city-enfance': json.dumps(['alice@city.fan']), + 'city-vip': json.dumps(['kirk@city.fan']), + 'city-dsi': json.dumps(['sheldon@city.fan', 'eliot@city.fan']), + 'city-culture': json.dumps(['chelsea@city.fan']), + } + + roles = [ + {'uuid': 'e0f821033649475abf70448350796676', 'slug': 'city-agents', 'name': 'Narnia Agents'}, + {'uuid': 'f61a77e094894bd089039afb4f5ce64b', 'slug': 'city-enfance', 'name': 'Narnia Enfance'}, + {'uuid': '63516d5f7d444e63b639325d7b53fd3a', 'slug': 'city-vip', 'name': 'Narnia VIP'}, + {'uuid': 'f091a140421949208b1c1ebf78d15b35', 'slug': 'city-dsi', 'name': 'Narnia DSI'}, + {'uuid': '535583de8f544c7f9eb46725c72acf0a', 'slug': 'city-culture', 'name': 'Narnia Culture'}, + ] + + for role in roles: + ou = weird_ou if role['slug'] == 'city-vip' else city_ou + Role.objects.create(ou=ou, **role) + + concerned_roles = Role.objects.exclude(slug__startswith='_a2') + + for role in concerned_roles: + RoleAttribute.objects.create( + role=role, name='emails', kind='json', + value=emails[role.slug] + ) + + RoleAttribute.objects.create( + role=role, name='emails_to_members', kind='json', value=False) + + agents = Role.objects.get(slug='city-agents') + + for slug in ['city-enfance', 'city-culture']: + Role.objects.get(slug=slug).add_parent(agents) + + role_dsi = Role.objects.get(slug='city-dsi') + role_dsi.service = city_service + role_dsi.save() + + +@pytest.fixture +def city_attributes(db): + attributes = [ + {'name': 'street', 'label': 'street', 'kind': 'string'}, + {'name': 'city', 'label': 'city', 'kind': 'string'}, + {'name': 'zip_code', 'label': 'zip_code', 'kind': 'string'}, + ] + + for attr in attributes: + Attribute.objects.create(**attr) + + +@pytest.fixture +def city_users(city_ou, weird_ou, city_roles, city_attributes): + + users = [ + {'username': 'josh', 'email': 'josh@loking.fan', + 'attributes': {'street': '302 Main Street', 'city': 'Vancouver', 'zip_code': 'V5K 0A6'}, + 'roles': ['city-vip', 'city-agents', 'city-dsi'], + 'external_ids': [{'source': 'dedsec', 'external_id': 'r3tr0'}, + {'source': 'nuddle', 'external_id': 'horatio'}]}, + {'username': 'kim', 'email': 'kim@loking.fan', + 'attributes': {'street': '37 Bvd Henry Orion', 'city': 'Nantes', 'zip_code': '44000'}, + 'roles': ['city-enfance']}, + {'username': 'chelsea', 'email': 'chelsea@loking.fan', + 'attributes': {'street': '740 Studebaker Dr.', 'city': 'Baton Rouge', 'zip_code': '70806'}, + 'roles': ['city-enfance']}, + {'username': 'mandla', 'email': 'mandla@loking.fan', + 'attributes': {'street': '497 Jacob Mare Street', 'city': 'Pretoria', 'zip_code': '0001'}, + 'roles': ['city-culture']}, + {'username': 'tux', 'email': 'tux@linux.org', + 'attributes': {}, + 'roles': ['city-vip']}, + ] + + for user in users: + create_user(ou=city_ou, **user) + + others = [ + {'username': 'éric', 'email': 'eric@loking.fan', + 'attributes': {'street': '342 Lincon Blvd', 'city': 'Los Angeles', 'zip_code': '90291'}, + 'roles': ['city-vip', 'city-agents', 'city-dsi']}, + {'username': 'hannibal', 'email': 'hannibal@loking.fan', + 'attributes': {'street': '12 5th Ave', 'city': 'New York', 'zip_code': '10002'}, + 'roles': ['city-enfance']}, + ] + + for other in others: + create_user(ou=weird_ou, **other) + + +# TESTS + +def test_roles_import_export(tmp_export_dir, city_roles): + dest_file = tmp_export_dir.join("roles.json").strpath + call_command('export-roles', dest_file) + data = json.load(file(dest_file)) + + assert len(data) == 5 + + for datum in data: + if datum['slug'] in ['city-enfance', 'city-culture']: + assert len(datum['parents']) == 1 + assert datum['parents'][0]['uuid'] == 'e0f821033649475abf70448350796676' + assert datum['parents'][0]['slug'] == 'city-agents' + + if datum['slug'] == 'city-agents': + assert len(datum['parents']) == 0 + assert datum['uuid'] == 'e0f821033649475abf70448350796676' + + assert len(datum['attributes']) == 2 + for attr in datum['attributes']: + if attr['name'] == 'emails_to_members': + continue + + assert attr['kind'] == 'json' + assert attr["value"] == '["smith@city.fan", "coolsen@city.fan", "fury@city.fan"]' + + if datum['slug'] == 'city-vip': + assert datum['ou']['slug'] == 'weird' + assert len(datum['attributes']) == 2 + for attr in datum['attributes']: + if attr['name'] == 'emails': + assert attr['value'] == '["kirk@city.fan"]' + else: + assert attr['value'] == "False" + + # import roles + Role = get_role_model() + Role.objects.exclude(slug__startswith='_').delete() + call_command('import-roles', dest_file) + + roles = Role.objects.exclude(slug__startswith='_') + + assert len(roles) == 5 + + for role in roles: + if role.slug == 'city-vip': + continue + + assert role.ou.uuid == '6b0d73622a694ca7a0440cbc809487f7' + assert role.ou.slug == 'city' + assert role.ou.name == 'Narnia' + + agent_role = roles.get(slug='city-agents') + + assert agent_role.uuid == 'e0f821033649475abf70448350796676' + assert agent_role.name == 'Narnia Agents' + + assert len(agent_role.parents(include_self=False)) == 0 + + for attr in agent_role.attributes.all(): + assert attr.kind == 'json' + if attr.name == 'emails': + assert attr.value == u'["smith@city.fan", "coolsen@city.fan", "fury@city.fan"]' + + enfance_role = roles.get(slug='city-enfance') + + assert enfance_role.uuid == 'f61a77e094894bd089039afb4f5ce64b' + assert enfance_role.name == 'Narnia Enfance' + assert len(enfance_role.parents(include_self=False)) == 1 + assert agent_role.child_relation.filter(child__slug='city-enfance').exists() is True + assert Role.objects.exclude(slug__startswith='_').count() == 5 + + # re-import the same roles and make sure the number or role is the same + call_command('import-roles', dest_file) + assert Role.objects.exclude(slug__startswith='_').count() == 5 + + +def test_roles_import_export_by_ou_slug(tmp_export_dir, city_roles): + dest_file = tmp_export_dir.join("roles_ou.json").strpath + call_command('export-roles', dest_file, ou='city') + data = json.load(file(dest_file)) + assert len(data) == 4 + + # test import + call_command('export-roles', dest_file, ou='weird') + Role = get_role_model() + Role.objects.exclude(slug__startswith='_').delete() + + dest_file = tmp_export_dir.join('roles_ou.json').strpath + call_command('import-roles', dest_file, ou='weird') + + roles = Role.objects.exclude(slug__startswith='_') + assert len(roles) == 1 + for role in roles: + role.uuid = 'aca57fbf08e1406c86900c86b335d004' + role.slug = 'weird' + role.name = 'Weird' + assert Role.objects.exclude(slug__startswith='_').count() == 1 + + +def test_roles_import_export_with_missing_ou(tmp_export_dir, city_roles): + dest_file = tmp_export_dir.join("roles.json").strpath + call_command('export-roles', dest_file) + Role = get_role_model() + Role.objects.exclude(slug__startswith='_').delete() + get_ou_model().objects.get(slug='city').delete() + with pytest.raises(CommandError) as excinfo: + call_command('import-roles', dest_file) + assert Role.objects.exclude(slug__startswith='_').count() == 0 + + +def test_roles_import_export_with_missing_service(tmp_export_dir, city_roles): + dest_file = tmp_export_dir.join("roles.json").strpath + call_command('export-roles', dest_file) + Role = get_role_model() + Role.objects.exclude(slug__startswith='_').delete() + Service.objects.first().delete() + with pytest.raises(CommandError) as excinfo: + call_command('import-roles', dest_file) + assert Role.objects.exclude(slug__startswith='_').count() == 0 + + +def test_roles_import_export_with_missing_parent_role(tmp_export_dir, city_roles, city_ou): + Role = get_role_model() + dest_file = tmp_export_dir.join("roles.json").strpath + role_tmp = Role.objects.create(uuid='tmp', slug='_tmp', name='tmp', ou=city_ou) + Role.objects.get(slug='city-agents').add_parent(role_tmp) + call_command('export-roles', dest_file) + Role.objects.exclude(slug__startswith='_').delete() + role_tmp.delete() + with pytest.raises(CommandError) as excinfo: + call_command('import-roles', dest_file) + assert Role.objects.exclude(slug__startswith='_').count() == 0 -- 2.11.0