From 36dec7ceb1f16b89645077b5c3e31484a0df2880 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Tue, 13 Sep 2016 17:13:31 +0200 Subject: [PATCH] send provisionning messages after request treatment in a thread (fixes #9396) All objects to provision are collected into the Provisionning singleton object in thread local dictionnaries. When request processing is finished the ProvisionningMiddleware launch a thread which will send provisionning messages. --- debian/debian_config_common.py | 4 + hobo/agent/authentic2/apps.py | 208 ++----------------------------- hobo/multitenant/apps.py | 1 + hobo/multitenant/models.py | 4 +- tests_authentic/conftest.py | 7 +- tests_authentic/test_provisionning.py | 222 ++++++++++++++++++++++++++++------ tox.ini | 2 +- 7 files changed, 210 insertions(+), 238 deletions(-) diff --git a/debian/debian_config_common.py b/debian/debian_config_common.py index 1486d26..7ade363 100644 --- a/debian/debian_config_common.py +++ b/debian/debian_config_common.py @@ -209,6 +209,10 @@ if 'authentic2' not in INSTALLED_APPS: MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ( 'mellon.middleware.PassiveAuthenticationMiddleware', ) +else: + MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ( + 'hobo.agent.authentic2.middleware.ProvisionningMiddleware', + ) TENANT_SETTINGS_LOADERS = ( 'hobo.multitenant.settings_loaders.TemplateVars', diff --git a/hobo/agent/authentic2/apps.py b/hobo/agent/authentic2/apps.py index b2f437d..deb0e71 100644 --- a/hobo/agent/authentic2/apps.py +++ b/hobo/agent/authentic2/apps.py @@ -14,191 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json -from urlparse import urljoin - from django.apps import AppConfig -from django.db.models.signals import post_save, post_delete, pre_delete, m2m_changed -from django.conf import settings -from django.contrib.auth import get_user_model -from django.db import connection -from django.core.urlresolvers import reverse - -from django_rbac.utils import get_role_model - -from hobo.agent.common import notify_agents -from authentic2.models import AttributeValue -from authentic2.saml.models import LibertyProvider -from authentic2.a2_rbac.models import OrganizationalUnit - - -def get_ou(role_or_through): - if hasattr(role_or_through, 'ou_id'): - return role_or_through.ou - else: - return role_or_through.role.ou - - -def get_audience(role_or_through): - ou = get_ou(role_or_through) - if ou: - qs = LibertyProvider.objects.filter(ou=ou) - else: - qs = LibertyProvider.objects.filter(ou__isnull=True) - return [(service, service.entity_id) for service in qs] - - -def get_related_roles(role_or_through): - ou = get_ou(role_or_through) - Role = get_role_model() - qs = Role.objects.filter(admin_scope_id__isnull=True) \ - .prefetch_related('attributes') - if ou: - qs = qs.filter(ou=ou) - else: - qs = qs.filter(ou__isnull=True) - for role in qs: - role.emails = [] - role.emails_to_members = False - role.details = u'' - for attribute in role.attributes.all(): - if (attribute.name in ('emails', 'emails_to_members', 'details') - and attribute.kind == 'json'): - setattr(role, attribute.name, json.loads(attribute.value)) - return qs - - -def notify_roles(sender, instance, **kwargs): - if not getattr(settings, 'HOBO_ROLE_EXPORT', True): - return - if instance.slug.startswith('_'): - return - try: - notify_agents({ - '@type': 'provision', - 'audience': [audience for service, audience in get_audience(instance)], - 'full': True, - 'objects': { - '@type': 'role', - 'data': [ - { - 'uuid': role.uuid, - 'name': role.name, - 'slug': role.slug, - 'description': role.description, - 'details': role.details, - 'emails': role.emails, - 'emails_to_members': role.emails_to_members, - } for role in get_related_roles(instance) - ], - } - }) - except OrganizationalUnit.DoesNotExist: - pass - - -def get_entity_id(): - tenant = getattr(connection, 'tenant', None) - assert tenant - base_url = tenant.get_base_url() - return urljoin(base_url, reverse('a2-idp-saml-metadata')) - - -def provision_user(sender, instance, **kwargs): - User = get_user_model() - if not isinstance(instance, User): - return - if not getattr(settings, 'HOBO_ROLE_EXPORT', True): - return - data = {} - for av in AttributeValue.objects.with_owner(instance): - data[str(av.attribute.name)] = av.to_python() - - roles = instance.roles_and_parents() \ - .prefetch_related('attributes') - is_superuser = instance.is_superuser - data.update({ - 'uuid': instance.uuid, - 'username': instance.username, - 'first_name': instance.first_name, - 'last_name': instance.last_name, - 'email': instance.email, - 'roles': [ - { - 'uuid': role.uuid, - 'name': role.name, - 'slug': role.slug, - } for role in roles], - }) - - for service, audience in get_audience(instance): - role_is_superuser = False - for role in roles: - if role.service_id != service.pk: - continue - for attribute in role.attributes.all(): - if attribute.name == 'is_superuser' and attribute.value == 'true': - role_is_superuser = True - data['is_superuser'] = is_superuser or role_is_superuser - notify_agents({ - '@type': 'provision', - 'issuer': unicode(get_entity_id()), - 'audience': [audience], - 'full': False, - 'objects': { - '@type': 'user', - 'data': [data], - } - }) - - -def deprovision_user(sender, instance, **kwargs): - User = get_user_model() - if not isinstance(instance, User): - return - if not getattr(settings, 'HOBO_ROLE_EXPORT', True): - return - notify_agents({ - '@type': 'deprovision', - 'issuer': unicode(get_entity_id()), - 'audience': [audience for service, audience in get_audience(instance)], - 'full': False, - 'objects': { - '@type': 'user', - 'data': [ - { - 'uuid': instance.uuid, - } - ], - } - }) - - -def provision_user_on_role_change(sender, action, instance, model, pk_set, - reverse, **kwargs): - if not action.startswith('post'): - return - if action.endswith('_clear'): - return - if reverse: - provision_user(sender, instance, **kwargs) - else: - for user in model.objects.filter(pk__in=pk_set): - provision_user(sender, user, **kwargs) - - -def provision_user_on_attribute_value_save(sender, instance, **kwargs): - User = get_user_model() - if not isinstance(instance.owner, User): - return - provision_user(User, instance.owner) - - -def provision_user_on_attribute_value_delete(sender, instance, **kwargs): - User = get_user_model() - if not isinstance(instance.owner, User): - return - provision_user(User, instance.owner) +from django.db.models.signals import pre_save, pre_delete, m2m_changed, post_save class Authentic2AgentConfig(AppConfig): @@ -207,18 +24,11 @@ class Authentic2AgentConfig(AppConfig): verbose_name = 'Authentic2 Agent' def ready(self): - Role = get_role_model() - post_save.connect(notify_roles, sender=Role) - post_delete.connect(notify_roles, sender=Role) - post_save.connect(notify_roles, Role) - post_delete.connect(notify_roles, Role) - post_save.connect(notify_roles, Role.members.through) - post_delete.connect(notify_roles, Role.members.through) - post_save.connect(provision_user) - pre_delete.connect(deprovision_user) - post_save.connect(provision_user_on_attribute_value_save, sender=AttributeValue) - post_delete.connect(provision_user_on_attribute_value_delete, sender=AttributeValue) - m2m_changed.connect(provision_user_on_role_change, - sender=Role.members.through) - settings.A2_MANAGER_ROLE_FORM_CLASS = \ - 'hobo.agent.authentic2.role_forms.RoleForm' + from . import provisionning + + engine = provisionning.Provisionning() + pre_save.connect(engine.pre_save) + post_save.connect(engine.post_save) + pre_delete.connect(engine.pre_delete) + m2m_changed.connect(engine.m2m_changed) + provisionning.provisionning = engine diff --git a/hobo/multitenant/apps.py b/hobo/multitenant/apps.py index 7bc418f..51e2ff3 100644 --- a/hobo/multitenant/apps.py +++ b/hobo/multitenant/apps.py @@ -19,6 +19,7 @@ class TenantAwareThread(threading.Thread): super(TenantAwareThread, self).run() finally: connection.set_tenant(old_tenant) + connection.close() class _Timer(TenantAwareThread): diff --git a/hobo/multitenant/models.py b/hobo/multitenant/models.py index fc450ba..2fa97c8 100644 --- a/hobo/multitenant/models.py +++ b/hobo/multitenant/models.py @@ -65,8 +65,8 @@ class Tenant(TenantMixin): "the public schema. Current schema is %s." % connection.schema_name) - os.rename(self.get_directory(), self.get_directory()+'.invalid') + os.rename(self.get_directory(), self.get_directory() + '.invalid') - if schema_exists(self.schema_name) and (self.auto_drop_schema or force_drop) and not django_is_in_test_mode(): + if schema_exists(self.schema_name) and (self.auto_drop_schema or force_drop): cursor = connection.cursor() cursor.execute('DROP SCHEMA %s CASCADE' % self.schema_name) diff --git a/tests_authentic/conftest.py b/tests_authentic/conftest.py index a37ba62..2f82b9b 100644 --- a/tests_authentic/conftest.py +++ b/tests_authentic/conftest.py @@ -5,19 +5,23 @@ import json import pytest + @pytest.fixture def tenant_base(request, settings): base = tempfile.mkdtemp('combo-tenant-base') settings.TENANT_BASE = base + def fin(): shutil.rmtree(base) request.addfinalizer(fin) return base + @pytest.fixture(scope='function') -def tenant(db, request, settings, tenant_base): +def tenant(transactional_db, request, settings, tenant_base): from hobo.multitenant.models import Tenant base = tenant_base + @pytest.mark.django_db def make_tenant(name): tenant_dir = os.path.join(base, name) @@ -54,6 +58,7 @@ def tenant(db, request, settings, tenant_base): t.create_schema() return t tenants = [make_tenant('authentic.example.net')] + def fin(): from django.db import connection connection.set_schema_to_public() diff --git a/tests_authentic/test_provisionning.py b/tests_authentic/test_provisionning.py index 377211e..ed6e88e 100644 --- a/tests_authentic/test_provisionning.py +++ b/tests_authentic/test_provisionning.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +import json + import pytest +import lasso from mock import patch, call, ANY @@ -7,17 +10,24 @@ from django.contrib.auth import get_user_model from tenant_schemas.utils import tenant_context -from authentic2.a2_rbac.models import Role +from authentic2.saml.models import LibertyProvider +from authentic2.a2_rbac.models import Role, RoleAttribute from authentic2.a2_rbac.utils import get_default_ou from authentic2.models import Attribute, AttributeValue +from hobo.agent.authentic2.provisionning import provisionning pytestmark = pytest.mark.django_db -def test_provision_role(tenant): - with patch('hobo.agent.authentic2.apps.notify_agents') as notify_agents: +def test_provision_role(transactional_db, tenant, caplog): + with patch('hobo.agent.authentic2.provisionning.notify_agents') as notify_agents: with tenant_context(tenant): - role = Role.objects.create(name='coin') + LibertyProvider.objects.create(ou=get_default_ou(), name='provider', + entity_id='http://provider.com', + protocol_conformance=lasso.PROTOCOL_SAML_2_0) + with provisionning: + role = Role.objects.create(name='coin', ou=get_default_ou()) + assert notify_agents.call_count == 1 arg = notify_agents.call_args assert arg == call(ANY) @@ -25,51 +35,100 @@ def test_provision_role(tenant): assert isinstance(arg, dict) assert set(arg.keys()) == set([ 'audience', '@type', 'objects', 'full']) - assert arg['audience'] == [] + assert arg['audience'] == ['http://provider.com'] assert arg['@type'] == 'provision' - assert arg['full'] is True + assert arg['full'] is False objects = arg['objects'] assert isinstance(objects, dict) assert set(objects.keys()) == set(['data', '@type']) assert objects['@type'] == 'role' data = objects['data'] assert isinstance(data, list) - assert len(data) == 2 - like_role = 0 - for o in data: - assert set(o.keys()) == set(['details', 'emails_to_members', - 'description', 'uuid', 'name', - 'slug', 'emails']) - assert o['details'] == '' - assert o['emails_to_members'] is False - assert o['emails'] == [] - if o['uuid'] == role.uuid and o['name'] == role.name \ - and o['description'] == role.description \ - and o['slug'] == role.slug: - like_role += 1 - assert like_role == 1 + assert len(data) == 1 + o = data[0] + assert set(o.keys()) == set(['details', 'emails_to_members', + 'description', 'uuid', 'name', + 'slug', 'emails']) + assert o['details'] == '' + assert o['emails_to_members'] is False + assert o['emails'] == [] + notify_agents.reset_mock() + emails = ['john.doe@example.com', 'toto@entrouvert.com'] + with provisionning: + RoleAttribute.objects.create( + role=role, name='emails', kind='json', + value=json.dumps(emails)) -def test_provision_user(tenant): - import lasso - from authentic2.saml.models import LibertyProvider + assert notify_agents.call_count == 1 + arg = notify_agents.call_args + assert arg == call(ANY) + arg = arg[0][0] + assert isinstance(arg, dict) + assert set(arg.keys()) == set([ + 'audience', '@type', 'objects', 'full']) + assert arg['audience'] == ['http://provider.com'] + assert arg['@type'] == 'provision' + assert arg['full'] is False + objects = arg['objects'] + assert isinstance(objects, dict) + assert set(objects.keys()) == set(['data', '@type']) + assert objects['@type'] == 'role' + data = objects['data'] + assert isinstance(data, list) + assert len(data) == 1 + o = data[0] + assert set(o.keys()) == set(['details', 'emails_to_members', + 'description', 'uuid', 'name', + 'slug', 'emails']) + assert o['details'] == '' + assert o['emails_to_members'] is False + assert o['emails'] == emails + + notify_agents.reset_mock() + with provisionning: + role.delete() - with patch('hobo.agent.authentic2.apps.notify_agents') as notify_agents: + assert notify_agents.call_count == 1 + arg = notify_agents.call_args + assert arg == call(ANY) + arg = arg[0][0] + assert isinstance(arg, dict) + assert set(arg.keys()) == set([ + 'audience', '@type', 'objects', 'full']) + assert arg['audience'] == ['http://provider.com'] + assert arg['@type'] == 'deprovision' + assert arg['full'] is False + objects = arg['objects'] + assert isinstance(objects, dict) + assert set(objects.keys()) == set(['data', '@type']) + assert objects['@type'] == 'role' + data = objects['data'] + assert isinstance(data, list) + assert len(data) == 1 + o = data[0] + assert set(o.keys()) == set(['uuid']) + + +def test_provision_user(transactional_db, tenant, caplog): + with patch('hobo.agent.authentic2.provisionning.notify_agents') as notify_agents: with tenant_context(tenant): service = LibertyProvider.objects.create(ou=get_default_ou(), name='provider', entity_id='http://provider.com', protocol_conformance=lasso.PROTOCOL_SAML_2_0) role = Role.objects.create(name='coin', service=service, ou=get_default_ou()) role.attributes.create(kind='string', name='is_superuser', value='true') + child_role = Role.objects.create(name='child', ou=get_default_ou()) notify_agents.reset_mock() User = get_user_model() attribute = Attribute.objects.create(label='Code postal', name='code_postal', kind='string') - user = User.objects.create(username=u'Étienne', - email='etienne.dugenou@example.net', - first_name=u'Étienne', - last_name=u'Dugenou', - ou=get_default_ou()) + with provisionning: + user = User.objects.create(username=u'Étienne', + email='etienne.dugenou@example.net', + first_name=u'Étienne', + last_name=u'Dugenou', + ou=get_default_ou()) assert notify_agents.call_count == 1 arg = notify_agents.call_args assert arg == call(ANY) @@ -103,9 +162,10 @@ def test_provision_user(tenant): notify_agents.reset_mock() attribute.set_value(user, '13400') user.is_superuser = True - user.save() + with provisionning: + user.save() - assert notify_agents.call_count == 2 + assert notify_agents.call_count == 1 arg = notify_agents.call_args assert arg == call(ANY) arg = arg[0][0] @@ -137,7 +197,8 @@ def test_provision_user(tenant): assert o['is_superuser'] is True notify_agents.reset_mock() - AttributeValue.objects.get().delete() + with provisionning: + AttributeValue.objects.get().delete() assert notify_agents.call_count == 1 arg = notify_agents.call_args @@ -171,8 +232,10 @@ def test_provision_user(tenant): user.is_superuser = False user.save() + notify_agents.reset_mock() - role.members.add(user) + with provisionning: + role.members.add(user) assert notify_agents.call_count == 1 arg = notify_agents.call_args @@ -209,7 +272,8 @@ def test_provision_user(tenant): assert o['is_superuser'] is True notify_agents.reset_mock() - user.roles.remove(role) + with provisionning: + user.roles.remove(role) assert notify_agents.call_count == 1 arg = notify_agents.call_args @@ -240,8 +304,96 @@ def test_provision_user(tenant): assert o['email'] == user.email assert o['roles'] == [] assert o['is_superuser'] is False + + notify_agents.reset_mock() + with provisionning: + user.roles.add(child_role) + child_role.add_parent(role) + + assert notify_agents.call_count == 1 + arg = notify_agents.call_args + assert arg == call(ANY) + arg = arg[0][0] + assert isinstance(arg, dict) + assert set(arg.keys()) == set([ + 'issuer', 'audience', '@type', 'objects', 'full']) + assert arg['issuer'] == \ + 'http://%s/idp/saml2/metadata' % tenant.domain_url + assert arg['audience'] == ['http://provider.com'] + assert arg['@type'] == 'provision' + assert arg['full'] is False + objects = arg['objects'] + assert isinstance(objects, dict) + assert set(objects.keys()) == set(['data', '@type']) + assert objects['@type'] == 'user' + data = objects['data'] + assert isinstance(data, list) + assert len(data) == 1 + for o in data: + assert set(o.keys()) == set(['uuid', 'username', 'first_name', + 'is_superuser', 'last_name', 'email', 'roles']) + assert o['uuid'] == user.uuid + assert o['username'] == user.username + assert o['first_name'] == user.first_name + assert o['last_name'] == user.last_name + assert o['email'] == user.email + assert len(o['roles']) == 2 + for r in o['roles']: + r1 = { + 'uuid': role.uuid, + 'name': role.name, + 'slug': role.slug + } + r2 = { + 'uuid': child_role.uuid, + 'name': child_role.name, + 'slug': child_role.slug + } + assert r == r1 or r == r2 + assert len(set(r['uuid'] for r in o['roles'])) == 2 + assert o['is_superuser'] is True + + notify_agents.reset_mock() + with provisionning: + child_role.remove_parent(role) + + assert notify_agents.call_count == 1 + arg = notify_agents.call_args + assert arg == call(ANY) + arg = arg[0][0] + assert isinstance(arg, dict) + assert set(arg.keys()) == set([ + 'issuer', 'audience', '@type', 'objects', 'full']) + assert arg['issuer'] == \ + 'http://%s/idp/saml2/metadata' % tenant.domain_url + assert arg['audience'] == ['http://provider.com'] + assert arg['@type'] == 'provision' + assert arg['full'] is False + objects = arg['objects'] + assert isinstance(objects, dict) + assert set(objects.keys()) == set(['data', '@type']) + assert objects['@type'] == 'user' + data = objects['data'] + assert isinstance(data, list) + assert len(data) == 1 + for o in data: + assert set(o.keys()) == set(['uuid', 'username', 'first_name', + 'is_superuser', 'last_name', 'email', 'roles']) + assert o['uuid'] == user.uuid + assert o['username'] == user.username + assert o['first_name'] == user.first_name + assert o['last_name'] == user.last_name + assert o['email'] == user.email + assert o['roles'] == [{ + 'uuid': child_role.uuid, + 'name': child_role.name, + 'slug': child_role.slug + }] + assert o['is_superuser'] is False + notify_agents.reset_mock() - user.delete() + with provisionning: + user.delete() assert notify_agents.call_count == 1 arg = notify_agents.call_args assert arg == call(ANY) diff --git a/tox.ini b/tox.ini index f7e7ea5..7fb1199 100644 --- a/tox.ini +++ b/tox.ini @@ -31,12 +31,12 @@ deps: pytest-cov pytest-django pytest-mock + pytest-capturelog coverage raven cssselect WebTest django-mellon - multitenant: pytest-capturelog multitenant,passerelle: celery authentic: http://git.entrouvert.org/authentic.git/snapshot/authentic-master.tar.gz passerelle: http://git.entrouvert.org/passerelle.git/snapshot/passerelle-master.tar.gz -- 2.1.4