From 40c5d4414bd3215b01562ea6e4dbaace47fdaa2d Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Wed, 9 Sep 2015 09:28:15 +0200 Subject: [PATCH] add new agent task to provision objects to tenants (fixes #8217) First use is to connect it to post_save, post_delete signal on Role model of authentic, to propagate roles to tenants. --- debian/agent/sudo-hobo-agent | 5 +++ debian/debian_config_common.py | 4 +++ hobo/agent/authentic2/apps.py | 73 ++++++++++++++++++++++++++++++++++++++++++ hobo/agent/common/__init__.py | 25 +++++++++++++++ hobo/agent/worker/celery.py | 8 +++++ hobo/agent/worker/services.py | 33 ++++++++++++++++--- 6 files changed, 144 insertions(+), 4 deletions(-) diff --git a/debian/agent/sudo-hobo-agent b/debian/agent/sudo-hobo-agent index fc4839c..81bd93c 100644 --- a/debian/agent/sudo-hobo-agent +++ b/debian/agent/sudo-hobo-agent @@ -3,3 +3,8 @@ hobo-agent ALL=(authentic-multitenant)NOPASSWD:/usr/bin/authentic2-multitenant-m hobo-agent ALL=(combo)NOPASSWD:/usr/bin/combo-manage hobo_deploy * - hobo-agent ALL=(passerelle)NOPASSWD:/usr/bin/passerelle-manage hobo_deploy * - hobo-agent ALL=(fargo)NOPASSWD:/usr/bin/fargo-manage hobo_deploy * - +hobo-agent ALL=(wcs-au-quotidien)NOPASSWD:/usr/sbin/wcsctl -f /etc/wcs/wcs-au-quotidien.cfg hobo_notify - +hobo-agent ALL=(authentic-multitenant)NOPASSWD:/usr/bin/authentic2-multitenant-manage hobo_notify - +hobo-agent ALL=(combo)NOPASSWD:/usr/bin/combo-manage hobo_notify - +hobo-agent ALL=(passerelle)NOPASSWD:/usr/bin/passerelle-manage hobo_notify - +hobo-agent ALL=(fargo)NOPASSWD:/usr/bin/fargo-manage hobo_notify - diff --git a/debian/debian_config_common.py b/debian/debian_config_common.py index c28edb4..365832f 100644 --- a/debian/debian_config_common.py +++ b/debian/debian_config_common.py @@ -176,3 +176,7 @@ TIME_ZONE = 'Europe/Paris' LANGUAGES = (('fr', u'Fran\xe7ais'),) USE_L10N = True USE_TZ = True + +# Celery configuration +BROKER_URL = 'amqp://' +BROKER_TASK_EXPIRES = 120 diff --git a/hobo/agent/authentic2/apps.py b/hobo/agent/authentic2/apps.py index 0996c09..321568d 100644 --- a/hobo/agent/authentic2/apps.py +++ b/hobo/agent/authentic2/apps.py @@ -1,6 +1,79 @@ +import json + from django.apps import AppConfig +from django.db.models.signals import post_save, post_delete +from django.db.models import Q +from django.conf import settings + +from django_rbac.utils import get_role_model + +from hobo.agent.common import notify_agents +from authentic2.utils import to_list +from authentic2.saml.models import LibertyProvider + + +def get_ou(role_or_through): + if hasattr(role_or_through, 'ou'): + return role_or_through.ou + else: + return role_or_through.role.ou + + +def get_audience(role_or_through): + ou = get_ou(role_or_through) + if ou: + qs = LibertyProvider.objects.filter(ou=ou) + else: + qs = LibertyProvider.objects.filter(ou__isnull=True) + return list(qs.values_list('entity_id', flat=True)) + + +def get_related_roles(role_or_through): + ou = get_ou(role_or_through) + Role = get_role_model() + qs = Role.objects.filter(admin_scope_id__isnull=True) \ + .prefetch_related('attributes') + if ou: + qs = qs.filter(ou=ou) + else: + qs = qs.filter(ou__isnull=True) + for role in qs: + role.emails = [] + role.emails_to_members = False + for attribute in role.attributes.all(): + if attribute.name in ('emails', 'emails_to_members') and attribute.kind == 'json': + setattr(role, attribute.name, json.loads(attribute.value)) + return qs + + +def notify_roles(sender, instance, **kwargs): + notify_agents({ + '@type': 'provision', + 'audience': get_audience(instance), + 'full': True, + 'objects': [ + { + '@type': 'role', + 'uuid': role.uuid, + 'name': role.name, + 'slug': role.slug, + 'description': role.description, + 'emails': role.emails, + 'emails_to_members': role.emails_to_members, + } for role in get_related_roles(instance) + ] + }) + class Authentic2AgentConfig(AppConfig): name = 'hobo.agent.authentic2' label = 'authentic2_agent' verbose_name = 'Authentic2 Agent' + + def ready(self): + Role = get_role_model() + post_save.connect(notify_roles, Role) + post_delete.connect(notify_roles, Role) + post_save.connect(notify_roles, Role.members.through) + post_delete.connect(notify_roles, Role.members.through) + settings.A2_MANAGER_ROLE_FORM_CLASS = 'hobo.agent.authentic2.role_forms.RoleForm' diff --git a/hobo/agent/common/__init__.py b/hobo/agent/common/__init__.py index e69de29..5b50cc6 100644 --- a/hobo/agent/common/__init__.py +++ b/hobo/agent/common/__init__.py @@ -0,0 +1,25 @@ +from celery import Celery +from kombu.common import Broadcast + +from django.conf import settings +from django.db import connection + + +def notify_agents(data): + '''Send notifications to all other tenants''' + notification = { + 'tenant': connection.get_tenant().domain_url, + 'data': data, + } + with Celery('hobo', broker=settings.BROKER_URL) as app: + app.conf.update( + CELERY_TASK_SERIALIZER='json', + CELERY_ACCEPT_CONTENT=['json'], + CELERY_RESULT_SERIALIZER='json', + CELERY_QUEUES=(Broadcast('broadcast_tasks'), ) + ) + # see called method in hobo.agent.worker.celery + app.send_task('hobo-notify', + (notification,), + expires=settings.BROKER_TASK_EXPIRES, + queue='broadcast_tasks') diff --git a/hobo/agent/worker/celery.py b/hobo/agent/worker/celery.py index 3db4b0d..6292013 100644 --- a/hobo/agent/worker/celery.py +++ b/hobo/agent/worker/celery.py @@ -13,6 +13,14 @@ app.conf.update( CELERY_QUEUES=(Broadcast('broadcast_tasks'), ) ) + @app.task(name='hobo-deploy', bind=True) def deploy(self, environment): services.deploy(environment) + + +@app.task(name='hobo-notify', bind=True, acks_late=True) +def hobo_notify(self, notification): + assert 'tenant' in notification + assert 'data' in notification + services.notify(notification['data']) diff --git a/hobo/agent/worker/services.py b/hobo/agent/worker/services.py index 01cfff9..42956fe 100644 --- a/hobo/agent/worker/services.py +++ b/hobo/agent/worker/services.py @@ -1,3 +1,4 @@ +import sys import ConfigParser import fnmatch import json @@ -19,7 +20,8 @@ class BaseService(object): self.title = title self.secret_key = secret_key - def is_for_us(self): + @classmethod + def is_for_us(cls, url): # This function checks if the requested service is to be hosted # on this server, and return True if appropriate. # @@ -30,10 +32,10 @@ class BaseService(object): # (ex: "! *.dev.au-quotidien.com"). if not settings.AGENT_HOST_PATTERNS: return True - patterns = settings.AGENT_HOST_PATTERNS.get(self.service_id) + patterns = settings.AGENT_HOST_PATTERNS.get(cls.service_id) if patterns is None: return True - parsed_url = urllib2.urlparse.urlsplit(self.base_url) + parsed_url = urllib2.urlparse.urlsplit(url) netloc = parsed_url.netloc match = False for pattern in patterns: @@ -55,6 +57,23 @@ class BaseService(object): shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdout = cmd_process.communicate(input=json.dumps(environment)) + @classmethod + def notify(cls, data): + for audience in data.get('audience', []): + if cls.is_for_us(audience): + break + else: + return + cmd = cls.service_manage_cmd + ' hobo_notify -' + try: + cmd_process = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + return + stdout, stderr = cmd_process.communicate(input=json.dumps(data)) + if cmd_process.returncode != 0: + raise RuntimeError('command "%s" failed: %r %r' % (cmd, stdout, stderr)) + class Passerelle(BaseService): service_id = 'passerelle' @@ -106,10 +125,16 @@ def deploy(environment): if not service_id in service_classes: continue service_obj = service_classes.get(service_id)(**service) - if not service_obj.is_for_us(): + if not service_obj.is_for_us(service_obj.base_url): logger.debug('skipping as not for us: %r', service_obj) continue if service_obj.check_timestamp(hobo_timestamp): logger.debug('skipping uptodate site: %r', service_obj) continue service_obj.execute(environment) + +def notify(data): + for klassname, service in globals().items(): + if not hasattr(service, 'service_id'): + continue + service.notify(data) -- 2.1.4