From f8e6a2a566ea04801074ad7ae9df0a1fdaab46d9 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Wed, 9 Sep 2015 09:28:15 +0200 Subject: [PATCH 2/3] add new agent task to provision objects to tenants (fixes #8217) First use is to connect it to post_save, post_delete signal on Role model of authentic, to propagate roles to tenants. --- MANIFEST.in | 1 + debian/agent/sudo-hobo-agent | 5 ++ debian/debian_config_common.py | 4 + hobo/agent/authentic2/apps.py | 73 ++++++++++++++++ .../authentic2/locale/fr/LC_MESSAGES/django.po | 35 ++++++++ hobo/agent/authentic2/role_forms.py | 96 ++++++++++++++++++++++ hobo/agent/common/__init__.py | 25 ++++++ .../common/management/commands/hobo_notify.py | 6 ++ hobo/agent/worker/celery.py | 8 ++ hobo/agent/worker/services.py | 33 +++++++- 10 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 hobo/agent/authentic2/locale/fr/LC_MESSAGES/django.po create mode 100644 hobo/agent/authentic2/role_forms.py create mode 100644 hobo/agent/common/management/commands/hobo_notify.py diff --git a/MANIFEST.in b/MANIFEST.in index 39b4e31..590d81b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ recursive-include hobo/templates *.html *.txt recursive-include hobo/profile/templates *.html *.txt recursive-include hobo/environment/templates *.html *.txt recursive-include hobo/locale *.po *.mo +recursive-include hobo/agent/authentic2/locale *.po *.mo recursive-include hobo/environment/locale *.po *.mo recursive-include tests *.py include hobo/multitenant/README diff --git a/debian/agent/sudo-hobo-agent b/debian/agent/sudo-hobo-agent index fc4839c..81bd93c 100644 --- a/debian/agent/sudo-hobo-agent +++ b/debian/agent/sudo-hobo-agent @@ -3,3 +3,8 @@ hobo-agent ALL=(authentic-multitenant)NOPASSWD:/usr/bin/authentic2-multitenant-m hobo-agent ALL=(combo)NOPASSWD:/usr/bin/combo-manage hobo_deploy * - hobo-agent ALL=(passerelle)NOPASSWD:/usr/bin/passerelle-manage hobo_deploy * - hobo-agent ALL=(fargo)NOPASSWD:/usr/bin/fargo-manage hobo_deploy * - +hobo-agent ALL=(wcs-au-quotidien)NOPASSWD:/usr/sbin/wcsctl -f /etc/wcs/wcs-au-quotidien.cfg hobo_notify - +hobo-agent ALL=(authentic-multitenant)NOPASSWD:/usr/bin/authentic2-multitenant-manage hobo_notify - +hobo-agent ALL=(combo)NOPASSWD:/usr/bin/combo-manage hobo_notify - +hobo-agent ALL=(passerelle)NOPASSWD:/usr/bin/passerelle-manage hobo_notify - +hobo-agent ALL=(fargo)NOPASSWD:/usr/bin/fargo-manage hobo_notify - diff --git a/debian/debian_config_common.py b/debian/debian_config_common.py index c28edb4..365832f 100644 --- a/debian/debian_config_common.py +++ b/debian/debian_config_common.py @@ -176,3 +176,7 @@ TIME_ZONE = 'Europe/Paris' LANGUAGES = (('fr', u'Fran\xe7ais'),) USE_L10N = True USE_TZ = True + +# Celery configuration +BROKER_URL = 'amqp://' +BROKER_TASK_EXPIRES = 120 diff --git a/hobo/agent/authentic2/apps.py b/hobo/agent/authentic2/apps.py index 0996c09..321568d 100644 --- a/hobo/agent/authentic2/apps.py +++ b/hobo/agent/authentic2/apps.py @@ -1,6 +1,79 @@ +import json + from django.apps import AppConfig +from django.db.models.signals import post_save, post_delete +from django.db.models import Q +from django.conf import settings + +from django_rbac.utils import get_role_model + +from hobo.agent.common import notify_agents +from authentic2.utils import to_list +from authentic2.saml.models import LibertyProvider + + +def get_ou(role_or_through): + if hasattr(role_or_through, 'ou'): + return role_or_through.ou + else: + return role_or_through.role.ou + + +def get_audience(role_or_through): + ou = get_ou(role_or_through) + if ou: + qs = LibertyProvider.objects.filter(ou=ou) + else: + qs = LibertyProvider.objects.filter(ou__isnull=True) + return list(qs.values_list('entity_id', flat=True)) + + +def get_related_roles(role_or_through): + ou = get_ou(role_or_through) + Role = get_role_model() + qs = Role.objects.filter(admin_scope_id__isnull=True) \ + .prefetch_related('attributes') + if ou: + qs = qs.filter(ou=ou) + else: + qs = qs.filter(ou__isnull=True) + for role in qs: + role.emails = [] + role.emails_to_members = False + for attribute in role.attributes.all(): + if attribute.name in ('emails', 'emails_to_members') and attribute.kind == 'json': + setattr(role, attribute.name, json.loads(attribute.value)) + return qs + + +def notify_roles(sender, instance, **kwargs): + notify_agents({ + '@type': 'provision', + 'audience': get_audience(instance), + 'full': True, + 'objects': [ + { + '@type': 'role', + 'uuid': role.uuid, + 'name': role.name, + 'slug': role.slug, + 'description': role.description, + 'emails': role.emails, + 'emails_to_members': role.emails_to_members, + } for role in get_related_roles(instance) + ] + }) + class Authentic2AgentConfig(AppConfig): name = 'hobo.agent.authentic2' label = 'authentic2_agent' verbose_name = 'Authentic2 Agent' + + def ready(self): + Role = get_role_model() + post_save.connect(notify_roles, Role) + post_delete.connect(notify_roles, Role) + post_save.connect(notify_roles, Role.members.through) + post_delete.connect(notify_roles, Role.members.through) + settings.A2_MANAGER_ROLE_FORM_CLASS = 'hobo.agent.authentic2.role_forms.RoleForm' diff --git a/hobo/agent/authentic2/locale/fr/LC_MESSAGES/django.po b/hobo/agent/authentic2/locale/fr/LC_MESSAGES/django.po new file mode 100644 index 0000000..3b27327 --- /dev/null +++ b/hobo/agent/authentic2/locale/fr/LC_MESSAGES/django.po @@ -0,0 +1,35 @@ +# Translation of hobo +# Copyright (C) 2015 Entr'ouvert +# This file is distributed under the same license as the hobo package. +# Benjamin Dauvergne , 2015 +# +msgid "" +msgstr "" +"Project-Id-Version: hobo 0\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-06-11 15:06+0200\n" +"PO-Revision-Date: 2014-03-24 19:31+0100\n" +"Last-Translator: Benjamin Dauvergne \n" +"Language: French\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" + +#: management/commands/hobo_deploy.py:148 +#: management/commands/import-wcs-roles.py:49 +msgid "Superuser" +msgstr "Super-utilisateur" + +#: role_forms.py:23 +#, python-brace-format +msgid "Item {0} is invalid: {1}" +msgstr "L'élément {0} est invalide: {1}" + +#: role_forms.py:62 +msgid "Emails" +msgstr "Courriels" + +#: role_forms.py:65 +msgid "Emails to members" +msgstr "Propager les courriels à tous les utilisateurs ayant ce rôle" diff --git a/hobo/agent/authentic2/role_forms.py b/hobo/agent/authentic2/role_forms.py new file mode 100644 index 0000000..716e250 --- /dev/null +++ b/hobo/agent/authentic2/role_forms.py @@ -0,0 +1,96 @@ +import json + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ + +from authentic2.a2_rbac.models import RoleAttribute, Role +from authentic2.validators import EmailValidator +from authentic2.manager.forms import RoleEditForm + + +class ListValidator(object): + def __init__(self, item_validator): + self.item_validator = item_validator + + def __call__(self, value): + for i, item in enumerate(value): + try: + self.item_validator(item) + except ValidationError, e: + raise ValidationError( + _('Item {0} is invalid: {1}') % (i, e.args[0])) + + +class CommaSeparatedInput(forms.TextInput): + def _format_value(self, value): + return u', '.join(value) + + +class CommaSeparatedCharField(forms.Field): + widget = CommaSeparatedInput + + def __init__(self, dedup=True, max_length=None, min_length=None, *args, + **kwargs): + self.dedup = dedup + self.max_length = max_length + self.min_length = min_length + item_validators = kwargs.pop('item_validators', []) + super(CommaSeparatedCharField, self).__init__(*args, **kwargs) + for item_validator in item_validators: + self.validators.append(ListValidator(item_validator)) + + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return [] + + value = [item.strip() for item in value.split(',') if item.strip()] + if self.dedup: + value = list(set(value)) + + return value + + def clean(self, value): + value = self.to_python(value) + self.validate(value) + self.run_validators(value) + return value + + +class RoleForm(RoleEditForm): + emails = CommaSeparatedCharField(label=_('Emails'), + item_validators=[EmailValidator()], + required=False) + emails_to_members = forms.BooleanField(required=False, + label=_('Emails to members')) + + def __init__(self, *args, **kwargs): + instance = kwargs.get('instance') + if instance: + fields = Role._meta.get_all_field_names() + initial = kwargs.setdefault('initial', {}) + role_attributes = RoleAttribute.objects.filter(role=instance, + kind='json') + for role_attribute in role_attributes: + if role_attribute.name in fields: + continue + initial[role_attribute.name] = json.loads(role_attribute.value) + super(RoleForm, self).__init__(*args, **kwargs) + + def save(self, commit=True): + fields = Role._meta.get_all_field_names() + assert commit + instance = super(RoleForm, self).save(commit=commit) + for field in self.cleaned_data: + if field in fields: + continue + value = json.dumps(self.cleaned_data[field]) + ra, created = RoleAttribute.objects.get_or_create( + role=instance, name=field, kind='json', + defaults={'value': value}) + if not created and ra.value != value: + ra.value = value + ra.save() + instance.save() + return instance diff --git a/hobo/agent/common/__init__.py b/hobo/agent/common/__init__.py index e69de29..5b50cc6 100644 --- a/hobo/agent/common/__init__.py +++ b/hobo/agent/common/__init__.py @@ -0,0 +1,25 @@ +from celery import Celery +from kombu.common import Broadcast + +from django.conf import settings +from django.db import connection + + +def notify_agents(data): + '''Send notifications to all other tenants''' + notification = { + 'tenant': connection.get_tenant().domain_url, + 'data': data, + } + with Celery('hobo', broker=settings.BROKER_URL) as app: + app.conf.update( + CELERY_TASK_SERIALIZER='json', + CELERY_ACCEPT_CONTENT=['json'], + CELERY_RESULT_SERIALIZER='json', + CELERY_QUEUES=(Broadcast('broadcast_tasks'), ) + ) + # see called method in hobo.agent.worker.celery + app.send_task('hobo-notify', + (notification,), + expires=settings.BROKER_TASK_EXPIRES, + queue='broadcast_tasks') diff --git a/hobo/agent/common/management/commands/hobo_notify.py b/hobo/agent/common/management/commands/hobo_notify.py new file mode 100644 index 0000000..01c2cd0 --- /dev/null +++ b/hobo/agent/common/management/commands/hobo_notify.py @@ -0,0 +1,6 @@ +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + pass diff --git a/hobo/agent/worker/celery.py b/hobo/agent/worker/celery.py index 3db4b0d..6292013 100644 --- a/hobo/agent/worker/celery.py +++ b/hobo/agent/worker/celery.py @@ -13,6 +13,14 @@ app.conf.update( CELERY_QUEUES=(Broadcast('broadcast_tasks'), ) ) + @app.task(name='hobo-deploy', bind=True) def deploy(self, environment): services.deploy(environment) + + +@app.task(name='hobo-notify', bind=True, acks_late=True) +def hobo_notify(self, notification): + assert 'tenant' in notification + assert 'data' in notification + services.notify(notification['data']) diff --git a/hobo/agent/worker/services.py b/hobo/agent/worker/services.py index 01cfff9..42956fe 100644 --- a/hobo/agent/worker/services.py +++ b/hobo/agent/worker/services.py @@ -1,3 +1,4 @@ +import sys import ConfigParser import fnmatch import json @@ -19,7 +20,8 @@ class BaseService(object): self.title = title self.secret_key = secret_key - def is_for_us(self): + @classmethod + def is_for_us(cls, url): # This function checks if the requested service is to be hosted # on this server, and return True if appropriate. # @@ -30,10 +32,10 @@ class BaseService(object): # (ex: "! *.dev.au-quotidien.com"). if not settings.AGENT_HOST_PATTERNS: return True - patterns = settings.AGENT_HOST_PATTERNS.get(self.service_id) + patterns = settings.AGENT_HOST_PATTERNS.get(cls.service_id) if patterns is None: return True - parsed_url = urllib2.urlparse.urlsplit(self.base_url) + parsed_url = urllib2.urlparse.urlsplit(url) netloc = parsed_url.netloc match = False for pattern in patterns: @@ -55,6 +57,23 @@ class BaseService(object): shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdout = cmd_process.communicate(input=json.dumps(environment)) + @classmethod + def notify(cls, data): + for audience in data.get('audience', []): + if cls.is_for_us(audience): + break + else: + return + cmd = cls.service_manage_cmd + ' hobo_notify -' + try: + cmd_process = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + return + stdout, stderr = cmd_process.communicate(input=json.dumps(data)) + if cmd_process.returncode != 0: + raise RuntimeError('command "%s" failed: %r %r' % (cmd, stdout, stderr)) + class Passerelle(BaseService): service_id = 'passerelle' @@ -106,10 +125,16 @@ def deploy(environment): if not service_id in service_classes: continue service_obj = service_classes.get(service_id)(**service) - if not service_obj.is_for_us(): + if not service_obj.is_for_us(service_obj.base_url): logger.debug('skipping as not for us: %r', service_obj) continue if service_obj.check_timestamp(hobo_timestamp): logger.debug('skipping uptodate site: %r', service_obj) continue service_obj.execute(environment) + +def notify(data): + for klassname, service in globals().items(): + if not hasattr(service, 'service_id'): + continue + service.notify(data) -- 2.1.4