From 47d8c6513ec7cc5a428c1273b0e132eb12e74238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sun, 23 Aug 2015 19:33:43 +0200 Subject: [PATCH] agent: deploy twice in case of new services (#7550) When a service is added a request to deploy it will be sent to all servers and after sometime the new service will be online. In the meantime however the authentic service has tried to register the new service SAML metadata and failed to find them. Therefore when a new service is added a new message is broadcasted (new-site) and this will allow the hobo master site to send a new deployment message (and this time authentic will find the metadata). --- .../common/management/commands/hobo_deploy.py | 12 ++++++++++- hobo/agent/worker/celery.py | 25 ++++++++++++++++++++-- hobo/agent/worker/services.py | 13 +++++++++++ hobo/agent/worker/settings.py | 3 +++ 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/hobo/agent/common/management/commands/hobo_deploy.py b/hobo/agent/common/management/commands/hobo_deploy.py index a258b0e..6961a78 100644 --- a/hobo/agent/common/management/commands/hobo_deploy.py +++ b/hobo/agent/common/management/commands/hobo_deploy.py @@ -33,9 +33,11 @@ class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('--ignore-timestamp', dest='ignore_timestamp', action="store_true", default=False), + make_option('--exists', dest='exists', + action="store_true", default=False), ) - def handle(self, base_url, json_filename, ignore_timestamp, *args, **kwargs): + def handle(self, base_url, json_filename, ignore_timestamp, exists, *args, **kwargs): if json_filename == '-': hobo_environment = json.load(sys.stdin) else: @@ -43,6 +45,14 @@ class Command(BaseCommand): me = [x for x in hobo_environment.get('services') if x.get('base_url') == base_url][0] domain = urlparse.urlparse(me.get('base_url')).netloc.split(':')[0] + if exists: + # exit with 0 if the site exists already, 1 if it doesn't. + try: + TenantMiddleware.get_tenant_by_hostname(domain) + sys.exit(0) + except TenantNotFound: + sys.exit(1) + try: tenant = TenantMiddleware.get_tenant_by_hostname(domain) except TenantNotFound: diff --git a/hobo/agent/worker/celery.py b/hobo/agent/worker/celery.py index 3db4b0d..84bca84 100644 --- a/hobo/agent/worker/celery.py +++ b/hobo/agent/worker/celery.py @@ -1,4 +1,6 @@ from __future__ import absolute_import +import datetime +import time from celery import Celery from kombu.common import Broadcast @@ -10,9 +12,28 @@ app.conf.update( CELERY_TASK_SERIALIZER='json', CELERY_ACCEPT_CONTENT=['json'], CELERY_RESULT_SERIALIZER='json', - CELERY_QUEUES=(Broadcast('broadcast_tasks'), ) + CELERY_QUEUES=(Broadcast('broadcast_tasks'), + Broadcast('hobo_broadcast_tasks')) ) @app.task(name='hobo-deploy', bind=True) def deploy(self, environment): - services.deploy(environment) + has_new_service = services.deploy(environment) + if has_new_service: + # in case of a new service a "new-site" broadcast call is made and will + # get caught by the agent running on the master hobo site. + app.send_task('new-site', + (environment,), + expires=settings.BROKER_TASK_EXPIRES, + queue='hobo_broadcast_tasks') + +@app.task(name='new-site', bind=True) +def new_site(self, environment): + # we send a request for a new deployment, so existing services have a + # chance to take their new sibling in consideration. + timestamp = datetime.datetime.now() + environment['timestamp'] = str(time.mktime(timestamp.timetuple()) + timestamp.microsecond/1e6) + app.send_task('hobo-deploy', + (environment,), + expires=settings.BROKER_TASK_EXPIRES, + queue='broadcast_tasks') diff --git a/hobo/agent/worker/services.py b/hobo/agent/worker/services.py index 5f034a2..5e5d772 100644 --- a/hobo/agent/worker/services.py +++ b/hobo/agent/worker/services.py @@ -50,6 +50,13 @@ class BaseService(object): '''Return True if site is uptodate''' return False + def exists(self, environment): + cmd_process = subprocess.Popen( + self.service_manage_cmd + ' hobo_deploy ' + ' --exists ' + self.base_url + ' -', + shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = cmd_process.communicate(input=json.dumps(environment)) + return (cmd_process.returncode != 1) + def execute(self, environment): cmd_process = subprocess.Popen(self.service_manage_cmd + ' hobo_deploy ' + self.base_url + ' -', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) @@ -106,6 +113,7 @@ def deploy(environment): continue service_classes[service.service_id] = service + has_new_service = False for service in environment.get('services', []): service_id = service.get('service-id') if not service_id in service_classes: @@ -117,4 +125,9 @@ def deploy(environment): if service_obj.check_timestamp(hobo_timestamp): logger.debug('skipping uptodate site: %r', service_obj) continue + existing_service = service_obj.exists(environment) service_obj.execute(environment) + if not existing_service and service_obj.exists(environment): + has_new_service = service_obj.base_url + + return has_new_service diff --git a/hobo/agent/worker/settings.py b/hobo/agent/worker/settings.py index b26afd5..caab3b1 100644 --- a/hobo/agent/worker/settings.py +++ b/hobo/agent/worker/settings.py @@ -3,6 +3,9 @@ import os # AMQP message broker BROKER_URL = 'amqp://' +# Task expiry time, as seconds after task publish +BROKER_TASK_EXPIRES = 120 + # It's possible to limit agents to particular applications, or particular # hostnames, using the AGENT_HOST_PATTERNS configuration variable. # -- 2.5.0