From 7c6c85c812d9529b8869cdbd03fa8d32d9e03274 Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Fri, 23 Mar 2018 18:47:42 +0100 Subject: [PATCH] don't start a new cron if locked (#18519) --- tests/test_publisher.py | 29 +++++++++++++++++++++++++++++ wcs/qommon/management/commands/cron.py | 32 +++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/tests/test_publisher.py b/tests/test_publisher.py index c4184df0..25fb8893 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -8,8 +8,10 @@ import os import zipfile import mock +import pytest from django.core.management import call_command +from django.core.management.base import CommandError from quixote import cleanup from wcs.qommon.http_request import HTTPRequest @@ -172,3 +174,30 @@ def test_cron_command(): with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: call_command('cron') assert cron_worker.call_count == 1 + + # simulate another locked cron + import tempfile + from qommon.vendor import locket + lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock') + with locket.lock_file(lockfile, timeout=0): + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') # silent by default (verbosity=0) + assert cron_worker.call_count == 0 + with pytest.raises(CommandError, match='can not start cron job, locked'): + call_command('cron', verbosity=1) # CommandError with verbosity>0 + assert cron_worker.call_count == 0 + # verify that the lock is released + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') + assert cron_worker.call_count == 1 + + # simulate a cron crash + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + cron_worker.side_effect = NotImplementedError + with pytest.raises(NotImplementedError): + call_command('cron') + assert cron_worker.call_count == 1 + # verify that the lock is released + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') + assert cron_worker.call_count == 1 diff --git a/wcs/qommon/management/commands/cron.py b/wcs/qommon/management/commands/cron.py index da778bbc..121a362a 100644 --- a/wcs/qommon/management/commands/cron.py +++ b/wcs/qommon/management/commands/cron.py @@ -18,9 +18,10 @@ import tempfile import time import os -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from qommon.publisher import get_publisher_class +from qommon import get_logger, get_publisher from qommon.vendor import locket from qommon.cron import cron_worker @@ -28,13 +29,22 @@ from qommon.cron import cron_worker class Command(BaseCommand): help = 'Execute cronjobs' - def handle(self, verbosity, **options): - with locket.lock_file(os.path.join(tempfile.gettempdir(), 'wcs-cron')): - now = time.localtime() - publisher_class = get_publisher_class() - publisher_class.register_cronjobs() - publisher = publisher_class.create_publisher() - app_dir = publisher.app_dir - for hostname in publisher.get_tenants(): - publisher.app_dir = os.path.join(app_dir, hostname) - cron_worker(publisher, now) + def add_arguments(self, parser): + super(Command, self).add_arguments(parser) + parser.set_defaults(verbosity=0) + + def handle(self, verbosity, **kwargs): + lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock') + try: + with locket.lock_file(lockfile, timeout=0): + now = time.localtime() + publisher_class = get_publisher_class() + publisher_class.register_cronjobs() + publisher = publisher_class.create_publisher() + app_dir = publisher.app_dir + for hostname in publisher.get_tenants(): + publisher.app_dir = os.path.join(app_dir, hostname) + cron_worker(publisher, now) + except locket.LockError: + if verbosity > 0: + raise CommandError('can not start cron job, locked by %s' % lockfile) -- 2.16.2