From 5b582638d532c65a419c561a87ab7263ab8fd5a9 Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Fri, 23 Mar 2018 18:47:42 +0100 Subject: [PATCH] don't start a new cron if locked (#18519) --- tests/test_publisher.py | 28 ++++++++++++++++++++++++++++ wcs/qommon/management/commands/cron.py | 34 ++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/tests/test_publisher.py b/tests/test_publisher.py index c4184df0..df1efb93 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -8,8 +8,10 @@ import os import zipfile import mock +import pytest from django.core.management import call_command +from django.core.management.base import CommandError from quixote import cleanup from wcs.qommon.http_request import HTTPRequest @@ -172,3 +174,29 @@ def test_cron_command(): with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: call_command('cron') assert cron_worker.call_count == 1 + + # simulate another locked cron + import tempfile + from qommon.vendor import locket + lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock') + with locket.lock_file(lockfile, timeout=0): + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') # silent by default (verbosity=0) + assert cron_worker.call_count == 0 + call_command('cron', verbosity=2) # same if verbosity>0 + assert cron_worker.call_count == 0 + # verify that the lock is released + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') + assert cron_worker.call_count == 1 + + # simulate a cron crash + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + cron_worker.side_effect = NotImplementedError + with pytest.raises(NotImplementedError): + call_command('cron') + assert cron_worker.call_count == 1 + # verify that the lock is released + with mock.patch('wcs.qommon.management.commands.cron.cron_worker') as cron_worker: + call_command('cron') + assert cron_worker.call_count == 1 diff --git a/wcs/qommon/management/commands/cron.py b/wcs/qommon/management/commands/cron.py index da778bbc..ee1bf6b4 100644 --- a/wcs/qommon/management/commands/cron.py +++ b/wcs/qommon/management/commands/cron.py @@ -28,13 +28,27 @@ from qommon.cron import cron_worker class Command(BaseCommand): help = 'Execute cronjobs' - def handle(self, verbosity, **options): - with locket.lock_file(os.path.join(tempfile.gettempdir(), 'wcs-cron')): - now = time.localtime() - publisher_class = get_publisher_class() - publisher_class.register_cronjobs() - publisher = publisher_class.create_publisher() - app_dir = publisher.app_dir - for hostname in publisher.get_tenants(): - publisher.app_dir = os.path.join(app_dir, hostname) - cron_worker(publisher, now) + def add_arguments(self, parser): + parser.set_defaults(verbosity=0) + + def handle(self, verbosity, **kwargs): + lockfile = os.path.join(tempfile.gettempdir(), 'wcs-cron-in-progress.lock') + try: + with locket.lock_file(lockfile, timeout=0): + if verbosity > 2: + print('cron start (lock %s)' % lockfile) + now = time.localtime() + publisher_class = get_publisher_class() + publisher_class.register_cronjobs() + publisher = publisher_class.create_publisher() + app_dir = publisher.app_dir + for hostname in publisher.get_tenants(): + if verbosity > 1: + print('cron work on %s' % hostname) + publisher.app_dir = os.path.join(app_dir, hostname) + cron_worker(publisher, now) + if verbosity > 2: + print('cron end (release lock %s)' % lockfile) + except locket.LockError: + if verbosity > 0: + print('can not start cron job, locked by %s' % lockfile) -- 2.16.2