From 76b731af5cbbdb3db6d49e687f28555766b73302 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 27 Apr 2018 19:25:48 +0200 Subject: [PATCH] use pg_advisory_lock to prevent running cronjobs during migrations and on different servers (fixes #15470) --- .../management/commands/migrate_schemas.py | 5 ++++ .../management/commands/tenant_command.py | 4 ++++ tests_multitenant/test_tenant_command.py | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/hobo/multitenant/management/commands/migrate_schemas.py b/hobo/multitenant/management/commands/migrate_schemas.py index 4c81574..2eeff29 100644 --- a/hobo/multitenant/management/commands/migrate_schemas.py +++ b/hobo/multitenant/management/commands/migrate_schemas.py @@ -4,12 +4,14 @@ from optparse import NO_DEFAULT if django.VERSION >= (1, 7, 0): from django.core.management.commands.migrate import Command as MigrateCommand from django.db.migrations.recorder import MigrationRecorder +from django.core.management.base import CommandError from django.db import connection from django.conf import settings from tenant_schemas.utils import get_public_schema_name, schema_exists from hobo.multitenant.middleware import TenantMiddleware, TenantNotFound from hobo.multitenant.management.commands import SyncCommon +from hobo.multitenant.locking import take_global_lock class MigrateSchemasCommand(SyncCommon): @@ -34,6 +36,9 @@ class MigrateSchemasCommand(SyncCommon): def handle(self, *args, **options): super(MigrateSchemasCommand, self).handle(*args, **options) + # migration job can wait for finishing cron jobs + if not take_global_lock(wait=3600): + raise CommandError('global lock is taken, migration or another cron job is running') self.PUBLIC_SCHEMA_NAME = get_public_schema_name() if self.sync_public and not self.schema_name: diff --git a/hobo/multitenant/management/commands/tenant_command.py b/hobo/multitenant/management/commands/tenant_command.py index e80c9be..308ccb1 100644 --- a/hobo/multitenant/management/commands/tenant_command.py +++ b/hobo/multitenant/management/commands/tenant_command.py @@ -10,6 +10,8 @@ from django.db import connection from hobo.multitenant.management.commands import InteractiveTenantOption from hobo.multitenant.middleware import TenantMiddleware +from hobo.multitenant.locking import take_global_lock + class Command(InteractiveTenantOption, BaseCommand): help = "Wrapper around django commands for use with an individual tenant" @@ -43,6 +45,8 @@ class Command(InteractiveTenantOption, BaseCommand): args_namespace, args = args_parser.parse_known_args(argv) if args_namespace.all_tenants: + if not take_global_lock(): + raise CommandError('global lock is taken, migration or another cron job is running') for tenant in TenantMiddleware.get_tenants(): connection.set_tenant(tenant) klass.run_from_argv(args) diff --git a/tests_multitenant/test_tenant_command.py b/tests_multitenant/test_tenant_command.py index bd7b473..e45e947 100644 --- a/tests_multitenant/test_tenant_command.py +++ b/tests_multitenant/test_tenant_command.py @@ -1,6 +1,7 @@ import pytest import mock import os +import threading pytestmark = pytest.mark.django_db @@ -53,3 +54,25 @@ def test_delete_tenant(tenants): assert False all_tenants = list(TenantMiddleware.get_tenants()) assert len(all_tenants) == 1 + + +@mock.patch('django.contrib.sessions.management.commands.clearsessions.Command.handle') +def test_all_tenants_global_lock(handle, tenants): + from django.core.management import execute_from_command_line + from hobo.multitenant.locking import take_global_lock + + handle.side_effect = RecordTenant() + assert take_global_lock() + + l = [0] + + def thread_run(): + execute_from_command_line(['manage.py', 'tenant_command', 'clearsessions', '--all-tenants']) + l[0] = 1 + + t = threading.Thread(target=thread_run) + t.start() + t.join() + + assert l[0] == 0 + assert handle.call_count == 0 -- 2.17.0