From de633762791cb91bcda74ec62b0d952fe6293370 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Tue, 19 May 2020 18:32:22 +0200 Subject: [PATCH 1/3] misc: remove SQL objects in order to use less locks (#43108) --- wcs_olap/feeder.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/wcs_olap/feeder.py b/wcs_olap/feeder.py index 47d3507..036b69b 100644 --- a/wcs_olap/feeder.py +++ b/wcs_olap/feeder.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from collections import OrderedDict +import contextlib import datetime import six import copy @@ -12,6 +13,7 @@ import json import hashlib from .utils import Whatever import psycopg2 +import psycopg2.errorcodes from cached_property import cached_property from wcs_olap.wcs_api import WcsApiError @@ -20,6 +22,15 @@ psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY) +@contextlib.contextmanager +def ignore_undefined_object_or_table(): + try: + yield + except psycopg2.ProgrammingError as e: + if e.pgcode not in [psycopg2.errorcodes.UNDEFINED_TABLE, psycopg2.errorcodes.UNDEFINED_OBJECT]: + raise + + def quote(name): return '"%s"' % name @@ -326,10 +337,36 @@ class WcsOlapFeeder(object): """ Drop tables one by one in order to avoid reaching max_locks_per_transaction """ - self.ex("SELECT tablename FROM pg_tables WHERE schemaname = '%s'" % schema) + # drop foreign key constraints first + self.ex("SELECT table_name, constraint_name FROM " + "information_schema.key_column_usage " + "WHERE table_schema = %s AND constraint_name LIKE '%%_fkey'", vars=[schema]) + for table_name, constraint_name in self.cur.fetchall(): + # drop of PK constraints can have effects on FK constraint on other tables. + with ignore_undefined_object_or_table(): + print('Dropping s', constraint_name) + self.ex('ALTER TABLE %s.%s DROP CONSTRAINT IF EXISTS %s CASCADE' + % (quote(schema), quote(table_name), quote(constraint_name))) + # remove others + self.ex("SELECT table_name, constraint_name FROM " + "information_schema.key_column_usage " + "WHERE table_schema = %s", vars=[schema]) + for table_name, constraint_name in self.cur.fetchall(): + # drop of PK constraints can have effects on FK constraint on other tables. + with ignore_undefined_object_or_table(): + self.ex('ALTER TABLE %s.%s DROP CONSTRAINT IF EXISTS %s CASCADE' + % (quote(schema), quote(table_name), quote(constraint_name))) + # then drop indexes + self.ex("SELECT tablename, indexname FROM pg_indexes WHERE schemaname = %s", vars=[schema]) + for table_name, index_name in self.cur.fetchall(): + with ignore_undefined_object_or_table(): + self.ex('DROP INDEX %s.%s CASCADE' % (quote(schema), quote(index_name))) + + # finally drop tables, cascade will have no effect + self.ex("SELECT tablename FROM pg_tables WHERE schemaname = %s ORDER BY tablename DESC", vars=[schema]) for table in self.cur.fetchall(): tablename = '%s.%s' % (quote(schema), quote(table[0])) - self.ex('DROP TABLE IF EXISTS %s CASCADE;' % tablename) + self.ex('DROP TABLE IF EXISTS %s;' % tablename) def do_dates_table(self): self.ex("DROP TABLE IF EXISTS public.dates") -- 2.26.2