From b5c4d6f70236526222e99ac544e1a8c3cca32a5b Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Jaillet Date: Fri, 7 Apr 2017 16:03:11 +0200 Subject: [PATCH] command: add a delete_tenant command (#15636) --- tests/test_ctl.py | 139 +++++++++++++++++++++++++++++++++++++++++++++++ wcs/ctl/check_hobos.py | 9 ++- wcs/ctl/delete_tenant.py | 112 ++++++++++++++++++++++++++++++++++++++ wcs/sql.py | 42 +++++++++----- 4 files changed, 287 insertions(+), 15 deletions(-) create mode 100644 wcs/ctl/delete_tenant.py diff --git a/tests/test_ctl.py b/tests/test_ctl.py index d5fb0c72..ad70cfd0 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -3,6 +3,7 @@ import pytest import collections from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +import psycopg2 from wcs.formdef import FormDef from wcs.workflows import Workflow @@ -13,6 +14,8 @@ from wcs.ctl.collectstatic import CmdCollectStatic from wcs.ctl.process_bounce import CmdProcessBounce from wcs.ctl.wipe_data import CmdWipeData from wcs.ctl.trigger_jumps import select_and_jump_formdata +from wcs.ctl.delete_tenant import CmdDeleteTenant +from wcs.sql import get_connection_and_cursor, cleanup_connection from utilities import create_temporary_pub, clean_temporary_pub @@ -186,3 +189,139 @@ def test_trigger_jumps(pub): assert f1.status == f2.status == 'wf-%s' % st1.id assert not f1.workflow_data assert not f2.workflow_data + + +def test_delete_tenant_with_sql(): + pub = create_temporary_pub(sql_mode=True) + delete_cmd = CmdDeleteTenant() + + assert os.path.isdir(pub.app_dir) + + sub_options_class = collections.namedtuple('Options', ['force_drop']) + sub_options = sub_options_class(False) + + delete_cmd.delete_tenant(pub, sub_options, []) + + assert not os.path.isdir(pub.app_dir) + parent_dir = os.path.dirname(pub.app_dir) + if not [filename for filename in os.listdir(parent_dir) if 'removed' in filename]: + assert False + + conn, cur = get_connection_and_cursor() + cur.execute("""SELECT schema_name + FROM information_schema.schemata + WHERE schema_name like '%removed%'""") + + assert len(cur.fetchall()) == 1 + + clean_temporary_pub() + pub = create_temporary_pub(sql_mode=True) + + sub_options = sub_options_class(True) + delete_cmd.delete_tenant(pub, sub_options, []) + + conn, cur = get_connection_and_cursor(new=True) + + assert not os.path.isdir(pub.app_dir) + cur.execute("""SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE'""") + + assert not cur.fetchall() + + cur.execute("""SELECT datname + FROM pg_database + WHERE datname = '%s'""" % pub.cfg['postgresql']['database']) + + assert cur.fetchall() + + clean_temporary_pub() + pub = create_temporary_pub(sql_mode=True) + + cleanup_connection() + sub_options = sub_options_class(True) + pub.cfg['postgresql']['createdb-connection-params'] = { + 'user': pub.cfg['postgresql']['user'], + 'database': 'postgres' + } + delete_cmd.delete_tenant(pub, sub_options, []) + + pgconn = psycopg2.connect(**pub.cfg['postgresql']['createdb-connection-params']) + cur = pgconn.cursor() + + cur.execute("""SELECT datname + FROM pg_database + WHERE datname = '%s'""" % pub.cfg['postgresql']['database']) + assert not cur.fetchall() + cur.close() + pgconn.close() + + clean_temporary_pub() + pub = create_temporary_pub(sql_mode=True) + cleanup_connection() + + sub_options = sub_options_class(False) + pub.cfg['postgresql']['createdb-connection-params'] = { + 'user': pub.cfg['postgresql']['user'], + 'database': 'postgres' + } + delete_cmd.delete_tenant(pub, sub_options, []) + + pgconn = psycopg2.connect(**pub.cfg['postgresql']['createdb-connection-params']) + pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = pgconn.cursor() + + cur.execute("""SELECT datname + FROM pg_database + WHERE datname like '%removed%'""") + + result = cur.fetchall() + assert len(result) == 1 + + #clean this db after test + cur.execute("""DROP DATABASE %s""" % result[0][0]) + + cur.execute("""SELECT datname + FROM pg_database + WHERE datname = '%s'""" % pub.cfg['postgresql']['database']) + + assert not cur.fetchall() + cur.close() + conn.close() + + clean_temporary_pub() + + +def test_delete_tenant_without_sql(): + pub = create_temporary_pub() + delete_cmd = CmdDeleteTenant() + + assert os.path.isdir(pub.app_dir) + + sub_options_class = collections.namedtuple('Options', ['force_drop']) + sub_options = sub_options_class(False) + + delete_cmd.delete_tenant(pub, sub_options, []) + + assert not os.path.isdir(pub.app_dir) + parent_dir = os.path.dirname(pub.app_dir) + if not [filename for filename in os.listdir(parent_dir) if 'removed' in filename]: + assert False + + clean_temporary_pub() + + pub = create_temporary_pub() + assert os.path.isdir(pub.app_dir) + + sub_options = sub_options_class(True) + + delete_cmd.delete_tenant(pub, sub_options, []) + + assert not os.path.isdir(pub.app_dir) + parent_dir = os.path.dirname(pub.app_dir) + if [filename for filename in os.listdir(parent_dir) if 'removed' in filename]: + assert False + + clean_temporary_pub() + diff --git a/wcs/ctl/check_hobos.py b/wcs/ctl/check_hobos.py index a426d1c0..a13b26c8 100644 --- a/wcs/ctl/check_hobos.py +++ b/wcs/ctl/check_hobos.py @@ -422,7 +422,14 @@ class CmdCheckHobos(Command): cur.execute('''CREATE DATABASE %s''' % database_name) except psycopg2.Error as e: if e.pgcode == psycopg2.errorcodes.DUPLICATE_DATABASE: - new_database = False + cur.execute("""SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + AND table_name = 'wcs_meta'""") + + if cur.fetchall(): + new_database = False else: print >> sys.stderr, 'failed to create database (%s)' % \ psycopg2.errorcodes.lookup(e.pgcode) diff --git a/wcs/ctl/delete_tenant.py b/wcs/ctl/delete_tenant.py new file mode 100644 index 00000000..3af6d8b6 --- /dev/null +++ b/wcs/ctl/delete_tenant.py @@ -0,0 +1,112 @@ +#w.c.s. - web application for online forms +# Copyright (C) 2005-2014 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import os +import sys +import psycopg2 +import psycopg2.errorcodes +from datetime import datetime +from shutil import rmtree + +from qommon.ctl import Command, make_option + + +class CmdDeleteTenant(Command): + name = 'delete_tenant' + + def __init__(self): + Command.__init__(self, [ + make_option('--force-drop', action='store_true', default=False, + dest='force_drop'), + ]) + + def execute(self, base_options, sub_options, args): + import publisher + + publisher.WcsPublisher.configure(self.config) + pub = publisher.WcsPublisher.create_publisher( + register_cron=False, register_tld_names=False) + + hostname = args[0] + pub.app_dir = os.path.join(pub.app_dir, hostname) + pub.set_config() + self.delete_tenant(pub, sub_options, args) + + def delete_tenant(self, pub, options, args): + if options.force_drop: + rmtree(pub.app_dir) + else: + deletion_date = datetime.now().strftime('%Y%m%d_%H%M%S_%f') + os.rename(pub.app_dir, pub.app_dir + '_removed_%s.invalid' % deletion_date) + + # do this only if the wcs has a postgresql configuration + if pub.is_using_postgresql(): + postgresql_cfg = {} + for k, v in pub.cfg['postgresql'].items(): + if v and isinstance(v, basestring): + postgresql_cfg[k] = v + + # if there's a createdb-connection-params, we can do a DROP DATABASE with + # the option --force-drop, rename it if not + createdb_cfg = pub.cfg['postgresql'].get('createdb-connection-params', {}) + createdb = True + if not createdb_cfg: + createdb_cfg = postgresql_cfg + createdb = False + try: + pgconn = psycopg2.connect(**createdb_cfg) + except psycopg2.Error as e: + print >> sys.stderr, 'failed to connect to postgresql (%s)' % psycopg2.errorcodes.lookup(e.pgcode) + return + + pgconn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = pgconn.cursor() + try: + dbname = pub.cfg['postgresql']['database'] + if createdb: + if options.force_drop: + cur.execute('DROP DATABASE %s' % dbname) + else: + cur.execute('ALTER DATABASE %s RENAME TO removed_%s_%s' % (dbname, + deletion_date, + dbname)) + else: + cur.execute("""SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE'""") + + tables_names = [x[0] for x in cur.fetchall()] + + if options.force_drop: + for table_name in tables_names: + cur.execute('DROP TABLE %s CASCADE' % table_name) + + else: + schema_name = 'removed_%s_%s' % (deletion_date, dbname) + cur.execute("CREATE SCHEMA %s" % schema_name[:63]) + for table_name in tables_names: + cur.execute('ALTER TABLE %s SET SCHEMA %s' % + (table_name, schema_name[:63])) + + except psycopg2.Error as e: + print >> sys.stderr, 'failed to alter database %s: (%s)' % (createdb_cfg['database'], + psycopg2.errorcodes.lookup(e.pgcode)) + return + + cur.close() + +CmdDeleteTenant.register() diff --git a/wcs/sql.py b/wcs/sql.py index 850a759d..5de0d3c1 100644 --- a/wcs/sql.py +++ b/wcs/sql.py @@ -295,7 +295,8 @@ def get_formdef_new_id(id_start): conn, cur = get_connection_and_cursor() while True: cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name LIKE %s''', ('formdata\\_%s\\_%%' % new_id,)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('formdata\\_%s\\_%%' % new_id,)) if cur.fetchone()[0] == 0: break new_id += 1 @@ -306,7 +307,8 @@ def get_formdef_new_id(id_start): def formdef_wipe(): conn, cur = get_connection_and_cursor() cur.execute('''SELECT table_name FROM information_schema.tables - WHERE table_name LIKE %s''', ('formdata\\_%%\\_%%',)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('formdata\\_%%\\_%%',)) for table_name in [x[0] for x in cur.fetchall()]: cur.execute('''DROP TABLE %s CASCADE''' % table_name) conn.commit() @@ -338,7 +340,8 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild table_name = get_formdef_table_name(formdef) cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) if cur.fetchone()[0] == 0: cur.execute('''CREATE TABLE %s (id serial PRIMARY KEY, user_id varchar, @@ -358,7 +361,8 @@ def do_formdef_tables(formdef, conn=None, cur=None, rebuild_views=False, rebuild formdata_id integer REFERENCES %s (id) ON DELETE CASCADE)''' % ( table_name, table_name)) cur.execute('''SELECT column_name FROM information_schema.columns - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) existing_fields = set([x[0] for x in cur.fetchall()]) needed_fields = set(['id', 'user_id', 'receipt_time', @@ -469,7 +473,8 @@ def do_user_table(): table_name = 'users' cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) if cur.fetchone()[0] == 0: cur.execute('''CREATE TABLE %s (id serial PRIMARY KEY, name varchar, @@ -483,7 +488,8 @@ def do_user_table(): lasso_dump text, last_seen timestamp)''' % table_name) cur.execute('''SELECT column_name FROM information_schema.columns - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) existing_fields = set([x[0] for x in cur.fetchall()]) needed_fields = set(['id', 'name', 'email', 'roles', 'is_admin', @@ -546,13 +552,15 @@ def do_tracking_code_table(): table_name = 'tracking_codes' cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) if cur.fetchone()[0] == 0: cur.execute('''CREATE TABLE %s (id varchar PRIMARY KEY, formdef_id varchar, formdata_id varchar)''' % table_name) cur.execute('''SELECT column_name FROM information_schema.columns - WHERE table_name = %s''', (table_name,)) + WHERE table_schema = 'public' + AND table_name = %s''', (table_name,)) existing_fields = set([x[0] for x in cur.fetchall()]) needed_fields = set(['id', 'formdef_id', 'formdata_id']) @@ -572,7 +580,8 @@ def do_meta_table(conn=None, cur=None, insert_current_sql_level=True): conn, cur = get_connection_and_cursor() cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name = %s''', ('wcs_meta',)) + WHERE table_schema = 'public' + AND table_name = %s''', ('wcs_meta',)) if cur.fetchone()[0] == 0: cur.execute('''CREATE TABLE wcs_meta (id serial PRIMARY KEY, key varchar, @@ -607,11 +616,13 @@ def drop_views(formdef, conn, cur): if formdef: # remove the form view itself cur.execute('''SELECT table_name FROM information_schema.views - WHERE table_name LIKE %s''', ('wcs\\_view\\_%s\\_%%' % formdef.id ,)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('wcs\\_view\\_%s\\_%%' % formdef.id ,)) else: # if there's no formdef specified, remove all form views cur.execute('''SELECT table_name FROM information_schema.views - WHERE table_name LIKE %s''', ('wcs\\_view\\_%',)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('wcs\\_view\\_%',)) view_names = [] while True: row = cur.fetchone() @@ -715,7 +726,8 @@ def do_views(formdef, conn, cur, rebuild_global_views=True): def drop_global_views(conn, cur): cur.execute('''SELECT table_name FROM information_schema.views - WHERE table_name LIKE %s''', ('wcs\\_category\\_%',)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('wcs\\_category\\_%',)) view_names = [] while True: row = cur.fetchone() @@ -739,7 +751,8 @@ def do_global_views(conn, cur): view_names = [get_formdef_view_name(x) for x in FormDef.select()] cur.execute('''SELECT table_name FROM information_schema.views - WHERE table_name LIKE %s''', ('wcs\\_view\\_%',)) + WHERE table_schema = 'public' + AND table_name LIKE %s''', ('wcs\\_view\\_%',)) existing_views = set() while True: row = cur.fetchone() @@ -1884,7 +1897,8 @@ SQL_LEVEL = 22 def migrate_global_views(conn, cur): cur.execute('''SELECT COUNT(*) FROM information_schema.tables - WHERE table_name = %s''', ('wcs_all_forms',)) + WHERE table_schema = 'public' + AND table_name = %s''', ('wcs_all_forms',)) existing_fields = set([x[0] for x in cur.fetchall()]) if 'formdef_id' not in existing_fields: drop_global_views(conn, cur) -- 2.11.0