From 62bfa9fd2d21e46419b001d7682703affa15ff4c Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Jaillet Date: Tue, 25 Jul 2017 01:31:07 +0200 Subject: [PATCH] ctl: add dump in backup command if postgresql is true (#17726) --- tests/test_ctl.py | 40 ++++++++++++++++++ wcs/ctl/backup.py | 120 +++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 141 insertions(+), 19 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index ca6309b1..0e01e855 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -4,6 +4,8 @@ import collections from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart import psycopg2 +import shutil +import tarfile from wcs.formdef import FormDef from wcs.workflows import Workflow @@ -15,6 +17,7 @@ from wcs.ctl.process_bounce import CmdProcessBounce from wcs.ctl.wipe_data import CmdWipeData from wcs.ctl.trigger_jumps import select_and_jump_formdata from wcs.ctl.delete_tenant import CmdDeleteTenant +from wcs.ctl.backup import CmdBackup from wcs.sql import get_connection_and_cursor, cleanup_connection from utilities import create_temporary_pub, clean_temporary_pub @@ -324,3 +327,40 @@ def test_delete_tenant_without_sql(): assert False clean_temporary_pub() + + +def test_backup_tenant(tmpdir_factory): + pub = create_temporary_pub(sql_mode=True) + backup_cmd = CmdBackup() + test_dir = tmpdir_factory.mktemp('test').strpath + + sub_options_class = collections.namedtuple('Options', ['destination']) + sub_options = sub_options_class(test_dir) + + backup_cmd.backup_instance(pub, sub_options, ['example.test.com']) + + tar = [tar for tar in os.listdir(test_dir) if tar.endswith('.tar.gz')] + assert tar + + with tarfile.open(os.path.join(test_dir, tar[0]), 'r:gz') as tar_file: + tar_list = tar_file.getnames() + + assert [dump for dump in tar_list if dump.endswith('.sql')] + + clean_temporary_pub() + + pub = create_temporary_pub() + test_dir = tmpdir_factory.mktemp('test2').strpath + + sub_options = sub_options_class(test_dir) + backup_cmd.backup_instance(pub, sub_options, ['example.test.com']) + + tar = [t for t in os.listdir(test_dir) if t.endswith('.tar.gz')] + assert tar + with tarfile.open(os.path.join(test_dir, tar[0]), 'r:gz') as tar_file: + tar_list = tar_file.getnames() + + assert tar_list + assert not [dump for dump in tar_list if dump.endswith('.sql')] + + clean_temporary_pub() diff --git a/wcs/ctl/backup.py b/wcs/ctl/backup.py index 1654e8b1..548379fb 100644 --- a/wcs/ctl/backup.py +++ b/wcs/ctl/backup.py @@ -15,19 +15,27 @@ # along with this program; if not, see . import tarfile -import time import os +import sys +import shutil +import subprocess +import tempfile +import datetime from qommon.ctl import Command, make_option +class CmdError(Exception): + pass + + class CmdBackup(Command): name = 'backup' def __init__(self): Command.__init__(self, [ - make_option('--file', metavar='FILENAME', action='store', - dest='filename', default=None) + make_option('--destination', action='store', + dest='destination', default=None) ]) def execute(self, base_options, sub_options, args): @@ -42,25 +50,99 @@ class CmdBackup(Command): if not os.path.exists(pub.app_dir): return 1 - if sub_options.filename: - backup_filepath = sub_options.filename + pub.set_config() + try: + self.backup_instance(pub, sub_options, args) + except CmdError as e: + print >> sys.stderr, str(e) + return 1 + + def backup_instance(self, pub, sub_options, args): + instance_name = args[0].replace('.', '_').replace('-', '_') + file_name = 'wcs__%s__%s' % (instance_name, + datetime.date.today().isoformat()) + dest_dir = sub_options.destination + if dest_dir: + if not os.path.exists(dest_dir): + raise CmdError('destination directory does not exist') else: - backup_dir = os.path.join(pub.app_dir, 'backups') - if not os.path.exists(backup_dir): - os.mkdir(backup_dir) - backup_filepath = os.path.join(backup_dir, - 'backup-%s%s%s-%s%s%s.tar.gz' % time.localtime()[:6]) - - backup = tarfile.open(backup_filepath, mode='w:gz') - for basename, dirnames, filenames in os.walk(pub.app_dir): - if 'backups' in dirnames: # do not recurse in backup directory - idx = dirnames.index('backups') - dirnames[idx:idx+1] = [] - for filename in filenames: - backup.add(os.path.join(basename, filename), - os.path.join(basename, filename)[len(pub.app_dir):]) + dest_dir = os.path.join(pub.app_dir, 'backups') + if not os.path.exists(dest_dir): + os.mkdir(dest_dir) + try: + temp_dir = tempfile.mkdtemp() + except IOError as e: + raise CmdError('an error occurred while creating temporary directory: %s' % e.message) + + try: + dump_path = '' + if pub.is_using_postgresql(): + dump_path = self.backup_postgresql(pub, file_name, temp_dir) + + instance_tar_path = self.backup_instance_dir(pub, file_name, temp_dir, dump_path) + try: + shutil.move(instance_tar_path, dest_dir) + except IOError as e: + raise CmdError('could not move instance tar %s to %s: %s' % (instance_tar_path, + dest_dir, + e.message)) + finally: + try: + shutil.rmtree(temp_dir) + except IOError as e: + print 'could not remove temporary directory %s: %s' % (temp_dir, e.message) + + + def backup_instance_dir(self, pub, file_name, temp_dir, dump_path=''): + backup_filepath = os.path.join(temp_dir, '%s.tar.gz' % file_name) + try: + backup = tarfile.open(backup_filepath, mode='w:gz') + + for basename, dirnames, filenames in os.walk(pub.app_dir): + if 'backups' in dirnames: # do not recurse in backup directory + idx = dirnames.index('backups') + dirnames[idx:idx+1] = [] + for filename in filenames: + backup.add(os.path.join(basename, filename), + os.path.join(basename, filename)[len(pub.app_dir):]) + if dump_path: + backup.add(dump_path, os.path.basename(dump_path)) + + except IOError as e: + raise CmdError('could not open tar file %s: %s' % (backup_filepath, + e.message)) + except tarfile.TarError: + raise CmdError('an error occured while making tar archive for %s' % file_name) backup.close() + return backup_filepath + + def backup_postgresql(self, pub, file_name, temp_dir): + db_conf = pub.cfg['postgresql'] + # use 'or' as value are at None when not set + db_name = db_conf.get('database') or '' + db_host = db_conf.get('host') or '' + db_user = db_conf.get('user') or '' + db_port = db_conf.get('port') or '' + db_pwd = db_conf.get('password') or '' + if db_pwd: + os.environ['PGPASSWORD'] = db_pwd + + dump_path = os.path.join(temp_dir, '%s.sql' % file_name) + + try: + subprocess.check_call(['pg_dump', '-Oc', '-n', 'public', + '-h', db_host, '-U', db_user, + '-p', db_port, '-w', '-f', dump_path, + '-d', db_name]) + except subprocess.CalledProcessError as e: + raise CmdError('an error occured while dumping instance database %s: %s' % (db_name, + e.output)) + if db_pwd: + os.environ.pop('PGPASSWORD') + + return dump_path + CmdBackup.register() -- 2.11.0