From a87b8faacecdaa050839f6cf177850f2183952e0 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Jaillet Date: Tue, 25 Jul 2017 01:31:07 +0200 Subject: [PATCH] ctl: add dump in backup command if postgresql is true (#17726) --- tests/test_ctl.py | 44 ++++++++++++++++++ wcs/ctl/backup.py | 131 ++++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 156 insertions(+), 19 deletions(-) diff --git a/tests/test_ctl.py b/tests/test_ctl.py index ca6309b1..bec4a4bc 100644 --- a/tests/test_ctl.py +++ b/tests/test_ctl.py @@ -4,6 +4,8 @@ import collections from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart import psycopg2 +import shutil +import zipfile from wcs.formdef import FormDef from wcs.workflows import Workflow @@ -15,6 +17,7 @@ from wcs.ctl.process_bounce import CmdProcessBounce from wcs.ctl.wipe_data import CmdWipeData from wcs.ctl.trigger_jumps import select_and_jump_formdata from wcs.ctl.delete_tenant import CmdDeleteTenant +from wcs.ctl.backup import CmdBackup from wcs.sql import get_connection_and_cursor, cleanup_connection from utilities import create_temporary_pub, clean_temporary_pub @@ -324,3 +327,44 @@ def test_delete_tenant_without_sql(): assert False clean_temporary_pub() + + +def test_backup_tenant(): + pub = create_temporary_pub(sql_mode=True) + backup_cmd = CmdBackup() + if os.path.exists('/tmp/test'): + shutil.rmtree('/tmp/test') + os.mkdir('/tmp/test') + + sub_options_class = collections.namedtuple('Options', ['destination']) + sub_options = sub_options_class('/tmp/test') + + backup_cmd.backup_instance(pub, sub_options, ['example.test.com']) + + if not any(z for z in os.listdir('/tmp/test') if '.zip' in z): + assert False + + for z in os.listdir('/tmp/test'): + if '.zip' in z: + with zipfile.ZipFile(os.path.join('/tmp/test', z), 'r') as zip_file: + zip_list = zip_file.namelist() + + if not any(tar for tar in zip_list if '.tar.gz' in tar): + assert False + + if not any(dump for dump in zip_list if '.sql' in dump): + assert False + + clean_temporary_pub() + shutil.rmtree('/tmp/test') + + pub = create_temporary_pub() + os.mkdir('/tmp/test') + + backup_cmd.backup_instance(pub, sub_options, ['example.test.com']) + + if not any(tar for tar in os.listdir('/tmp/test') if '.tar.gz' in tar): + assert False + + clean_temporary_pub() + shutil.rmtree('/tmp/test') diff --git a/wcs/ctl/backup.py b/wcs/ctl/backup.py index 1654e8b1..26741c0c 100644 --- a/wcs/ctl/backup.py +++ b/wcs/ctl/backup.py @@ -15,19 +15,27 @@ # along with this program; if not, see . import tarfile -import time import os +import shutil +import zipfile +import subprocess +import tempfile +from datetime import datetime from qommon.ctl import Command, make_option +class CmdError(Exception): + pass + + class CmdBackup(Command): name = 'backup' def __init__(self): Command.__init__(self, [ - make_option('--file', metavar='FILENAME', action='store', - dest='filename', default=None) + make_option('--destination', action='store', + dest='destination', default=None) ]) def execute(self, base_options, sub_options, args): @@ -42,25 +50,110 @@ class CmdBackup(Command): if not os.path.exists(pub.app_dir): return 1 - if sub_options.filename: - backup_filepath = sub_options.filename + pub.set_config() + self.backup_instance(pub, sub_options, args) + + def backup_instance(self, pub, sub_options, args): + instance_name = args[0].replace('.', '_').replace('-', '_') + file_name = 'wcs__%s__%s' % (instance_name, + datetime.now().date().isoformat()) + dest_dir = sub_options.destination + if dest_dir: + if not os.path.exists(dest_dir): + raise CmdError('destination directory does not exist') else: - backup_dir = os.path.join(pub.app_dir, 'backups') - if not os.path.exists(backup_dir): - os.mkdir(backup_dir) - backup_filepath = os.path.join(backup_dir, - 'backup-%s%s%s-%s%s%s.tar.gz' % time.localtime()[:6]) - - backup = tarfile.open(backup_filepath, mode='w:gz') - for basename, dirnames, filenames in os.walk(pub.app_dir): - if 'backups' in dirnames: # do not recurse in backup directory - idx = dirnames.index('backups') - dirnames[idx:idx+1] = [] - for filename in filenames: - backup.add(os.path.join(basename, filename), - os.path.join(basename, filename)[len(pub.app_dir):]) + dest_dir = os.path.join(pub.app_dir, 'backups') + if not os.path.exists(dest_dir): + os.mkdir(dest_dir) + try: + temp_dir = tempfile.mkdtemp() + except IOError as e: + raise CmdError('an error occurred while creating temporary directory: %s' % e.message) + + try: + instance_tar_path = self.backup_instance_dir(pub, file_name, temp_dir) + if pub.is_using_postgresql(): + dump_path = self.backup_postgresql(pub, file_name, temp_dir) + self.create_backup_zip(dump_path, instance_tar_path, file_name, dest_dir) + else: + try: + # move only the tar file to the destination directory as there's no need + # to make a zip + shutil.move(instance_tar_path, dest_dir) + except IOError as e: + raise CmdError('could not move instance tar %s to %s: %s' % (instance_tar_path, + dest_dir, + e.message)) + finally: + try: + shutil.rmtree(temp_dir) + except IOError as e: + print 'could not remove temporary directory %s: %s' % (temp_dir, e.message) + + + def backup_instance_dir(self, pub, file_name, temp_dir): + backup_filepath = os.path.join(temp_dir, '%s.tar.gz' % file_name) + try: + backup = tarfile.open(backup_filepath, mode='w:gz') + + for basename, dirnames, filenames in os.walk(pub.app_dir): + if 'backups' in dirnames: # do not recurse in backup directory + idx = dirnames.index('backups') + dirnames[idx:idx+1] = [] + for filename in filenames: + backup.add(os.path.join(basename, filename), + os.path.join(basename, filename)[len(pub.app_dir):]) + except IOError as e: + raise CmdError('could not open tar file %s: %s' % (backup_filepath, + e.message)) + except tarfile.TarError: + raise CmdError('an error occured while making tar archive for %s' % file_name) backup.close() + return backup_filepath + + def backup_postgresql(self, pub, file_name, temp_dir): + db_conf = pub.cfg['postgresql'] + # use 'or' as value are at None when not set + db_name = db_conf.get('database') or '' + db_host = db_conf.get('host') or '' + db_user = db_conf.get('user') or '' + db_port = db_conf.get('port') or '' + db_pwd = db_conf.get('password') or '' + if db_pwd: + os.environ['PGPASSWORD'] = db_pwd + + dump_path = os.path.join(temp_dir, '%s.sql' % file_name) + + try: + subprocess.check_call(['pg_dump', '-Oc', '-h', db_host, + '-U', db_user, '-p', db_port, + '-w', '-f', dump_path, '-d', db_name]) + except subprocess.CalledProcessError as e: + raise CmdError('an error occured while dumping instance database %s: %s' % (db_name, + e.output)) + if db_pwd: + os.environ.pop('PGPASSWORD') + + return dump_path + + def create_backup_zip(self, dump_path, instance_tar_path, file_name, dest_dir): + zip_path = os.path.join(dest_dir, '%s.zip' % file_name) + try: + with zipfile.ZipFile(zip_path, 'a') as zip_file: + zip_file.write(instance_tar_path, + os.path.basename(instance_tar_path)) + zip_file.write(dump_path, + os.path.basename(dump_path)) + except RuntimeError as e: + if os.path.exists(zip_path): + try: + os.remove(zip_path) + except OSError as e: + print 'could not remove zip file %s: %s' % (zip_path, e.message) + raise CmdError('an error occured while making zip %s: %s' % (zip_path, + e.message)) + CmdBackup.register() -- 2.11.0