From aeaafe26df11daf08a4cab2c8f2d70b99429e4fc Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Wed, 13 Jun 2018 17:34:11 +0200 Subject: [PATCH 3/3] management: passer convert-to-sql en commande de management (#20410) --- tests/test_convert_to_sql.py | 59 ++++++++++++- wcs/ctl/management/commands/__init__.py | 23 +++++ wcs/ctl/management/commands/convert_to_sql.py | 116 ++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 wcs/ctl/management/commands/convert_to_sql.py diff --git a/tests/test_convert_to_sql.py b/tests/test_convert_to_sql.py index 1e196f57..f1e8397d 100644 --- a/tests/test_convert_to_sql.py +++ b/tests/test_convert_to_sql.py @@ -1,7 +1,64 @@ +import sys import pytest from django.core.management import get_commands from django.core.management import call_command +from django.core.management.base import CommandError +from wcs.ctl.management.commands.convert_to_sql import Command +from utilities import clean_temporary_pub +from utilities import create_temporary_pub +from utilities import force_connections_close +import json + +if sys.version_info[0] > 2: + from unittest.mock import MagicMock +else: + from mock import MagicMock + + + +def pytest_generate_tests(metafunc): + if 'cmd' in metafunc.fixturenames: + metafunc.parametrize('cmd', ['pickle', 'sql'], indirect=True) + + +def teardown_module(module): + force_connections_close() + clean_temporary_pub() + + +@pytest.fixture +def cmd(request): + c = Command() + c.pub = create_temporary_pub(sql_mode=(request.param == 'sql')) + c.get_publisher = MagicMock(return_value=c.pub) + if request.param != 'sql': + new_database_pub = create_temporary_pub(sql_mode=True) + cfg = '''{"postgresql": { + "database-template-name": "wcs_%%s", + "user": "%(user)s"}}''' % new_database_pub.cfg['postgresql'] + c.get_default_cfg = MagicMock(return_value=json.loads(cfg)) + return c def test_command_exists(): - assert 'convert_to_sql' in get_commands() + assert 'convert_to_sql' in get_commands() + + +def test_unknown_publisher_fails(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'unknown.net') + assert excinfo.value.message == 'unknown tenant' + + +def test_already_migrated_fails(cmd): + if cmd.pub.is_using_postgresql(): + with pytest.raises(CommandError) as excinfo: + cmd.execute(domain='example.net', skeleton='publik') + assert excinfo.value.message == 'tenant already using postgresql' + + +def test_migration(cmd): + if not cmd.pub.is_using_postgresql(): + assert 'postgresql' not in cmd.pub.cfg + cmd.execute(domain='example.net', skeleton='publik') + assert 'postgresql' in cmd.pub.cfg diff --git a/wcs/ctl/management/commands/__init__.py b/wcs/ctl/management/commands/__init__.py index e69de29b..91035985 100644 --- a/wcs/ctl/management/commands/__init__.py +++ b/wcs/ctl/management/commands/__init__.py @@ -0,0 +1,23 @@ +import os +from django.core.management.base import BaseCommand, CommandError +from qommon.publisher import get_publisher_class + + +class WcsTenantCommand(BaseCommand): + + def add_arguments(self, parser): + parser.add_argument('-d', '--domain', metavar='DOMAIN', required=True) + super(WcsTenantCommand, self).add_arguments(parser) + + def execute(self, *args, **kwargs): + self.publisher = self.get_publisher(kwargs['domain']) + super(WcsTenantCommand, self).execute(*args, **kwargs) + + def get_publisher(self, domain): + publisher_class = get_publisher_class() + publisher = publisher_class.create_publisher() + if domain not in publisher.get_tenants(): + raise CommandError('unknown tenant') + publisher.app_dir = os.path.join(publisher.app_dir, domain) + publisher.set_config() + return publisher diff --git a/wcs/ctl/management/commands/convert_to_sql.py b/wcs/ctl/management/commands/convert_to_sql.py new file mode 100644 index 00000000..85f3352e --- /dev/null +++ b/wcs/ctl/management/commands/convert_to_sql.py @@ -0,0 +1,116 @@ +import os +import sys +from django.core.management.base import CommandError +from wcs import publisher +# from wcs.ctl.convertsql import CmdConvertToSql +from wcs import sql +from wcs.ctl.management.commands import WcsTenantCommand +from wcs.formdef import FormDef +from wcs.users import User +import traceback + + +class Command(WcsTenantCommand): + + def add_arguments(self, parser): + parser.add_argument('-s', '--skeleton', default='publik') + super(Command, self).add_arguments(parser) + + def handle(self, *args, **options): + if self.publisher.is_using_postgresql(): + raise CommandError('tenant already using postgresql') + + self.publisher.cfg['postgresql'] = self.get_postgresql_cfg( + options['domain'], options['skeleton']) + + self.publisher.cleanup() + + def get_postgresql_cfg(self, domain, skeleton): + cfg = self.get_default_cfg(skeleton) + pg_cfg = cfg['postgresql'] + pg_cfg['dbname'] = pg_cfg['database-template-name'] % domain + del pg_cfg['database-template-name'] + return pg_cfg + + def get_default_cfg(self, skeleton): + tpub = publisher.WcsPublisher.create_publisher( + register_tld_names=False) + skeleton_fp = os.path.join(self.publisher.app_dir, 'skeletons', + '{}.zip'.format(skeleton)) + if not(os.path.isfile(skeleton_fp)): + raise CommandError('skeleton not found') + tpub.import_zip(skeleton_fp) + return tpub.cfg + + def convert_to_sql(self): + sql.get_connection_and_cursor(new=True) + + errors = [] + + print('converting users') + sql.do_user_table() + count = User.count() + for i, user_id in enumerate(User.keys()): + user = User.get(user_id) + user.__class__ = sql.SqlUser + try: + user.store() + except AssertionError: + errors.append((user, traceback.format_exc())) + # update_progress(100*i/count) + sql.SqlUser.fix_sequences() + + if errors: + error_log = open('error_user.log', 'w') + for user, trace in errors: + error_log.write('user_id {}'.format(user.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_user.log for details.') + + errors = [] + for formdef in FormDef.select(): + print('converting %s' % formdef.name) + sql.do_formdef_tables(formdef, rebuild_views=True, + rebuild_global_views=True) + data_class = formdef.data_class(mode='files') + count = data_class.count() + + # load all objects a first time, to allow the migrate() code to be + # run and the eventual changes properly saved. + for id in data_class.keys(): + formdata = data_class.get(id) + delattr(sys.modules['formdef'], formdef.url_name.title()) + + # once this is done, reload and store everything in postgresql + sql_data_class = formdef.data_class(mode='sql') + for i, id in enumerate(data_class.keys()): + formdata = data_class.get(id) + formdata._formdef = formdef + formdata._evolution = formdata.evolution + formdata.__class__ = sql_data_class + try: + formdata.store() + except AssertionError: + errors.append((formdata, traceback.format_exc())) + # update_progress(100*i/count) + sql_data_class.fix_sequences() + + sql.do_tracking_code_table() + sql.do_session_table() + sql.do_meta_table() + + if errors: + error_log = open('error_formdata.log', 'w') + for formdata, trace in errors: + error_log.write('{} {}'.format(formdata.fromdef, formdata.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_formdata.log.') + + if not self.publisher.has_site_option('postgresql'): + print('You still have to edit your site-options.cfg') + + self.publisher.write_cfg() -- 2.11.0