From 8aa574651b945929ea02b3db17d2d5bf3fe455b2 Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Thu, 21 Jun 2018 12:17:51 +0200 Subject: [PATCH 4/4] add management command convert_to_sql (#20410) --- tests/test_convert_to_sql.py | 54 ++++++++++++++++- wcs/ctl/management/commands/convert_to_sql.py | 85 +++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 wcs/ctl/management/commands/convert_to_sql.py diff --git a/tests/test_convert_to_sql.py b/tests/test_convert_to_sql.py index 1e196f57..46528aff 100644 --- a/tests/test_convert_to_sql.py +++ b/tests/test_convert_to_sql.py @@ -1,7 +1,59 @@ +import os import pytest from django.core.management import get_commands from django.core.management import call_command +from django.core.management.base import CommandError +from qommon.publisher import get_publisher_class +from test_api import local_user +from test_configure_connection import pub +from test_configure_connection import cursor +from test_configure_connection import database def test_command_exists(): - assert 'convert_to_sql' in get_commands() + assert 'convert_to_sql' in get_commands() + + +def test_unknown_publisher_fails(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'unknown.net', + '--database', 'foobar') + assert excinfo.value.message == 'unknown tenant' + + +def test_already_migrated_fails(pub): + if pub.is_using_postgresql(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', + '--database', 'foobar') + assert excinfo.value.message == 'tenant already using postgresql' + + +def test_migration(pub, database): + if not pub.is_using_postgresql(): + assert 'postgresql' not in pub.cfg + call_command('convert_to_sql', '-d', 'example.net', + '--database', database) + pub.set_config() + assert 'postgresql' in pub.cfg + + +def test_data_migrated(pub, database, local_user): + if pub.is_using_postgresql(): + return + + pub.load_site_options() + assert not pub.site_options.has_option('options', 'postgresql') + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.load_site_options() + assert pub.site_options.has_option('options', 'postgresql') + + # Open a new publisher, to ensure we use postgresql + # not sure if this is needed + publisher_class = get_publisher_class() + publisher = publisher_class.create_publisher() + publisher.app_dir = os.path.join(publisher.app_dir, 'example.net') + publisher.set_config() + assert publisher.site_options.has_option('options', 'postgresql') + assert len( + publisher.user_class.get_users_with_name_identifier('0123456789')) == 1 diff --git a/wcs/ctl/management/commands/convert_to_sql.py b/wcs/ctl/management/commands/convert_to_sql.py new file mode 100644 index 00000000..4b03b3a8 --- /dev/null +++ b/wcs/ctl/management/commands/convert_to_sql.py @@ -0,0 +1,85 @@ +from django.core.management.base import CommandError +from wcs.ctl.management.commands import configure_connection +import sys +from wcs import sql +from wcs.users import User +from wcs.formdef import FormDef +import traceback + + +class Command(configure_connection.Command): + + def handle(self, **options): + + if self.publisher.is_using_postgresql(): + raise CommandError('tenant already using postgresql') + + self.setup_connection(**options) + sql.get_connection_and_cursor(new=True) + self.store_users() + self.store_forms() + self.publisher.write_cfg() + self.enable() + self.publisher.cleanup() + + def store_users(self): + errors = [] + print('converting users') + sql.do_user_table() + for i, user_id in enumerate(User.keys()): + user = User.get(user_id) + user.__class__ = sql.SqlUser + try: + user.store() + except AssertionError: + errors.append((user, traceback.format_exc())) + sql.SqlUser.fix_sequences() + + if errors: + error_log = open('error_user.log', 'w') + for user, trace in errors: + error_log.write('user_id {}'.format(user.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_user.log for details.') + + def store_forms(self): + errors = [] + for formdef in FormDef.select(): + print('converting %s' % formdef.name) + sql.do_formdef_tables(formdef, rebuild_views=True, + rebuild_global_views=True) + data_class = formdef.data_class(mode='files') + + # load all objects a first time, to allow the migrate() code to be + # run and the eventual changes properly saved. + for id in data_class.keys(): + formdata = data_class.get(id) + delattr(sys.modules['formdef'], formdef.url_name.title()) + + # once this is done, reload and store everything in postgresql + sql_data_class = formdef.data_class(mode='sql') + for i, id in enumerate(data_class.keys()): + formdata = data_class.get(id) + formdata._formdef = formdef + formdata._evolution = formdata.evolution + formdata.__class__ = sql_data_class + try: + formdata.store() + except AssertionError: + errors.append((formdata, traceback.format_exc())) + sql_data_class.fix_sequences() + + sql.do_tracking_code_table() + sql.do_session_table() + sql.do_meta_table() + + if errors: + error_log = open('error_formdata.log', 'w') + for formdata, trace in errors: + error_log.write('{} {}'.format(formdata.fromdef, formdata.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_formdata.log.') -- 2.11.0