From a546c68e6978667c2e87eaf3d8ff1ff78b3c8799 Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Thu, 21 Jun 2018 17:08:39 +0200 Subject: [PATCH 2/2] add management command convert_to_sql (#20410) --- tests/test_convert_to_sql.py | 133 +++++++++++++++++++++++++ wcs/ctl/management/commands/convert_to_sql.py | 137 ++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 tests/test_convert_to_sql.py create mode 100644 wcs/ctl/management/commands/convert_to_sql.py diff --git a/tests/test_convert_to_sql.py b/tests/test_convert_to_sql.py new file mode 100644 index 00000000..c3b46c2f --- /dev/null +++ b/tests/test_convert_to_sql.py @@ -0,0 +1,133 @@ +import os +import random +import psycopg2 +import pytest +from django.core.management import get_commands +from django.core.management import call_command +from django.core.management.base import CommandError +from utilities import clean_temporary_pub +from utilities import create_temporary_pub +from utilities import force_connections_close +from test_api import local_user +from wcs.sql import cleanup_connection +from wcs.formdef import FormDef +from wcs.fields import BoolField + + +@pytest.fixture +def formdeffix(): + FormDef.wipe() + formdef = FormDef() + formdef.name = 'testform' + formdef.description = 'plop' + formdef.fields = [BoolField(id='1')] + formdef.store() + + data_class = formdef.data_class() + for value in (True, True, True, False): + formdata = data_class() + formdata.data = {'1': value} + formdata.store() + + return formdef + + +@pytest.fixture(scope='module') +def cursor(): + conn = psycopg2.connect(user=os.environ['USER']) + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = conn.cursor() + yield cur + cur.close() + + +@pytest.fixture +def database(cursor): + dbname = 'wcstests%d' % random.randint(0, 100000) + cursor.execute('CREATE DATABASE %s' % dbname) + yield dbname + cleanup_connection() + cursor.execute('DROP DATABASE %s' % dbname) + + +@pytest.fixture(params=['pickle', 'sql']) +def pub(request): + pub = create_temporary_pub(sql_mode=(request.param == 'sql')) + yield pub + cleanup_connection() + force_connections_close() + clean_temporary_pub() + + +def test_command_exists(): + assert 'convert_to_sql' in get_commands() + + +def test_unknown_publisher_fails(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'unknown.net', + '--database', 'foobar') + assert excinfo.value.message == 'unknown tenant' + + +def test_failing_connection(pub): + if pub.is_using_postgresql(): + return + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', + '--database', 'foobar', '--port', '666') + assert 'could not connect' in excinfo.value.message + + +def test_database_does_not_exist(pub): + if pub.is_using_postgresql(): + return + new_database = 'test_{}'.format(random.randint(1000, 9999)) + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', + '--database', new_database) + assert 'does not exist' in excinfo.value.message + + +def test_already_migrated_fails(pub): + if pub.is_using_postgresql(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', + '--database', 'foobar') + assert excinfo.value.message == 'tenant already using postgresql' + + +def test_setup_database(pub, database): + if pub.is_using_postgresql(): + return + call_command('convert_to_sql', '-d', 'example.net', + '--database', database) + pub.set_config() + assert pub.cfg['postgresql'].get('database') == database + + +def test_migration(pub, database): + if not pub.is_using_postgresql(): + assert 'postgresql' not in pub.cfg + call_command('convert_to_sql', '-d', 'example.net', + '--database', database) + pub.set_config() + assert 'postgresql' in pub.cfg + + +def test_data_is_migrated(pub, database, local_user, formdeffix): + if pub.is_using_postgresql(): + return + + pub.load_site_options() + assert not pub.site_options.has_option('options', 'postgresql') + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.load_site_options() + assert pub.site_options.has_option('options', 'postgresql') + assert len( + pub.user_class.get_users_with_name_identifier('0123456789')) == 1 + + formdefs = FormDef.select() + assert len(formdefs) == 1 + data_class = formdefs[0].data_class(mode='sql') + assert len(data_class.keys()) == 4 diff --git a/wcs/ctl/management/commands/convert_to_sql.py b/wcs/ctl/management/commands/convert_to_sql.py new file mode 100644 index 00000000..acfa67ec --- /dev/null +++ b/wcs/ctl/management/commands/convert_to_sql.py @@ -0,0 +1,137 @@ +# w.c.s. - web application for online forms +# Copyright (C) 2005-2017 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import os +import sys +import traceback +from django.core.management.base import CommandError +from wcs import sql +from wcs.users import User +from wcs.formdef import FormDef +from wcs.sql import get_connection +from django.core.management.base import BaseCommand +from qommon.publisher import get_publisher_class + + +class Command(BaseCommand): + + help = 'Setup postgresql connection parameters and migrate existing objects.' + + def add_arguments(self, parser): + parser.add_argument('-d', '--domain', required=True) + parser.add_argument('--database', required=True) + parser.add_argument('--host') + parser.add_argument('--port', type=int) + parser.add_argument('--user') + parser.add_argument('--password') + + def handle(self, **options): + self.publisher = self.get_publisher(options['domain']) + if self.publisher.is_using_postgresql(): + raise CommandError('tenant already using postgresql') + + self.setup_connection(**options) + get_connection(new=True) + self.store_users() + self.store_forms() + self.publisher.write_cfg() + self.enable_connection() + self.publisher.cleanup() + + def get_publisher(self, domain): + publisher_class = get_publisher_class() + publisher = publisher_class.create_publisher() + if domain not in publisher.get_tenants(): + raise CommandError('unknown tenant') + + publisher.app_dir = os.path.join(publisher.app_dir, domain) + publisher.set_config() + return publisher + + def setup_connection(self, **kwargs): + options = {} + for k in ['host', 'port', 'database', 'user', 'password']: + if k in kwargs: + options[k] = kwargs.get(k) + self.publisher.cfg['postgresql'] = options + + def enable_connection(self): + if not self.publisher.site_options.has_option('options', 'postgresql'): + self.publisher.site_options.set('options', 'postgresql', 'true') + options_file = os.path.join(self.publisher.app_dir, 'site-options.cfg') + self.publisher.site_options.write(open(options_file, 'w')) + + def store_users(self): + errors = [] + print('converting users') + sql.do_user_table() + for i, user_id in enumerate(User.keys()): + user = User.get(user_id) + user.__class__ = sql.SqlUser + try: + user.store() + except AssertionError: + errors.append((user, traceback.format_exc())) + sql.SqlUser.fix_sequences() + + if errors: + error_log = open('error_user.log', 'w') + for user, trace in errors: + error_log.write('user_id {}'.format(user.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_user.log for details.') + + def store_forms(self): + errors = [] + for formdef in FormDef.select(): + print('converting %s' % formdef.name) + sql.do_formdef_tables(formdef, rebuild_views=True, + rebuild_global_views=True) + data_class = formdef.data_class(mode='files') + + # load all objects a first time, to allow the migrate() code to be + # run and the eventual changes properly saved. + for id in data_class.keys(): + formdata = data_class.get(id) + delattr(sys.modules['formdef'], formdef.url_name.title()) + + # once this is done, reload and store everything in postgresql + sql_data_class = formdef.data_class(mode='sql') + for i, id in enumerate(data_class.keys()): + formdata = data_class.get(id) + formdata._formdef = formdef + formdata._evolution = formdata.evolution + formdata.__class__ = sql_data_class + try: + formdata.store() + except AssertionError: + errors.append((formdata, traceback.format_exc())) + sql_data_class.fix_sequences() + + sql.do_tracking_code_table() + sql.do_session_table() + sql.do_meta_table() + + if errors: + error_log = open('error_formdata.log', 'w') + for formdata, trace in errors: + error_log.write('{} {}'.format(formdata.fromdef, formdata.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_formdata.log.') -- 2.11.0