From 0330a1b022271a6d28d700f84eb5f1c763ef6b2a Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Tue, 26 Jun 2018 17:13:15 +0200 Subject: [PATCH 2/2] move convert_to_sql to management command (#20410) --- tests/test_convert_to_sql.py | 127 ++++++++++++++++++++++ wcs/ctl/management/commands/convert_to_sql.py | 151 ++++++++++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 tests/test_convert_to_sql.py create mode 100644 wcs/ctl/management/commands/convert_to_sql.py diff --git a/tests/test_convert_to_sql.py b/tests/test_convert_to_sql.py new file mode 100644 index 00000000..4bf107a8 --- /dev/null +++ b/tests/test_convert_to_sql.py @@ -0,0 +1,127 @@ +import collections +import os +import random +import psycopg2 +import pytest + +from django.core.management import get_commands +from django.core.management import call_command +from django.core.management.base import CommandError + +from wcs.sql import cleanup_connection +from wcs.formdef import FormDef +from wcs.fields import BoolField +from wcs.ctl.convertsql import CmdConvertToSql + +from utilities import clean_temporary_pub +from utilities import create_temporary_pub +from utilities import force_connections_close +from test_api import local_user + + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + +@pytest.fixture +def formdeffix(): + FormDef.wipe() + formdef = FormDef() + formdef.name = 'testform' + formdef.description = 'plop' + formdef.fields = [BoolField(id='1')] + formdef.store() + + data_class = formdef.data_class() + for value in (True, True, True, False): + formdata = data_class() + formdata.data = {'1': value} + formdata.store() + + return formdef + + +@pytest.fixture(scope='module') +def cursor(): + conn = psycopg2.connect(user=os.environ.get('USER')) + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = conn.cursor() + yield cur + cur.close() + + +@pytest.fixture +def database(cursor): + dbname = 'wcstests%d' % random.randint(0, 100000) + cursor.execute('CREATE DATABASE %s' % dbname) + yield dbname + cleanup_connection() + cursor.execute('DROP DATABASE %s' % dbname) + + +@pytest.fixture() +def pub(request): + pub = create_temporary_pub(sql_mode=False) + yield pub + clean_temporary_pub() + + +def test_command_exists(): + assert 'convert_to_sql' in get_commands() + + +def test_unknown_publisher_fails(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'unknown.net', '--database', 'foobar') + assert excinfo.value.message == 'unknown tenant' + + +def test_failing_connection(pub): + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', 'foobar', '--port', '666') + assert 'could not connect' in excinfo.value.message + + +def test_database_does_not_exist(pub): + new_database = 'test_%s' % random.randint(1000, 9999) + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', new_database) + assert 'exist' in excinfo.value.message # works for english + french postgresql + + +def test_already_migrated_fails(): + pub = create_temporary_pub(sql_mode=True) + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', 'foobar') + assert excinfo.value.message == 'tenant already using postgresql' + cleanup_connection() + force_connections_close() + clean_temporary_pub() + + +def test_setup_database(pub, database): + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.set_config() + assert pub.cfg['postgresql'].get('database') == database + + +def test_migration(pub, database): + assert 'postgresql' not in pub.cfg + call_command('convert_to_sql', '-d', 'example.net', + '--database', database) + pub.set_config() + assert 'postgresql' in pub.cfg + + +def test_data_is_migrated(pub, database, local_user, formdeffix): + pub.load_site_options() + assert not pub.site_options.has_option('options', 'postgresql') + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.load_site_options() + assert pub.site_options.has_option('options', 'postgresql') + assert len(pub.user_class.get_users_with_name_identifier('0123456789')) == 1 + formdefs = FormDef.select() + assert len(formdefs) == 1 + data_class = formdefs[0].data_class(mode='sql') + assert len(data_class.keys()) == 4 diff --git a/wcs/ctl/management/commands/convert_to_sql.py b/wcs/ctl/management/commands/convert_to_sql.py new file mode 100644 index 00000000..2784139c --- /dev/null +++ b/wcs/ctl/management/commands/convert_to_sql.py @@ -0,0 +1,151 @@ +# w.c.s. - web application for online forms +# Copyright (C) 2005-2018 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import os +import sys +import StringIO +import traceback + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError + +from qommon.publisher import get_publisher_class + +from wcs import sql +from wcs.formdef import FormDef +from wcs.qommon.storage import atomic_write +from wcs.users import User + + +class Command(BaseCommand): + + help = 'Setup postgresql connection parameters and migrate existing objects.' + + def add_arguments(self, parser): + parser.add_argument('-d', '--domain', required=True) + parser.add_argument('--database', required=True) + parser.add_argument('--host') + parser.add_argument('--port', type=int) + parser.add_argument('--user') + parser.add_argument('--password') + + def handle(self, **options): + self.publisher = self.get_publisher(options['domain']) + if self.publisher.is_using_postgresql(): + raise CommandError('tenant already using postgresql') + + self.setup_connection(**options) + sql.get_connection(new=True) + self.store_users() + self.store_forms() + self.publisher.write_cfg() + self.enable_connection() + self.publisher.cleanup() + + def get_publisher(self, domain): + publisher_class = get_publisher_class() + publisher = publisher_class.create_publisher() + if domain not in publisher.get_tenants(): + raise CommandError('unknown tenant') + + publisher.app_dir = os.path.join(publisher.app_dir, domain) + publisher.set_config() + return publisher + + def setup_connection(self, **kwargs): + options = {} + for k in ['host', 'port', 'database', 'user', 'password']: + if k in kwargs: + options[k] = kwargs.get(k) + self.publisher.cfg['postgresql'] = options + + def enable_connection(self): + if not self.publisher.site_options.has_option('options', 'postgresql'): + self.publisher.site_options.set('options', 'postgresql', 'true') + options_file = os.path.join(self.publisher.app_dir, 'site-options.cfg') + stringio = StringIO.StringIO() + self.publisher.site_options.write(stringio) + atomic_write(options_file, stringio.getvalue()) + + def store_users(self): + errors = [] + print('converting users') + sql.do_user_table() + count = User.count() + for i, user_id in enumerate(User.keys()): + user = User.get(user_id) + user.__class__ = sql.SqlUser + try: + user.store() + except AssertionError: + errors.append((user, traceback.format_exc())) + self.update_progress(100*i/count) + sql.SqlUser.fix_sequences() + + if errors: + error_log = open('error_user.log', 'w') + for user, trace in errors: + error_log.write('user_id %s' % user.id) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_user.log for details.') + + def store_forms(self): + errors = [] + for formdef in FormDef.select(): + print('converting %s' % formdef.name) + sql.do_formdef_tables(formdef, rebuild_views=True, + rebuild_global_views=True) + data_class = formdef.data_class(mode='files') + count = data_class.count() + + # load all objects a first time, to allow the migrate() code to be + # run and the eventual changes properly saved. + for id in data_class.keys(): + formdata = data_class.get(id) + delattr(sys.modules['formdef'], formdef.url_name.title()) + + # once this is done, reload and store everything in postgresql + sql_data_class = formdef.data_class(mode='sql') + for i, id in enumerate(data_class.keys()): + formdata = data_class.get(id) + formdata._formdef = formdef + formdata._evolution = formdata.evolution + formdata.__class__ = sql_data_class + try: + formdata.store() + except AssertionError: + errors.append((formdata, traceback.format_exc())) + self.update_progress(100*i/count) + sql_data_class.fix_sequences() + + sql.do_tracking_code_table() + sql.do_session_table() + sql.do_meta_table() + + if errors: + error_log = open('error_formdata.log', 'w') + for formdata, trace in errors: + error_log.write('%s %s' % (formdata.fromdef, formdata.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_formdata.log.') + + def update_progress(self, progress, num_columns=120): + sys.stdout.write('[%s] %s%%\r' % ( + ('#'*((num_columns-10)*progress/100)).ljust(num_columns-15), progress)) -- 2.11.0