From 460e33913b85e79da0eca5248e77f97287444146 Mon Sep 17 00:00:00 2001 From: Christophe Siraut Date: Thu, 21 Jun 2018 17:08:39 +0200 Subject: [PATCH 2/2] move convert_to_sql to management command (#20410) --- tests/test_convert_to_sql.py | 145 +++++++++++++++++++++++++ wcs/ctl/convertsql.py | 118 ++------------------ wcs/ctl/management/commands/convert_to_sql.py | 151 ++++++++++++++++++++++++++ 3 files changed, 305 insertions(+), 109 deletions(-) create mode 100644 tests/test_convert_to_sql.py create mode 100644 wcs/ctl/management/commands/convert_to_sql.py diff --git a/tests/test_convert_to_sql.py b/tests/test_convert_to_sql.py new file mode 100644 index 00000000..2043db5d --- /dev/null +++ b/tests/test_convert_to_sql.py @@ -0,0 +1,145 @@ +import collections +import os +import random +import psycopg2 +import pytest + +from django.core.management import get_commands +from django.core.management import call_command +from django.core.management.base import CommandError + +from wcs.sql import cleanup_connection +from wcs.formdef import FormDef +from wcs.fields import BoolField +from wcs.ctl.convertsql import CmdConvertToSql + +from utilities import clean_temporary_pub +from utilities import create_temporary_pub +from utilities import force_connections_close +from test_api import local_user + + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + +@pytest.fixture +def formdeffix(): + FormDef.wipe() + formdef = FormDef() + formdef.name = 'testform' + formdef.description = 'plop' + formdef.fields = [BoolField(id='1')] + formdef.store() + + data_class = formdef.data_class() + for value in (True, True, True, False): + formdata = data_class() + formdata.data = {'1': value} + formdata.store() + + return formdef + + +@pytest.fixture(scope='module') +def cursor(): + conn = psycopg2.connect(user=os.environ.get('USER')) + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + cur = conn.cursor() + yield cur + cur.close() + + +@pytest.fixture +def database(cursor): + dbname = 'wcstests%d' % random.randint(0, 100000) + cursor.execute('CREATE DATABASE %s' % dbname) + yield dbname + cleanup_connection() + cursor.execute('DROP DATABASE %s' % dbname) + + +@pytest.fixture() +def pub(request): + pub = create_temporary_pub(sql_mode=False) + yield pub + clean_temporary_pub() + + +def test_command_exists(): + assert 'convert_to_sql' in get_commands() + + +def test_unknown_publisher_fails(): + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'unknown.net', '--database', 'foobar') + assert excinfo.value.message == 'unknown tenant' + + +def test_failing_connection(pub): + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', 'foobar', '--port', '666') + assert 'could not connect' in excinfo.value.message + + +def test_database_does_not_exist(pub): + new_database = 'test_{}'.format(random.randint(1000, 9999)) + with pytest.raises(psycopg2.OperationalError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', new_database) + assert 'does not exist' in excinfo.value.message + + +def test_already_migrated_fails(): + pub = create_temporary_pub(sql_mode=True) + with pytest.raises(CommandError) as excinfo: + call_command('convert_to_sql', '-d', 'example.net', '--database', 'foobar') + assert excinfo.value.message == 'tenant already using postgresql' + cleanup_connection() + force_connections_close() + clean_temporary_pub() + + +def test_setup_database(pub, database): + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.set_config() + assert pub.cfg['postgresql'].get('database') == database + + +def test_migration(pub, database): + assert 'postgresql' not in pub.cfg + call_command('convert_to_sql', '-d', 'example.net', + '--database', database) + pub.set_config() + assert 'postgresql' in pub.cfg + + +def test_data_is_migrated(pub, database, local_user, formdeffix): + pub.load_site_options() + assert not pub.site_options.has_option('options', 'postgresql') + call_command('convert_to_sql', '-d', 'example.net', '--database', database) + pub.load_site_options() + assert pub.site_options.has_option('options', 'postgresql') + assert len(pub.user_class.get_users_with_name_identifier('0123456789')) == 1 + formdefs = FormDef.select() + assert len(formdefs) == 1 + data_class = formdefs[0].data_class(mode='sql') + assert len(data_class.keys()) == 4 + + +def test_compat_previous_ctl_command(pub, database, local_user, formdeffix): + pub.load_site_options() + assert not pub.site_options.has_option('options', 'postgresql') + cmd = CmdConvertToSql() + cmd.config.add_section('main') + cmd.config.set('main', 'error_log', '') + cmd.config.set('main', 'app_dir', os.path.dirname(pub.app_dir)) + suboptions = Struct(**{'data_dir': pub.app_dir, 'dbname': database, 'user': os.environ.get('USER')}) + cmd.execute(object, suboptions, ('example.net',)) + pub.load_site_options() + assert pub.site_options.has_option('options', 'postgresql') + assert len(pub.user_class.get_users_with_name_identifier('0123456789')) == 1 + formdefs = FormDef.select() + assert len(formdefs) == 1 + data_class = formdefs[0].data_class(mode='sql') + assert len(data_class.keys()) == 4 diff --git a/wcs/ctl/convertsql.py b/wcs/ctl/convertsql.py index 3a18b378..154d9c62 100644 --- a/wcs/ctl/convertsql.py +++ b/wcs/ctl/convertsql.py @@ -15,30 +15,9 @@ # along with this program; if not, see . import os -import sys -import traceback - - -num_columns = 0 -try: - import curses -except ImportError: - curses = None -else: - try: - curses.setupterm() - num_columns = curses.tigetnum('cols') - except: - pass - - from qommon.ctl import Command, make_option +from django.core.management import call_command -def update_progress(progress): - if not num_columns: - return - sys.stdout.write('[%s] %s%%\r' % ( - ('#'*((num_columns-10)*progress/100)).ljust(num_columns-15), progress)) class CmdConvertToSql(Command): name = 'convert-to-sql' @@ -69,94 +48,15 @@ class CmdConvertToSql(Command): pub.app_dir = os.path.join(pub.app_dir, hostname) pub.set_config() - from wcs.formdef import FormDef - from wcs import sql - - if sub_options.port: - sub_options.port = int(sub_options.port) - - pub.cfg['postgresql'] = { - 'database': sub_options.dbname, - 'user': sub_options.user, - 'password': sub_options.password, - 'host': sub_options.host, - 'port': sub_options.port, - } - - sql.get_connection_and_cursor(new=True) - - errors = [] - - print 'converting users' - from users import User - sql.do_user_table() - count = User.count() - for i, user_id in enumerate(User.keys()): - user = User.get(user_id) - user.__class__ = sql.SqlUser - try: - user.store() - except AssertionError: - errors.append((user, traceback.format_exc())) - update_progress(100*i/count) - sql.SqlUser.fix_sequences() - - if errors: - error_log = file('error_user.log', 'w') - for user, trace in errors: - print >> error_log, 'user_id', user.id - print >> error_log, trace - print >> error_log, '-'*80 - error_log.close() - print 'There were some errors, see error_user.log for details.' - - errors = [] - for formdef in FormDef.select(): - print ('converting %s' % formdef.name).ljust(num_columns-1) - sql.do_formdef_tables(formdef, rebuild_views=True, - rebuild_global_views=True) - data_class = formdef.data_class(mode='files') - count = data_class.count() - - # load all objects a first time, to allow the migrate() code to be - # run and the eventual changes properly saved. - for id in data_class.keys(): - formdata = data_class.get(id) - delattr(sys.modules['formdef'], formdef.url_name.title()) - - # once this is done, reload and store everything in postgresql - sql_data_class = formdef.data_class(mode='sql') - for i, id in enumerate(data_class.keys()): - formdata = data_class.get(id) - formdata._formdef = formdef - formdata._evolution = formdata.evolution - formdata.__class__ = sql_data_class - try: - formdata.store() - except AssertionError: - errors.append((formdata, traceback.format_exc())) - update_progress(100*i/count) - sql_data_class.fix_sequences() - - print 'done'.ljust(num_columns-1) - - sql.do_tracking_code_table() - sql.do_session_table() - sql.do_meta_table() - - if errors: - error_log = file('error_formdata.log', 'w') - for formdata, trace in errors: - print >> error_log, formdata.formdef, formdata.id - print >> error_log, trace - print >> error_log, '-'*80 - error_log.close() - print 'There were some errors, see error_formdata.log for details.' - - if not pub.has_site_option('postgresql'): - print 'You still have to edit your site-options.cfg' + cmd = ['convert_to_sql', + '-d', hostname, + '--database', sub_options.dbname] - pub.write_cfg() + for k in ['user', 'password', 'host', 'port']: + value = getattr(sub_options, k, False) + if value: + cmd.extend(['--%s' % k, value]) + call_command(*cmd) CmdConvertToSql.register() diff --git a/wcs/ctl/management/commands/convert_to_sql.py b/wcs/ctl/management/commands/convert_to_sql.py new file mode 100644 index 00000000..975d3354 --- /dev/null +++ b/wcs/ctl/management/commands/convert_to_sql.py @@ -0,0 +1,151 @@ +# w.c.s. - web application for online forms +# Copyright (C) 2005-2018 Entr'ouvert +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +import os +import sys +import StringIO +import traceback + +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError + +from qommon.publisher import get_publisher_class + +from wcs import sql +from wcs.formdef import FormDef +from wcs.qommon.storage import atomic_write +from wcs.users import User + + +class Command(BaseCommand): + + help = 'Setup postgresql connection parameters and migrate existing objects.' + + def add_arguments(self, parser): + parser.add_argument('-d', '--domain', required=True) + parser.add_argument('--database', required=True) + parser.add_argument('--host') + parser.add_argument('--port', type=int) + parser.add_argument('--user') + parser.add_argument('--password') + + def handle(self, **options): + self.publisher = self.get_publisher(options['domain']) + if self.publisher.is_using_postgresql(): + raise CommandError('tenant already using postgresql') + + self.setup_connection(**options) + sql.get_connection(new=True) + self.store_users() + self.store_forms() + self.publisher.write_cfg() + self.enable_connection() + self.publisher.cleanup() + + def get_publisher(self, domain): + publisher_class = get_publisher_class() + publisher = publisher_class.create_publisher() + if domain not in publisher.get_tenants(): + raise CommandError('unknown tenant') + + publisher.app_dir = os.path.join(publisher.app_dir, domain) + publisher.set_config() + return publisher + + def setup_connection(self, **kwargs): + options = {} + for k in ['host', 'port', 'database', 'user', 'password']: + if k in kwargs: + options[k] = kwargs.get(k) + self.publisher.cfg['postgresql'] = options + + def enable_connection(self): + if not self.publisher.site_options.has_option('options', 'postgresql'): + self.publisher.site_options.set('options', 'postgresql', 'true') + options_file = os.path.join(self.publisher.app_dir, 'site-options.cfg') + stringio = StringIO.StringIO() + self.publisher.site_options.write(stringio) + atomic_write(options_file, stringio.getvalue()) + + def store_users(self): + errors = [] + print('converting users') + sql.do_user_table() + count = User.count() + for i, user_id in enumerate(User.keys()): + user = User.get(user_id) + user.__class__ = sql.SqlUser + try: + user.store() + except AssertionError: + errors.append((user, traceback.format_exc())) + self.update_progress(100*i/count) + sql.SqlUser.fix_sequences() + + if errors: + error_log = open('error_user.log', 'w') + for user, trace in errors: + error_log.write('user_id {}'.format(user.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_user.log for details.') + + def store_forms(self): + errors = [] + for formdef in FormDef.select(): + print('converting %s' % formdef.name) + sql.do_formdef_tables(formdef, rebuild_views=True, + rebuild_global_views=True) + data_class = formdef.data_class(mode='files') + count = data_class.count() + + # load all objects a first time, to allow the migrate() code to be + # run and the eventual changes properly saved. + for id in data_class.keys(): + formdata = data_class.get(id) + delattr(sys.modules['formdef'], formdef.url_name.title()) + + # once this is done, reload and store everything in postgresql + sql_data_class = formdef.data_class(mode='sql') + for i, id in enumerate(data_class.keys()): + formdata = data_class.get(id) + formdata._formdef = formdef + formdata._evolution = formdata.evolution + formdata.__class__ = sql_data_class + try: + formdata.store() + except AssertionError: + errors.append((formdata, traceback.format_exc())) + self.update_progress(100*i/count) + sql_data_class.fix_sequences() + + sql.do_tracking_code_table() + sql.do_session_table() + sql.do_meta_table() + + if errors: + error_log = open('error_formdata.log', 'w') + for formdata, trace in errors: + error_log.write('{} {}'.format(formdata.fromdef, formdata.id)) + error_log.write(trace) + error_log.write('-'*80) + error_log.close() + print('There were some errors, see error_formdata.log.') + + def update_progress(self, progress, num_columns=120): + sys.stdout.write('[%s] %s%%\r' % ( + ('#'*((num_columns-10)*progress/100)).ljust(num_columns-15), progress)) -- 2.11.0