From d2b5a32997b6257d4ff35fc6766bd62b93050a5a Mon Sep 17 00:00:00 2001 From: Agate Berriot Date: Wed, 3 Aug 2022 10:56:29 +0200 Subject: [PATCH 1/2] Parallelize tests in CI (#67911) --- Jenkinsfile | 2 +- hobo/test_utils.py | 19 +++++++++++++++++++ tests/settings.py | 7 +++---- tests_authentic/settings.py | 7 +++---- tests_multipublik/settings.py | 6 ++---- tests_multitenant/settings.py | 7 +++---- tests_passerelle/settings.py | 8 ++++---- tests_schemas/settings.py | 6 ++++-- tox.ini | 3 ++- 9 files changed, 41 insertions(+), 24 deletions(-) create mode 100644 hobo/test_utils.py diff --git a/Jenkinsfile b/Jenkinsfile index fc5ab17..9eeb782 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,7 +6,7 @@ pipeline { stages { stage('Unit Tests') { steps { - sh 'tox -rv' + sh 'tox -rv -p 8' } post { always { diff --git a/hobo/test_utils.py b/hobo/test_utils.py new file mode 100644 index 0000000..91adf24 --- /dev/null +++ b/hobo/test_utils.py @@ -0,0 +1,19 @@ +import hashlib +import os + + +def get_safe_db_name(prefix): + """ + PostgreSQL database name limit is 68 characters, which can become + an issue during testing, because we need to build a unique + database name using the branch and tox env. + + Ergo, the following code to ensure the database name is always shorter than 68 + characters. + """ + BRANCH_NAME = os.environ.get('BRANCH_NAME', '').replace('/', '-') + TOX_ENV_NAME = os.environ.get('TOX_ENV_NAME') + DB_NAME = '_'.join([part for part in [prefix, BRANCH_NAME, TOX_ENV_NAME] if part]) + DB_NAME = hashlib.sha256(DB_NAME.encode()).hexdigest()[:10] + + return DB_NAME diff --git a/tests/settings.py b/tests/settings.py index 6022280..70e9153 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -1,5 +1,7 @@ import os +import hobo.test_utils + LANGUAGE_CODE = 'en-us' BROKER_URL = 'memory://' @@ -16,13 +18,10 @@ MIDDLEWARE = MIDDLEWARE + ( ) HOBO_MANAGER_HOMEPAGE_URL_VAR = 'portal_agent_url' - DATABASES = { 'default': { 'ENGINE': os.environ.get('DB_ENGINE', 'django.db.backends.sqlite3'), - 'TEST': { - 'NAME': ('hobo-test-%s' % os.environ.get("BRANCH_NAME", "").replace('/', '-'))[:63], - }, + 'NAME': hobo.test_utils.get_safe_db_name('test-hobo'), } } diff --git a/tests_authentic/settings.py b/tests_authentic/settings.py index 9044bfd..0db531a 100644 --- a/tests_authentic/settings.py +++ b/tests_authentic/settings.py @@ -3,6 +3,8 @@ import os from mock import mock_open, patch +import hobo.test_utils + # Debian defaults DEBUG = False @@ -13,10 +15,7 @@ open_backup = open with patch.object(builtins, 'open', mock_open(read_data=b'xxx')): exec(open_backup(os.environ['DEBIAN_CONFIG_COMMON']).read()) -BRANCH_NAME = os.environ.get("BRANCH_NAME", "").replace('/', '-') -DATABASES['default']['TEST'] = { - 'NAME': (DATABASES['default']['NAME'] + '-%s' % BRANCH_NAME)[:63], -} +DATABASES['default']['NAME'] = hobo.test_utils.get_safe_db_name('test-authentic') # Avoid conflic with real tenants # that might exist in /var/lib/authentic2_multitenant/tenants diff --git a/tests_multipublik/settings.py b/tests_multipublik/settings.py index e0482bf..a3e24c1 100644 --- a/tests_multipublik/settings.py +++ b/tests_multipublik/settings.py @@ -3,6 +3,7 @@ import os.path from mock import mock_open, patch +import hobo.test_utils from hobo.settings import * LANGUAGE_CODE = 'en-us' @@ -13,10 +14,7 @@ open_backup = open with patch.object(builtins, 'open', mock_open(read_data=b'xxx')): exec(open_backup(os.path.join(os.path.dirname(__file__), '../debian/debian_config_common.py')).read()) -BRANCH_NAME = os.environ.get("BRANCH_NAME", "").replace('/', '-') -DATABASES['default']['TEST'] = { - 'NAME': (DATABASES['default']['NAME'] + '-%s' % BRANCH_NAME)[:63], -} +DATABASES['default']['NAME'] = hobo.test_utils.get_safe_db_name('test-multipublik') CACHES = { 'default': { diff --git a/tests_multitenant/settings.py b/tests_multitenant/settings.py index c0e24de..ac102f0 100644 --- a/tests_multitenant/settings.py +++ b/tests_multitenant/settings.py @@ -3,6 +3,8 @@ import os from mock import mock_open, patch +import hobo.test_utils + LANGUAGE_CODE = 'en-us' INSTALLED_APPS = ( @@ -50,10 +52,7 @@ open_backup = open with patch.object(builtins, 'open', mock_open(read_data=b'xxx')): exec(open_backup(os.path.join(os.path.dirname(__file__), '../debian/debian_config_common.py')).read()) -BRANCH_NAME = os.environ.get("BRANCH_NAME", "").replace('/', '-') -DATABASES['default']['TEST'] = { - 'NAME': (DATABASES['default']['NAME'] + '-%s' % BRANCH_NAME)[:63], -} +DATABASES['default']['NAME'] = hobo.test_utils.get_safe_db_name('test-multitenant') TENANT_APPS = ( 'django.contrib.auth', diff --git a/tests_passerelle/settings.py b/tests_passerelle/settings.py index 73e90de..9c3db5a 100644 --- a/tests_passerelle/settings.py +++ b/tests_passerelle/settings.py @@ -3,6 +3,8 @@ import os from mock import mock_open, patch +import hobo.test_utils + # Debian defaults DEBUG = False @@ -15,10 +17,8 @@ open_backup = open with patch.object(builtins, 'open', mock_open(read_data=b'xxx')): exec(open_backup(os.environ['DEBIAN_CONFIG_COMMON']).read()) -BRANCH_NAME = os.environ.get("BRANCH_NAME", "").replace('/', '-') -DATABASES['default']['TEST'] = { - 'NAME': (DATABASES['default']['NAME'] + '-%s' % BRANCH_NAME)[:63], -} +BRANCH_NAME = os.environ.get("BRANCH_NAME", "").replace('/', '-')[:15] +DATABASES['default']['NAME'] = hobo.test_utils.get_safe_db_name('test-passerelle') # Avoid conflic with real tenants # that might exist in /var/lib/passerelle/tenants diff --git a/tests_schemas/settings.py b/tests_schemas/settings.py index 437b5a1..74c9586 100644 --- a/tests_schemas/settings.py +++ b/tests_schemas/settings.py @@ -1,16 +1,18 @@ import os import tempfile +import hobo.test_utils + TENANT_BASE = tempfile.mkdtemp('hobo-tenant-base') TENANT_MODEL = 'multitenant.Tenant' MIDDLEWARE = ('hobo.multitenant.middleware.TenantMiddleware',) + MIDDLEWARE BRANCH_NAME = ( os.environ.get("BRANCH_NAME", "").replace('/', '_').replace('-', '_').encode('ascii', 'ignore').decode() -) +)[:15] DATABASES = { 'default': { 'ENGINE': 'tenant_schemas.postgresql_backend', - 'TEST': {'NAME': ('hobo_test_schemas_' + BRANCH_NAME)[:63]}, + 'NAME': hobo.test_utils.get_safe_db_name('test-schemas'), } } DATABASE_ROUTERS = ('tenant_schemas.routers.TenantSyncRouter',) diff --git a/tox.ini b/tox.ini index 1db8b7e..5bfe079 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ usedevelop = True setenv = DISABLE_GLOBAL_HANDLERS=1 BRANCH_NAME={env:BRANCH_NAME:} + COVERAGE_FILE={envdir}/coverage DB_ENGINE=django.db.backends.postgresql_psycopg2 SETUPTOOLS_USE_DISTUTILS=stdlib JUNIT=--junitxml=junit-{envname}.xml @@ -31,7 +32,7 @@ setenv = passerelle: DEBIAN_CONFIG_COMMON=debian/debian_config_common.py passerelle: PASSERELLE_SETTINGS_FILE=tests_passerelle/settings.py passerelle: DJANGO_SETTINGS_MODULE=passerelle.settings - coverage: COVERAGE=--cov-report xml:coverage-{envname}.xml --cov-report html:htmlcov-{envname} --cov=hobo/ --cov-config .coveragerc + coverage: COVERAGE=--cov-report xml:coverage-{envname}.xml --cov-report html:htmlcov-{envname} --cov=hobo/ --cov-config .coveragerc fast: NOMIGRATIONS=--nomigrations deps: drf39: djangorestframework>=3.9.2,<3.10 -- 2.36.1