From 2893a9fd2e503c60f890a1ace27a5c00e2bf2d08 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Thu, 11 Apr 2019 11:06:44 +0200 Subject: [PATCH] Jenkinsfile: use workspace as temporary directory (#31192) - git repository is checked-out in the repository/ sub-directory. - tox is launched using the common step from jenkins-lib. - pg_virtualenv temporary database is used for running tests. --- Jenkinsfile | 35 ++++++++++++++++++++++--------- jenkins.sh | 9 -------- tests/settings.py | 9 +++++--- tests/wcs/conftest.py | 49 ++++++++++++++++++++++++++----------------- tox.ini | 24 +++++++++++++++------ 5 files changed, 79 insertions(+), 47 deletions(-) delete mode 100755 jenkins.sh diff --git a/Jenkinsfile b/Jenkinsfile index ee172a99..2374dd52 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,20 +2,33 @@ pipeline { agent any + options { + disableConcurrentBuilds() + skipDefaultCheckout() + } + environment { + TMPDIR = "$WORKSPACE" + } stages { stage('Unit Tests') { steps { - sh 'tox -rv' + cleanWs() + dir ("repository") { + checkout scm + tox() + } } post { always { - script { - utils = new Utils() - utils.publish_coverage('coverage.xml') - utils.publish_coverage_native('index.html') - utils.publish_pylint('pylint.out') + dir("repository") { + script { + utils = new Utils() + utils.publish_coverage('coverage.xml') + utils.publish_coverage_native('index.html') + utils.publish_pylint('pylint.out') + } + mergeJunitResults() } - junit '*_results.xml' } } } @@ -31,9 +44,11 @@ pipeline { } post { always { - script { - utils = new Utils() - utils.mail_notify(currentBuild, env, 'admin+jenkins-passerelle@entrouvert.com') + dir("repository") { + script { + utils = new Utils() + utils.mail_notify(currentBuild, env, 'admin+jenkins-passerelle@entrouvert.com') + } } } success { diff --git a/jenkins.sh b/jenkins.sh deleted file mode 100755 index 7ae0277b..00000000 --- a/jenkins.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -set -e - -rm -f coverage.xml -rm -f test_results.xml - -pip install --upgrade tox -tox -r diff --git a/tests/settings.py b/tests/settings.py index 80cd3f26..52951bd9 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -62,8 +62,11 @@ CACHES = { DATABASES = { 'default': { 'ENGINE': os.environ.get('DB_ENGINE', 'django.db.backends.sqlite3'), - 'TEST': { - 'NAME': 'passerelle-test-%s' % os.environ.get("BRANCH_NAME", "").replace('/', '-')[:63], - }, + 'NAME': 'passerelle', } } + +if 'postgres' in DATABASES['default']['ENGINE']: + for key in ('PGPORT', 'PGHOST', 'PGUSER', 'PGPASSWORD'): + if key in os.environ: + DATABASES['default'][key[2:]] = os.environ[key] diff --git a/tests/wcs/conftest.py b/tests/wcs/conftest.py index b34382e6..d3f47b23 100644 --- a/tests/wcs/conftest.py +++ b/tests/wcs/conftest.py @@ -41,36 +41,45 @@ def find_free_tcp_port(): @contextlib.contextmanager def postgres_db_factory(): - database = 'db%s' % random.getrandbits(20) - - with contextlib.closing(psycopg2.connect('')) as conn: + dsn = { + 'dbname': 'postgres', + } + for key, env in [ + ('host', 'PGHOST'), + ('port', 'PGPORT'), + ('user', 'PGUSER'), + ('password', 'PGPASSWORD')]: + if env in os.environ: + dsn[key] = os.environ[env] + + dbname = 'db%s' % random.getrandbits(20) + + with contextlib.closing(psycopg2.connect(**dsn)) as conn: conn.set_isolation_level(0) with conn.cursor() as cursor: - cursor.execute('CREATE DATABASE %s' % database) + cursor.execute('CREATE DATABASE %s' % dbname) try: - yield PostgresDB(database) + db_dsn = dsn.copy() + db_dsn['dbname'] = dbname + yield PostgresDB(db_dsn) finally: - with contextlib.closing(psycopg2.connect('')) as conn: + with contextlib.closing(psycopg2.connect(**dsn)) as conn: conn.set_isolation_level(0) with conn.cursor() as cursor: - cursor.execute('DROP DATABASE IF EXISTS %s' % database) + cursor.execute('DROP DATABASE IF EXISTS %s' % dbname) class PostgresDB(object): - def __init__(self, database): - self.database = database - - @property - def dsn(self): - return 'dbname={self.database}'.format(self=self) + def __init__(self, dsn): + self.dsn = dsn @contextlib.contextmanager def conn(self): - with contextlib.closing(psycopg2.connect(self.dsn)) as conn: + with contextlib.closing(psycopg2.connect(**self.dsn)) as conn: yield conn def __repr__(self): - return '' % self.database + return '' % self.dsn @pytest.fixture @@ -193,9 +202,11 @@ class WcsHost(object): config.set('options', 'postgresql', 'true') with self.config_pck as config: - config['postgresql'] = { - 'database': database, - } + wcs_dsn = database.dsn.copy() + # w.c.s. only support the deprecated "database" key in a PostgreSQL DSN + if 'dbname' in wcs_dsn: + wcs_dsn['database'] = wcs_dsn['dbname'] + config['postgresql'] = wcs_dsn self.run_in_context(self._wcs_init_sql) def _wcs_init_sql(self): @@ -446,5 +457,5 @@ def wcs(tmp_path_factory): @pytest.fixture def wcs_host(wcs, postgres_db, datasource): - with wcs.host('127.0.0.1', database=postgres_db.database) as wcs_host: + with wcs.host('127.0.0.1', database=postgres_db) as wcs_host: yield wcs_host diff --git a/tox.ini b/tox.ini index a6c4b132..134b44b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/passerelle/{env:BRANCH_NAME:} -envlist = django{18,111}-{sqlite,pg} +envlist = py27-django{18,111}-{sqlite,pg},pylint +toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/passerelle/ [testenv] usedevelop = True -basepython = python2 setenv = DJANGO_SETTINGS_MODULE=passerelle.settings PASSERELLE_SETTINGS_FILE=tests/settings.py @@ -13,6 +12,13 @@ setenv = fast: FAST=--nomigrations sqlite: DB_ENGINE=django.db.backends.sqlite3 pg: DB_ENGINE=django.db.backends.postgresql_psycopg2 + PGPORT={env:PGPORT:} + PGHOST={env:PGHOST:} + PGUSER={env:PGUSER:} + PGPASSWORD={env:PGPASSWORD:} + JUNIT={tty::-o junit_suite_name={envname} --junit-xml=junit-{envname}.xml} + COVERAGE=--cov=passerelle/ --cov-branch --cov-append --cov-report xml --cov-report html --cov-config .coveragerc + BASETEMP=--basetemp={envtmpdir} deps = django18: django>=1.8,<1.9 django111: django>=1.11,<1.12 @@ -37,6 +43,12 @@ deps = vobject commands = ./get_wcs.sh - django18: py.test {posargs: {env:FAST:} --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=passerelle/ --cov-config .coveragerc tests/} - django18: ./pylint.sh passerelle/ - django111: py.test {posargs: --junitxml=test_{envname}_results.xml tests/} + py.test {posargs: {env:BASETEMP:} {env:FAST:} {env:JUNIT:} {env:COVERAGE:} tests/} + +[testenv:pylint] +basepython = python2.7 +deps = + pylint<1.8 + pylint-django<0.8.1 +commands = + /bin/bash -c "./pylint.sh passerelle/" -- 2.20.1