0002-Jenkinsfile-use-temporary-directory-and-database-311.patch

Benjamin Dauvergne, 31 mai 2019 13:20

Voir les différences: en ligne côte à côte

Subject: [PATCH 2/2] Jenkinsfile: use temporary directory and database
 (#31192)

pg_virtualenv is used to allocate a temporary postgressql database with
fsync=off option.

 Jenkinsfile            | 13 ++++++++-
 merge-junit-results.py | 65 ++++++++++++++++++++++++++++++++++++++++++
 tests/settings.py      |  9 ++++--
 tests/wcs/conftest.py  | 45 +++++++++++++++++------------
 tox.ini                | 20 ++++++++++---
 5 files changed, 125 insertions(+), 27 deletions(-)
 create mode 100755 merge-junit-results.py

     pipeline {
         agent any
         options { disableConcurrentBuilds() }
         environment {
            TMPDIR = "/tmp/$BUILD_TAG"
+        }
         stages {
             stage('Unit Tests') {
                 steps {
                     sh 'tox -rv'
                     sh "mkdir ${env.TMPDIR}"
                     sh """
     virtualenv -p python3 ${env.TMPDIR}/venv/
     ${env.TMPDIR}/venv/bin/pip install tox
     PGPORT=`python -c 'import struct; import socket; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` pg_virtualenv -o fsync=off ${env.TMPDIR}/venv/bin/tox -rv"""
+                }
                 post {
                     always {
-...
                             utils.publish_coverage('coverage.xml')
                             utils.publish_coverage_native('index.html')
                             utils.publish_pylint('pylint.out')
                             sh './merge-junit-results.py junit-*.xml >junit.xml'
                             sh 'rm junit-*.xml'
+                        }
                         junit '*_results.xml'
+                    }
-...
                     utils = new Utils()
                     utils.mail_notify(currentBuild, env, 'admin+jenkins-passerelle@entrouvert.com')
+                }
                 sh "rm -rf ${env.TMPDIR}"
+            }
             success {
                 cleanWs()

     #!/usr/bin/env python
+    #
     #  Corey Goldberg, Dec 2012
+    #
     import os
     import sys
     import xml.etree.ElementTree as ET
     """Merge multiple JUnit XML files into a single results file.
     Output dumps to sdtdout.
     example usage:
         $ python merge_junit_results.py results1.xml results2.xml > results.xml
     """
     def main():
         args = sys.argv[1:]
         if not args:
             usage()
             sys.exit(2)
         if '-h' in args or '--help' in args:
             usage()
             sys.exit(2)
         merge_results(args[:])
     def merge_results(xml_files):
         failures = 0
         tests = 0
         errors = 0
         time = 0.0
         cases = []
         for file_name in xml_files:
             tree = ET.parse(file_name)
             test_suite = tree.getroot()
             failures += int(test_suite.attrib['failures'])
             tests += int(test_suite.attrib['tests'])
             errors += int(test_suite.attrib['errors'])
             time += float(test_suite.attrib['time'])
             name = test_suite.attrib.get('name', '')
             for child in test_suite.getchildren():
                 child.attrib['classname'] = '%s-%s' % (name, child.attrib.get('classname', ''))
             cases.append(test_suite.getchildren())
         new_root = ET.Element('testsuite')
         new_root.attrib['failures'] = '%s' % failures
         new_root.attrib['tests'] = '%s' % tests
         new_root.attrib['errors'] = '%s' % errors
         new_root.attrib['time'] = '%s' % time
         for case in cases:
             new_root.extend(case)
         new_tree = ET.ElementTree(new_root)
         ET.dump(new_tree)
     def usage():
         this_file = os.path.basename(__file__)
         print('Usage:  %s results1.xml results2.xml' % this_file)
     if __name__ == '__main__':
         main()

     DATABASES = {
         'default': {
             'ENGINE': os.environ.get('DB_ENGINE', 'django.db.backends.sqlite3'),
             'TEST': {
                 'NAME': 'passerelle-test-%s' % os.environ.get("BRANCH_NAME", "").replace('/', '-')[:63],
             },
             'NAME': 'passerelle',
+        }
+    }
     if 'postgres' in DATABASES['default']['ENGINE']:
         for key in ('PGPORT', 'PGHOST', 'PGUSER', 'PGPASSWORD'):
             if key in os.environ:
                 DATABASES['default'][key[2:]] = os.environ[key]

     @contextlib.contextmanager
     def postgres_db_factory():
         database = 'db%s' % random.getrandbits(20)
         with contextlib.closing(psycopg2.connect('')) as conn:
         dsn = {
             'database': 'postgres',
+        }
         for key, env in [
                 ('host', 'PGHOST'),
                 ('port', 'PGPORT'),
                 ('user', 'PGUSER'),
                 ('password', 'PGPASSWORD')]:
             if env in os.environ:
                 dsn[key] = os.environ[env]
         dbname = 'db%s' % random.getrandbits(20)
         with contextlib.closing(psycopg2.connect(**dsn)) as conn:
             conn.set_isolation_level(0)
             with conn.cursor() as cursor:
                 cursor.execute('CREATE DATABASE %s' % database)
                 cursor.execute('CREATE DATABASE %s' % dbname)
         try:
             yield PostgresDB(database)
             db_dsn = dsn.copy()
             db_dsn['database'] = dbname
             yield PostgresDB(db_dsn)
         finally:
             with contextlib.closing(psycopg2.connect('')) as conn:
             with contextlib.closing(psycopg2.connect(**dsn)) as conn:
                 conn.set_isolation_level(0)
                 with conn.cursor() as cursor:
                     cursor.execute('DROP DATABASE IF EXISTS %s' % database)
                     cursor.execute('DROP DATABASE IF EXISTS %s' % dbname)
     class PostgresDB(object):
         def __init__(self, database):
             self.database = database
         @property
         def dsn(self):
             return 'dbname={self.database}'.format(self=self)
         def __init__(self, dsn):
             self.dsn = dsn
         @contextlib.contextmanager
         def conn(self):
             with contextlib.closing(psycopg2.connect(self.dsn)) as conn:
             with contextlib.closing(psycopg2.connect(**self.dsn)) as conn:
                 yield conn
         def __repr__(self):
             return '<Postgres Database %r>' % self.database
             return '<Postgres Database %r>' % self.dsn
     @pytest.fixture
-...
                 config.set('options', 'postgresql', 'true')
             with self.config_pck as config:
                 config['postgresql'] = {
                     'database': database,
+                }
                 config['postgresql'] = database.dsn
             self.run_in_context(self._wcs_init_sql)
         def _wcs_init_sql(self):
-...
     @pytest.fixture
     def wcs_host(wcs, postgres_db, datasource):
             with wcs.host('127.0.0.1', database=postgres_db.database) as wcs_host:
             with wcs.host('127.0.0.1', database=postgres_db) as wcs_host:
                 yield wcs_host

     [tox]
     toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/passerelle/{env:BRANCH_NAME:}
     toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/passerelle/
     envlist = py27-django{18,111}-{sqlite,pg}
     [testenv]
-...
       fast: FAST=--nomigrations
       sqlite: DB_ENGINE=django.db.backends.sqlite3
       pg: DB_ENGINE=django.db.backends.postgresql_psycopg2
       PGPORT={env:PGPORT:}
       PGHOST={env:PGHOST:}
       PGUSER={env:PGUSER:}
       PGPASSWORD={env:PGPASSWORD:}
       JUNIT={tty::-o junit_suite_name={envname} --junit-xml=junit-{envname}.xml}
       coverage: COVERAGE=--cov=passerelle/ --cov-branch --cov-append --cov-report xml --cov-report html --cov-config .coveragerc
     deps =
       django18: django>=1.8,<1.9
       django111: django>=1.11,<1.12
-...
       vobject
     commands =
       ./get_wcs.sh
       django18: py.test {posargs: {env:FAST:} --junitxml=test_{envname}_results.xml --cov-report xml --cov-report html --cov=passerelle/ --cov-config .coveragerc tests/}
       django18: ./pylint.sh passerelle/
       django111: py.test {posargs: --junitxml=test_{envname}_results.xml tests/}
       py.test {posargs: {env:FAST:} {env:JUNIT:} {env:COVERAGE:} tests/}
     [testenv:pylint]
     basepython = python2.7
     deps =
         pylint<1.8
         pylint-django<0.8.1
     commands =
         /bin/bash -c "./pylint.sh passerelle/"
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0002-Jenkinsfile-use-temporary-directory-and-database-311.patch