From c3af717876795ca4265d582d7c27db7b853fdf5f Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 18 Jan 2019 14:11:59 +0100 Subject: [PATCH] wcs_api: return w.c.s. API errors (fixes #17924) --- setup.py | 2 +- tests/conftest.py | 33 ++++++++++++++++++++++++++ tests/test_wcs.py | 55 +++++++++++++++++++++++++++----------------- tox.ini | 1 + wcs_olap/feeder.py | 41 +++++++++++++++++++++------------ wcs_olap/wcs_api.py | 56 ++++++++++++++++++++++++++++++++++++++++----- 6 files changed, 145 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index efb0750..e7dd8d9 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup(name="wcs-olap", maintainer_email="bdauvergne@entrouvert.com", packages=find_packages(), include_package_data=True, - install_requires=['requests', 'psycopg2', 'isodate'], + install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'], entry_points={ 'console_scripts': ['wcs-olap=wcs_olap.cmd:main'], }, diff --git a/tests/conftest.py b/tests/conftest.py index d5514aa..1bfdd00 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -200,3 +200,36 @@ ALLOWED_HOSTS = ['%s'] yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID) os.kill(WCS_PID, 9) shutil.rmtree(str(WCS_DIR)) + + +@pytest.fixture +def olap_cmd(wcs, tmpdir, postgres_db): + config_ini = tmpdir / 'config.ini' + model_dir = tmpdir / 'model_dir' + model_dir.mkdir() + with config_ini.open('w') as fd: + fd.write(u''' +[wcs-olap] +cubes_model_dirs = {model_dir} +pg_dsn = {dsn} + +[{wcs.url}] +orig = olap +key = olap +schema = olap +'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn)) + + from wcs_olap import cmd + import sys + + def f(no_log_errors=True): + old_argv = sys.argv + try: + sys.argv = ['', str(config_ini)] + if no_log_errors: + sys.argv.insert(1, '--no-log-errors') + cmd.main2() + finally: + sys.argv = old_argv + f.model_dir = model_dir + return f diff --git a/tests/test_wcs.py b/tests/test_wcs.py index 31f615a..a64485f 100644 --- a/tests/test_wcs.py +++ b/tests/test_wcs.py @@ -1,28 +1,14 @@ import json -import pathlib2 - -def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog): - config_ini = tmpdir / 'config.ini' - model_dir = tmpdir / 'model_dir' - model_dir.mkdir() - with config_ini.open('w') as fd: - fd.write(u''' -[wcs-olap] -cubes_model_dirs = {model_dir} -pg_dsn = {dsn} +import pytest -[{wcs.url}] -orig = olap -key = olap -schema = olap -'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn)) +import requests +import pathlib2 +import mock - from wcs_olap import cmd - import sys - sys.argv = ['', '--no-log-errors', str(config_ini)] - cmd.main2() +def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog): + olap_cmd() expected_schema = [ ('agent', 'id'), @@ -94,8 +80,35 @@ schema = olap assert list(c.fetchall()) == expected_schema # verify JSON schema - with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2: + with (olap_cmd.model_dir / 'olap.model').open() as fd, \ + (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2: json_schema = json.load(fd) expected_json_schema = json.load(fd2) expected_json_schema['pg_dsn'] = postgres_db.dsn assert json_schema == expected_json_schema + + +def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog): + with mock.patch('requests.get', side_effect=requests.RequestException('wat!')): + with pytest.raises(SystemExit): + olap_cmd(no_log_errors=False) + assert 'wat!' in caplog.text + + +def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog): + with mock.patch('requests.get') as mocked_get: + mocked_get.return_value.ok = False + mocked_get.return_value.status_code = 401 + mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}' + with pytest.raises(SystemExit): + olap_cmd(no_log_errors=False) + assert 'invalid signature' in caplog.text + + +def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog): + with mock.patch('requests.get') as mocked_get: + mocked_get.return_value.ok = True + mocked_get.return_value.json.side_effect = ValueError('invalid JSON') + with pytest.raises(SystemExit): + olap_cmd(no_log_errors=False) + assert 'Invalid JSON content' in caplog.text diff --git a/tox.ini b/tox.ini index 857802f..70894e7 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,7 @@ deps = psycopg2-binary vobject gadjo + mock django>=1.11,<1.12 commands = ./get_wcs.sh diff --git a/wcs_olap/feeder.py b/wcs_olap/feeder.py index 59b0459..1aee076 100644 --- a/wcs_olap/feeder.py +++ b/wcs_olap/feeder.py @@ -7,6 +7,7 @@ import hashlib from utils import Whatever import psycopg2 +from cached_property import cached_property from wcs_olap.wcs_api import WcsApiError @@ -48,12 +49,6 @@ class WcsOlapFeeder(object): self.fake = fake self.logger = logger or Whatever() self.schema = schema - self.connection = psycopg2.connect(dsn=pg_dsn) - self.connection.autocommit = True - self.cur = self.connection.cursor() - self.formdefs = api.formdefs - self.roles = api.roles - self.categories = api.categories self.do_feed = do_feed self.ctx = Context() self.ctx.push({ @@ -239,9 +234,29 @@ class WcsOlapFeeder(object): self.base_cube = self.model['cubes'][0] self.agents_mapping = {} self.formdata_json_index = [] - self.has_jsonb = self.detect_jsonb() - if self.has_jsonb: - cube['json_field'] = 'json_data' + # keep at end of __init__ to prevent leak if __init__ raises + self.connection = psycopg2.connect(dsn=pg_dsn) + self.connection.autocommit = True + self.cur = self.connection.cursor() + try: + self.has_jsonb = self.detect_jsonb() + if self.has_jsonb: + cube['json_field'] = 'json_data' + except Exception: + self.connection.close() + raise + + @cached_property + def formdefs(self): + return self.api.formdefs + + @cached_property + def roles(self): + return self.api.roles + + @cached_property + def categories(self): + return self.api.categories def detect_jsonb(self): self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'") @@ -436,17 +451,13 @@ CREATE TABLE public.dates AS (SELECT formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed) formdef_feeder.feed() except WcsApiError as e: - # ignore authorization errors - if (len(e.args) > 2 and - getattr(e.args[2], 'response', None) and - e.args[2].response.status_code == 403): - continue - self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e) + self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e) if 'cubes_model_dirs' in self.config: model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema) with open(model_path, 'w') as f: json.dump(self.model, f, indent=2, sort_keys=True) except Exception: + # keep temporary schema alive for debugging raise else: if self.do_feed: diff --git a/wcs_olap/wcs_api.py b/wcs_olap/wcs_api.py index 8f56aa0..a83b7e1 100644 --- a/wcs_olap/wcs_api.py +++ b/wcs_olap/wcs_api.py @@ -1,3 +1,4 @@ +import six import requests import urlparse import urllib @@ -11,8 +12,43 @@ from . import signature logger = logging.getLogger(__name__) +def exception_to_text(e): + try: + return six.text_type(e) + except Exception: + pass + + try: + return six.text_type(e.decode('utf8')) + except Exception: + pass + + try: + return six.text_type(repr(e)) + except Exception: + pass + + try: + args = e.args + try: + content = six.text_type(repr(args)) if args != [] else '' + except Exception: + content = '' + except AttributeError: + content = '' + return u'%s(%s)' % (e.__class__.__name__, content) + + class WcsApiError(Exception): - pass + def __init__(self, message, **kwargs): + super(WcsApiError, self).__init__(message) + self.kwargs = kwargs + + def __str__(self): + kwargs = self.kwargs.copy() + if 'exception' in kwargs: + kwargs['exception'] = exception_to_text(kwargs['exception']) + return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items())) class BaseObject(object): @@ -205,16 +241,24 @@ class WcsApi(object): signed_url = signature.sign_url(presigned_url, self.key) try: response = requests.get(signed_url, verify=self.verify) - response.raise_for_status() - except requests.RequestException, e: - raise WcsApiError('GET request failed', signed_url, e) + except requests.RequestException as e: + raise WcsApiError('GET request failed', url=signed_url, exception=e) else: + if not response.ok: + try: + text = response.text + except UnicodeError: + text = '' + repr(response.content) + raise WcsApiError('GET response is not 200', + url=signed_url, + status_code=response.status_code, + content=text) try: content = response.json() self.cache[presigned_url] = content return content - except ValueError, e: - raise WcsApiError('Invalid JSON content', signed_url, e) + except ValueError as e: + raise WcsApiError('Invalid JSON content', url=signed_url, exception=e) @property def roles(self): -- 2.20.1