Projet

Général

Profil

0001-wcs_api-return-w.c.s.-API-errors-fixes-17924.patch

Benjamin Dauvergne, 18 janvier 2019 15:30

Télécharger (10,5 ko)

Voir les différences:

Subject: [PATCH] wcs_api: return w.c.s. API errors (fixes #17924)

 setup.py            |  2 +-
 tests/conftest.py   | 33 ++++++++++++++++++++++++++
 tests/test_wcs.py   | 55 +++++++++++++++++++++++++++-----------------
 tox.ini             |  1 +
 wcs_olap/feeder.py  | 41 +++++++++++++++++++++------------
 wcs_olap/wcs_api.py | 56 ++++++++++++++++++++++++++++++++++++++++-----
 6 files changed, 145 insertions(+), 43 deletions(-)
setup.py
54 54
      maintainer_email="bdauvergne@entrouvert.com",
55 55
      packages=find_packages(),
56 56
      include_package_data=True,
57
      install_requires=['requests', 'psycopg2', 'isodate'],
57
      install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'],
58 58
      entry_points={
59 59
          'console_scripts': ['wcs-olap=wcs_olap.cmd:main'],
60 60
      },
tests/conftest.py
200 200
    yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID)
201 201
    os.kill(WCS_PID, 9)
202 202
    shutil.rmtree(str(WCS_DIR))
203

  
204

  
205
@pytest.fixture
206
def olap_cmd(wcs, tmpdir, postgres_db):
207
    config_ini = tmpdir / 'config.ini'
208
    model_dir = tmpdir / 'model_dir'
209
    model_dir.mkdir()
210
    with config_ini.open('w') as fd:
211
        fd.write(u'''
212
[wcs-olap]
213
cubes_model_dirs = {model_dir}
214
pg_dsn = {dsn}
215

  
216
[{wcs.url}]
217
orig = olap
218
key = olap
219
schema = olap
220
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
221

  
222
    from wcs_olap import cmd
223
    import sys
224

  
225
    def f(no_log_errors=True):
226
        old_argv = sys.argv
227
        try:
228
            sys.argv = ['', str(config_ini)]
229
            if no_log_errors:
230
                sys.argv.insert(1, '--no-log-errors')
231
            cmd.main2()
232
        finally:
233
            sys.argv = old_argv
234
    f.model_dir = model_dir
235
    return f
tests/test_wcs.py
1 1
import json
2
import pathlib2
3

  
4 2

  
5
def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog):
6
    config_ini = tmpdir / 'config.ini'
7
    model_dir = tmpdir / 'model_dir'
8
    model_dir.mkdir()
9
    with config_ini.open('w') as fd:
10
        fd.write(u'''
11
[wcs-olap]
12
cubes_model_dirs = {model_dir}
13
pg_dsn = {dsn}
3
import pytest
14 4

  
15
[{wcs.url}]
16
orig = olap
17
key = olap
18
schema = olap
19
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn))
5
import requests
6
import pathlib2
7
import mock
20 8

  
21
    from wcs_olap import cmd
22
    import sys
23 9

  
24
    sys.argv = ['', '--no-log-errors', str(config_ini)]
25
    cmd.main2()
10
def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog):
11
    olap_cmd()
26 12

  
27 13
    expected_schema = [
28 14
        ('agent', 'id'),
......
94 80
            assert list(c.fetchall()) == expected_schema
95 81

  
96 82
    # verify JSON schema
97
    with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
83
    with (olap_cmd.model_dir / 'olap.model').open() as fd, \
84
            (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2:
98 85
            json_schema = json.load(fd)
99 86
            expected_json_schema = json.load(fd2)
100 87
            expected_json_schema['pg_dsn'] = postgres_db.dsn
101 88
            assert json_schema == expected_json_schema
89

  
90

  
91
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog):
92
    with mock.patch('requests.get', side_effect=requests.RequestException('wat!')):
93
        with pytest.raises(SystemExit):
94
            olap_cmd(no_log_errors=False)
95
    assert 'wat!' in caplog.text
96

  
97

  
98
def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog):
99
    with mock.patch('requests.get') as mocked_get:
100
        mocked_get.return_value.ok = False
101
        mocked_get.return_value.status_code = 401
102
        mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}'
103
        with pytest.raises(SystemExit):
104
            olap_cmd(no_log_errors=False)
105
    assert 'invalid signature' in caplog.text
106

  
107

  
108
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog):
109
    with mock.patch('requests.get') as mocked_get:
110
        mocked_get.return_value.ok = True
111
        mocked_get.return_value.json.side_effect = ValueError('invalid JSON')
112
        with pytest.raises(SystemExit):
113
            olap_cmd(no_log_errors=False)
114
    assert 'Invalid JSON content' in caplog.text
tox.ini
22 22
	psycopg2-binary
23 23
	vobject
24 24
	gadjo
25
	mock
25 26
	django>=1.11,<1.12
26 27
commands =
27 28
	./get_wcs.sh
wcs_olap/feeder.py
7 7
from utils import Whatever
8 8
import psycopg2
9 9

  
10
from cached_property import cached_property
10 11
from wcs_olap.wcs_api import WcsApiError
11 12

  
12 13

  
......
48 49
        self.fake = fake
49 50
        self.logger = logger or Whatever()
50 51
        self.schema = schema
51
        self.connection = psycopg2.connect(dsn=pg_dsn)
52
        self.connection.autocommit = True
53
        self.cur = self.connection.cursor()
54
        self.formdefs = api.formdefs
55
        self.roles = api.roles
56
        self.categories = api.categories
57 52
        self.do_feed = do_feed
58 53
        self.ctx = Context()
59 54
        self.ctx.push({
......
239 234
        self.base_cube = self.model['cubes'][0]
240 235
        self.agents_mapping = {}
241 236
        self.formdata_json_index = []
242
        self.has_jsonb = self.detect_jsonb()
243
        if self.has_jsonb:
244
            cube['json_field'] = 'json_data'
237
        # keep at end of __init__ to prevent leak if __init__ raises
238
        self.connection = psycopg2.connect(dsn=pg_dsn)
239
        self.connection.autocommit = True
240
        self.cur = self.connection.cursor()
241
        try:
242
            self.has_jsonb = self.detect_jsonb()
243
            if self.has_jsonb:
244
                cube['json_field'] = 'json_data'
245
        except Exception:
246
            self.connection.close()
247
            raise
248

  
249
    @cached_property
250
    def formdefs(self):
251
        return self.api.formdefs
252

  
253
    @cached_property
254
    def roles(self):
255
        return self.api.roles
256

  
257
    @cached_property
258
    def categories(self):
259
        return self.api.categories
245 260

  
246 261
    def detect_jsonb(self):
247 262
        self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'")
......
436 451
                    formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed)
437 452
                    formdef_feeder.feed()
438 453
                except WcsApiError as e:
439
                    # ignore authorization errors
440
                    if (len(e.args) > 2 and
441
                            getattr(e.args[2], 'response', None) and
442
                            e.args[2].response.status_code == 403):
443
                        continue
444
                    self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e)
454
                    self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e)
445 455
            if 'cubes_model_dirs' in self.config:
446 456
                model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema)
447 457
                with open(model_path, 'w') as f:
448 458
                    json.dump(self.model, f, indent=2, sort_keys=True)
449 459
        except Exception:
460
            # keep temporary schema alive for debugging
450 461
            raise
451 462
        else:
452 463
            if self.do_feed:
wcs_olap/wcs_api.py
1
import six
1 2
import requests
2 3
import urlparse
3 4
import urllib
......
11 12
logger = logging.getLogger(__name__)
12 13

  
13 14

  
15
def exception_to_text(e):
16
    try:
17
        return six.text_type(e)
18
    except Exception:
19
        pass
20

  
21
    try:
22
        return six.text_type(e.decode('utf8'))
23
    except Exception:
24
        pass
25

  
26
    try:
27
        return six.text_type(repr(e))
28
    except Exception:
29
        pass
30

  
31
    try:
32
        args = e.args
33
        try:
34
            content = six.text_type(repr(args)) if args != [] else ''
35
        except Exception:
36
            content = '<exception-while-rendering-args>'
37
    except AttributeError:
38
        content = ''
39
    return u'%s(%s)' % (e.__class__.__name__, content)
40

  
41

  
14 42
class WcsApiError(Exception):
15
    pass
43
    def __init__(self, message, **kwargs):
44
        super(WcsApiError, self).__init__(message)
45
        self.kwargs = kwargs
46

  
47
    def __str__(self):
48
        kwargs = self.kwargs.copy()
49
        if 'exception' in kwargs:
50
            kwargs['exception'] = exception_to_text(kwargs['exception'])
51
        return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items()))
16 52

  
17 53

  
18 54
class BaseObject(object):
......
205 241
        signed_url = signature.sign_url(presigned_url, self.key)
206 242
        try:
207 243
            response = requests.get(signed_url, verify=self.verify)
208
            response.raise_for_status()
209
        except requests.RequestException, e:
210
            raise WcsApiError('GET request failed', signed_url, e)
244
        except requests.RequestException as e:
245
            raise WcsApiError('GET request failed', url=signed_url, exception=e)
211 246
        else:
247
            if not response.ok:
248
                try:
249
                    text = response.text
250
                except UnicodeError:
251
                    text = '<undecodable>' + repr(response.content)
252
                raise WcsApiError('GET response is not 200',
253
                                  url=signed_url,
254
                                  status_code=response.status_code,
255
                                  content=text)
212 256
            try:
213 257
                content = response.json()
214 258
                self.cache[presigned_url] = content
215 259
                return content
216
            except ValueError, e:
217
                raise WcsApiError('Invalid JSON content', signed_url, e)
260
            except ValueError as e:
261
                raise WcsApiError('Invalid JSON content', url=signed_url, exception=e)
218 262

  
219 263
    @property
220 264
    def roles(self):
221
-