0001-wcs_api-return-w.c.s.-API-errors-fixes-17924.patch
setup.py | ||
---|---|---|
54 | 54 |
maintainer_email="bdauvergne@entrouvert.com", |
55 | 55 |
packages=find_packages(), |
56 | 56 |
include_package_data=True, |
57 |
install_requires=['requests', 'psycopg2', 'isodate'], |
|
57 |
install_requires=['requests', 'psycopg2', 'isodate', 'six', 'cached-property'],
|
|
58 | 58 |
entry_points={ |
59 | 59 |
'console_scripts': ['wcs-olap=wcs_olap.cmd:main'], |
60 | 60 |
}, |
tests/conftest.py | ||
---|---|---|
200 | 200 |
yield Wcs(url='http://%s:%s/' % (HOSTNAME, PORT), appdir=WCS_DIR, pid=WCS_PID) |
201 | 201 |
os.kill(WCS_PID, 9) |
202 | 202 |
shutil.rmtree(str(WCS_DIR)) |
203 | ||
204 | ||
205 |
@pytest.fixture |
|
206 |
def olap_cmd(wcs, tmpdir, postgres_db): |
|
207 |
config_ini = tmpdir / 'config.ini' |
|
208 |
model_dir = tmpdir / 'model_dir' |
|
209 |
model_dir.mkdir() |
|
210 |
with config_ini.open('w') as fd: |
|
211 |
fd.write(u''' |
|
212 |
[wcs-olap] |
|
213 |
cubes_model_dirs = {model_dir} |
|
214 |
pg_dsn = {dsn} |
|
215 | ||
216 |
[{wcs.url}] |
|
217 |
orig = olap |
|
218 |
key = olap |
|
219 |
schema = olap |
|
220 |
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn)) |
|
221 | ||
222 |
from wcs_olap import cmd |
|
223 |
import sys |
|
224 | ||
225 |
def f(no_log_errors=True): |
|
226 |
old_argv = sys.argv |
|
227 |
try: |
|
228 |
sys.argv = ['', str(config_ini)] |
|
229 |
if no_log_errors: |
|
230 |
sys.argv.insert(1, '--no-log-errors') |
|
231 |
cmd.main2() |
|
232 |
finally: |
|
233 |
sys.argv = old_argv |
|
234 |
f.model_dir = model_dir |
|
235 |
return f |
tests/test_wcs.py | ||
---|---|---|
1 | 1 |
import json |
2 |
import pathlib2 |
|
3 | ||
4 | 2 | |
5 |
def test_wcs_fixture(wcs, postgres_db, tmpdir, caplog): |
|
6 |
config_ini = tmpdir / 'config.ini' |
|
7 |
model_dir = tmpdir / 'model_dir' |
|
8 |
model_dir.mkdir() |
|
9 |
with config_ini.open('w') as fd: |
|
10 |
fd.write(u''' |
|
11 |
[wcs-olap] |
|
12 |
cubes_model_dirs = {model_dir} |
|
13 |
pg_dsn = {dsn} |
|
3 |
import pytest |
|
14 | 4 | |
15 |
[{wcs.url}] |
|
16 |
orig = olap |
|
17 |
key = olap |
|
18 |
schema = olap |
|
19 |
'''.format(wcs=wcs, model_dir=model_dir, dsn=postgres_db.dsn)) |
|
5 |
import requests |
|
6 |
import pathlib2 |
|
7 |
import mock |
|
20 | 8 | |
21 |
from wcs_olap import cmd |
|
22 |
import sys |
|
23 | 9 | |
24 |
sys.argv = ['', '--no-log-errors', str(config_ini)] |
|
25 |
cmd.main2()
|
|
10 |
def test_wcs_fixture(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
|
11 |
olap_cmd()
|
|
26 | 12 | |
27 | 13 |
expected_schema = [ |
28 | 14 |
('agent', 'id'), |
... | ... | |
94 | 80 |
assert list(c.fetchall()) == expected_schema |
95 | 81 | |
96 | 82 |
# verify JSON schema |
97 |
with (model_dir / 'olap.model').open() as fd, (pathlib2.Path(__file__).parent / 'olap.model').open() as fd2: |
|
83 |
with (olap_cmd.model_dir / 'olap.model').open() as fd, \ |
|
84 |
(pathlib2.Path(__file__).parent / 'olap.model').open() as fd2: |
|
98 | 85 |
json_schema = json.load(fd) |
99 | 86 |
expected_json_schema = json.load(fd2) |
100 | 87 |
expected_json_schema['pg_dsn'] = postgres_db.dsn |
101 | 88 |
assert json_schema == expected_json_schema |
89 | ||
90 | ||
91 |
def test_requests_exception(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
|
92 |
with mock.patch('requests.get', side_effect=requests.RequestException('wat!')): |
|
93 |
with pytest.raises(SystemExit): |
|
94 |
olap_cmd(no_log_errors=False) |
|
95 |
assert 'wat!' in caplog.text |
|
96 | ||
97 | ||
98 |
def test_requests_not_ok(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
|
99 |
with mock.patch('requests.get') as mocked_get: |
|
100 |
mocked_get.return_value.ok = False |
|
101 |
mocked_get.return_value.status_code = 401 |
|
102 |
mocked_get.return_value.text = '{"err": 1, "err_desc": "invalid signature"}' |
|
103 |
with pytest.raises(SystemExit): |
|
104 |
olap_cmd(no_log_errors=False) |
|
105 |
assert 'invalid signature' in caplog.text |
|
106 | ||
107 | ||
108 |
def test_requests_not_json(wcs, postgres_db, tmpdir, olap_cmd, caplog): |
|
109 |
with mock.patch('requests.get') as mocked_get: |
|
110 |
mocked_get.return_value.ok = True |
|
111 |
mocked_get.return_value.json.side_effect = ValueError('invalid JSON') |
|
112 |
with pytest.raises(SystemExit): |
|
113 |
olap_cmd(no_log_errors=False) |
|
114 |
assert 'Invalid JSON content' in caplog.text |
tox.ini | ||
---|---|---|
22 | 22 |
psycopg2-binary |
23 | 23 |
vobject |
24 | 24 |
gadjo |
25 |
mock |
|
25 | 26 |
django>=1.11,<1.12 |
26 | 27 |
commands = |
27 | 28 |
./get_wcs.sh |
wcs_olap/feeder.py | ||
---|---|---|
7 | 7 |
from utils import Whatever |
8 | 8 |
import psycopg2 |
9 | 9 | |
10 |
from cached_property import cached_property |
|
10 | 11 |
from wcs_olap.wcs_api import WcsApiError |
11 | 12 | |
12 | 13 | |
... | ... | |
48 | 49 |
self.fake = fake |
49 | 50 |
self.logger = logger or Whatever() |
50 | 51 |
self.schema = schema |
51 |
self.connection = psycopg2.connect(dsn=pg_dsn) |
|
52 |
self.connection.autocommit = True |
|
53 |
self.cur = self.connection.cursor() |
|
54 |
self.formdefs = api.formdefs |
|
55 |
self.roles = api.roles |
|
56 |
self.categories = api.categories |
|
57 | 52 |
self.do_feed = do_feed |
58 | 53 |
self.ctx = Context() |
59 | 54 |
self.ctx.push({ |
... | ... | |
239 | 234 |
self.base_cube = self.model['cubes'][0] |
240 | 235 |
self.agents_mapping = {} |
241 | 236 |
self.formdata_json_index = [] |
242 |
self.has_jsonb = self.detect_jsonb() |
|
243 |
if self.has_jsonb: |
|
244 |
cube['json_field'] = 'json_data' |
|
237 |
# keep at end of __init__ to prevent leak if __init__ raises |
|
238 |
self.connection = psycopg2.connect(dsn=pg_dsn) |
|
239 |
self.connection.autocommit = True |
|
240 |
self.cur = self.connection.cursor() |
|
241 |
try: |
|
242 |
self.has_jsonb = self.detect_jsonb() |
|
243 |
if self.has_jsonb: |
|
244 |
cube['json_field'] = 'json_data' |
|
245 |
except Exception: |
|
246 |
self.connection.close() |
|
247 |
raise |
|
248 | ||
249 |
@cached_property |
|
250 |
def formdefs(self): |
|
251 |
return self.api.formdefs |
|
252 | ||
253 |
@cached_property |
|
254 |
def roles(self): |
|
255 |
return self.api.roles |
|
256 | ||
257 |
@cached_property |
|
258 |
def categories(self): |
|
259 |
return self.api.categories |
|
245 | 260 | |
246 | 261 |
def detect_jsonb(self): |
247 | 262 |
self.cur.execute("SELECT 1 FROM pg_type WHERE typname = 'jsonb'") |
... | ... | |
436 | 451 |
formdef_feeder = WcsFormdefFeeder(self, formdef, do_feed=self.do_feed) |
437 | 452 |
formdef_feeder.feed() |
438 | 453 |
except WcsApiError as e: |
439 |
# ignore authorization errors |
|
440 |
if (len(e.args) > 2 and |
|
441 |
getattr(e.args[2], 'response', None) and |
|
442 |
e.args[2].response.status_code == 403): |
|
443 |
continue |
|
444 |
self.logger.error('failed to retrieve formdef %s (%s)', formdef.slug, e) |
|
454 |
self.logger.error(u'failed to retrieve formdef %s, %s', formdef.slug, e) |
|
445 | 455 |
if 'cubes_model_dirs' in self.config: |
446 | 456 |
model_path = os.path.join(self.config['cubes_model_dirs'], '%s.model' % self.schema) |
447 | 457 |
with open(model_path, 'w') as f: |
448 | 458 |
json.dump(self.model, f, indent=2, sort_keys=True) |
449 | 459 |
except Exception: |
460 |
# keep temporary schema alive for debugging |
|
450 | 461 |
raise |
451 | 462 |
else: |
452 | 463 |
if self.do_feed: |
wcs_olap/wcs_api.py | ||
---|---|---|
1 |
import six |
|
1 | 2 |
import requests |
2 | 3 |
import urlparse |
3 | 4 |
import urllib |
... | ... | |
11 | 12 |
logger = logging.getLogger(__name__) |
12 | 13 | |
13 | 14 | |
15 |
def exception_to_text(e): |
|
16 |
try: |
|
17 |
return six.text_type(e) |
|
18 |
except Exception: |
|
19 |
pass |
|
20 | ||
21 |
try: |
|
22 |
return six.text_type(e.decode('utf8')) |
|
23 |
except Exception: |
|
24 |
pass |
|
25 | ||
26 |
try: |
|
27 |
return six.text_type(repr(e)) |
|
28 |
except Exception: |
|
29 |
pass |
|
30 | ||
31 |
try: |
|
32 |
args = e.args |
|
33 |
try: |
|
34 |
content = six.text_type(repr(args)) if args != [] else '' |
|
35 |
except Exception: |
|
36 |
content = '<exception-while-rendering-args>' |
|
37 |
except AttributeError: |
|
38 |
content = '' |
|
39 |
return u'%s(%s)' % (e.__class__.__name__, content) |
|
40 | ||
41 | ||
14 | 42 |
class WcsApiError(Exception): |
15 |
pass |
|
43 |
def __init__(self, message, **kwargs): |
|
44 |
super(WcsApiError, self).__init__(message) |
|
45 |
self.kwargs = kwargs |
|
46 | ||
47 |
def __str__(self): |
|
48 |
kwargs = self.kwargs.copy() |
|
49 |
if 'exception' in kwargs: |
|
50 |
kwargs['exception'] = exception_to_text(kwargs['exception']) |
|
51 |
return '%s: %s' % (self.args[0], ' '.join('%s=%s' % (key, value) for key, value in kwargs.items())) |
|
16 | 52 | |
17 | 53 | |
18 | 54 |
class BaseObject(object): |
... | ... | |
205 | 241 |
signed_url = signature.sign_url(presigned_url, self.key) |
206 | 242 |
try: |
207 | 243 |
response = requests.get(signed_url, verify=self.verify) |
208 |
response.raise_for_status() |
|
209 |
except requests.RequestException, e: |
|
210 |
raise WcsApiError('GET request failed', signed_url, e) |
|
244 |
except requests.RequestException as e: |
|
245 |
raise WcsApiError('GET request failed', url=signed_url, exception=e) |
|
211 | 246 |
else: |
247 |
if not response.ok: |
|
248 |
try: |
|
249 |
text = response.text |
|
250 |
except UnicodeError: |
|
251 |
text = '<undecodable>' + repr(response.content) |
|
252 |
raise WcsApiError('GET response is not 200', |
|
253 |
url=signed_url, |
|
254 |
status_code=response.status_code, |
|
255 |
content=text) |
|
212 | 256 |
try: |
213 | 257 |
content = response.json() |
214 | 258 |
self.cache[presigned_url] = content |
215 | 259 |
return content |
216 |
except ValueError, e:
|
|
217 |
raise WcsApiError('Invalid JSON content', signed_url, e)
|
|
260 |
except ValueError as e:
|
|
261 |
raise WcsApiError('Invalid JSON content', url=signed_url, exception=e)
|
|
218 | 262 | |
219 | 263 |
@property |
220 | 264 |
def roles(self): |
221 |
- |