0001-data_source-handle-err-in-JSON-outputs-41195.patch

Thomas Noël, 31 mars 2020 17:11

Voir les différences: en ligne côte à côte

Subject: [PATCH] data_source: handle err in JSON outputs (#41195)

 tests/test_datasource.py | 66 ++++++++++++++++++++++++++++++++++++++++
 tests/utilities.py       |  1 +
 wcs/data_sources.py      | 24 ++++++++++++---
 3 files changed, 86 insertions(+), 5 deletions(-)

         assert 'Error loading JSON data source' in caplog.records[-1].message
         assert 'error' in caplog.records[-1].message
         datasource = {'type': 'json', 'value': 'http://remote.example.net/json-list-err1'}
         assert data_sources.get_items(datasource) == []
         assert 'Error reading JSON data source output (err 1)' in caplog.records[-1].message
     def test_json_datasource_bad_url_scheme(caplog):
         datasource = {'type': 'json', 'value': ''}
-...
             assert urlopen.call_count == 3
     def test_named_datasource_id_parameter(requests_pub):
         NamedDataSource.wipe()
         datasource = NamedDataSource(name='foobar')
         datasource.data_source = {'type': 'json', 'value': 'http://whatever/'}
         datasource.id_parameter = 'id'
         datasource.store()
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             value = [{'id': '1', 'text': 'foo'}]
             urlopen.side_effect = lambda *args: StringIO(json.dumps({'data': value}))
             assert datasource.get_structured_value('1') == value[0]
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
             # try again, get from request.datasources_cache
             assert datasource.get_structured_value('1') == value[0]
             assert urlopen.call_count == 1  # no new call
         get_request().datasources_cache = {}
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             value = [{'id': '1', 'text': 'bar'}, {'id': '2', 'text': 'foo'}]
             urlopen.side_effect = lambda *args: StringIO(json.dumps({'data': value}))
             assert datasource.get_structured_value('1') == value[0]
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
         get_request().datasources_cache = {}
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             value = [{'id': '1', 'text': 'foo'}]
             urlopen.side_effect = lambda *args: StringIO(json.dumps({'data': value, 'err': 0}))
             assert datasource.get_structured_value('1') == value[0]
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
         get_request().datasources_cache = {}
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             value = [{'id': '1', 'text': 'foo'}]
             urlopen.side_effect = lambda *args: StringIO(json.dumps({'data': value, 'err': 1}))
             assert datasource.get_structured_value('1') is None
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
             # no cache for errors
             assert datasource.get_structured_value('1') is None
             assert urlopen.call_count == 2  # called
         get_request().datasources_cache = {}
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             value = {'id': '1', 'text': 'foo'}  # not a list
             urlopen.side_effect = lambda *args: StringIO(json.dumps({'data': value}))
             assert datasource.get_structured_value('1') is None
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
         get_request().datasources_cache = {}
         with mock.patch('wcs.qommon.misc.urlopen') as urlopen:
             urlopen.side_effect = lambda *args: StringIO('not json')
             assert datasource.get_structured_value('1') is None
             assert urlopen.call_count == 1
             assert urlopen.call_args[0][0] == 'http://whatever/?id=1'
     def test_named_datasource_in_formdef():
         from wcs.formdef import FormDef
         datasource = NamedDataSource(name='foobar')

tests/utilities.py
329	329	'http://remote.example.net/json-list': (200, '{"data": [{"id": "a", "text": "b"}]}', None),
330	330	'http://remote.example.net/json-err0': (200, '{"data": "foo", "err": 0}', None),
331	331	'http://remote.example.net/json-err1': (200, '{"data": "", "err": 1}', None),
	332	'http://remote.example.net/json-list-err1': (200, '{"data": [{"id": "a", "text": "b"}], "err": 1}', None),
332	333	'http://remote.example.net/json-errstr': (200, '{"data": "", "err": "bug"}', None),
333	334	'http://remote.example.net/json-errheader0': (200, '{"foo": "bar"}',
334	335	{'x-error-code': '0'}),

                 entries = misc.json_loads(misc.urlopen(url).read())
                 if type(entries) is not dict:
                     raise ValueError('not a json dict')
                 if entries.get('err') not in (None, 0, "0"):
                     raise ValueError('err %s' % entries['err'])
                 if type(entries.get('data')) is not list:
                     raise ValueError('not a json dict with a data list attribute')
                 items = []
-...
             unsigned_url = url
             url = sign_url_auto_orig(url)
             resp = misc.urlopen(url).read()
             try:
                 resp = misc.urlopen(url).read()
                 resp = misc.json_loads(resp)
                 if type(resp) is not dict:
                     raise ValueError('not a json dict')
                 if resp.get('err') not in (None, 0, "0"):
                     raise ValueError('err %s' % resp['err'])
                 if type(resp.get('data')) is not list:
                     raise ValueError('not a json dict with a data list attribute')
                 resp = resp['data'][0]
             except misc.ConnectionError as e:
                 get_logger().warn('Error loading JSON data source (%s)' % str(e))
                 return None
             except (ValueError, TypeError) as e:
                 get_logger().warn('Error reading JSON data source output (%s)' % str(e))
                 return None
             if hasattr(request, 'datasources_cache'):
                 request.datasources_cache[unsigned_url] = resp
             return resp
-...
         def get_structured_value(self, option_id):
             value = None
             if self.type == 'json' and self.id_parameter:
                 resp = self.load_json(self.id_parameter, option_id)
                 response = misc.json_loads(resp)
                 if response['data']:
                     value = response['data'][0]
                 value = self.load_json(self.id_parameter, option_id)
             else:
                 structured_items = get_structured_items(self.data_source, mode='lazy')
                 for item in structured_items:
+    -

Projet

Général

Profil

Produits Entr'ouvert » w.c.s.

0001-data_source-handle-err-in-JSON-outputs-41195.patch