From 59d3190118837dc661928c609fffb6beee302ffa Mon Sep 17 00:00:00 2001 From: Thomas NOEL Date: Mon, 2 May 2016 13:48:36 +0200 Subject: [PATCH] data_source: dont ignore list items without text (#10798) --- tests/test_datasource.py | 17 +++++++++++++++++ wcs/data_sources.py | 13 +++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/test_datasource.py b/tests/test_datasource.py index 1ddd09b..d0d1b5c 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -152,6 +152,23 @@ def test_json_datasource(): assert data_sources.get_structured_items(datasource) == [ {'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}] + # a json file with empty or no text values + json_file = open(json_file_path, 'w') + json.dump({'data': [{'id': '1', 'text': ''}, {'id': '2'}]}, json_file) + json_file.close() + assert data_sources.get_items(datasource) == [('1', '', '1'), ('2', '2', '2')] + assert data_sources.get_structured_items(datasource) == [ + {'id': '1', 'text': ''}, + {'id': '2', 'text': '2'}] + + # a json file with empty or no id + json_file = open(json_file_path, 'w') + json.dump({'data': [{'id': '', 'text': 'foo'}, {'text': 'bar'}, {'id': None}]}, json_file) + json_file.close() + assert data_sources.get_items(datasource) == [] + assert data_sources.get_structured_items(datasource) == [] + + def test_item_field_named_python_datasource(): NamedDataSource.wipe() data_source = NamedDataSource(name='foobar') diff --git a/wcs/data_sources.py b/wcs/data_sources.py index 980c174..07deafa 100644 --- a/wcs/data_sources.py +++ b/wcs/data_sources.py @@ -88,9 +88,6 @@ def get_items(data_source): structured_items = get_structured_items(data_source) tupled_items = [] for item in structured_items: - # skip malformed items - if not item.get('id') or not item.get('text'): - continue tupled_items.append((str(item['id']), str(item['text']), str(item.get('key', item['id'])))) return tupled_items @@ -147,7 +144,15 @@ def get_structured_items(data_source): raise ValueError('not a json dict') if type(entries.get('data')) is not list: raise ValueError('not a json dict with a data list attribute') - return entries.get('data') + items = [] + for item in entries.get('data'): + # skip malformed items + if not item.get('id'): + continue + if 'text' not in item: + item['text'] = item['id'] + items.append(item) + return items except urllib2.HTTPError as e: get_logger().warn('Error loading JSON data source (%s)' % str(e)) except urllib2.URLError as e: -- 2.8.1