From 31c2df52bfc47be3d8ca8bc48595d58c6b40603a Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 25 Mar 2016 19:46:49 +0100 Subject: [PATCH 1/2] encode data source value using qommon.misc.json_encode_helper (#10447) It prevents data sources from outputting unicode strings. --- tests/test_datasource.py | 21 +++++++++++++++++++++ wcs/data_sources.py | 19 ++++++++++--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/test_datasource.py b/tests/test_datasource.py index 19afdfe..1ddd09b 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -11,6 +11,7 @@ from wcs.qommon.form import * from wcs import fields, data_sources from wcs.data_sources import NamedDataSource, register_data_source_function +import mock from test_widgets import MockHtmlForm, mock_form_submission from utilities import create_temporary_pub @@ -215,6 +216,7 @@ def test_optional_item_field_with_data_source(): assert widget is not None assert widget.options == [('1', 'un', '1'), ('2', 'deux', '2')] + def test_data_source_unicode(): NamedDataSource.wipe() data_source = NamedDataSource(name='foobar') @@ -223,3 +225,22 @@ def test_data_source_unicode(): data_source2 = NamedDataSource.select()[0] assert data_source2.data_source == data_source.data_source + assert data_sources.get_items({'type': 'foobar'}) == [ + ('uné',) * 3, + ('deux',) * 3, + ] + + NamedDataSource.wipe() + data_source = NamedDataSource(name='foobar') + data_source.data_source = {'type': 'json', 'value': "https://whatever.com/json"} + data_source.store() + + data_source2 = NamedDataSource.select()[0] + assert data_source2.data_source == data_source.data_source + with mock.patch('wcs.data_sources.urllib2') as urllib2: + urllib2.urlopen.return_value.read.return_value = \ + '{"data": [{"id": 1, "text": "uné"}, {"id": 2, "text": "deux"}]}' + assert data_sources.get_items({'type': 'foobar'}) == [ + ('1', 'uné', '1'), + ('2', 'deux', '2'), + ] diff --git a/wcs/data_sources.py b/wcs/data_sources.py index a72e466..7027eed 100644 --- a/wcs/data_sources.py +++ b/wcs/data_sources.py @@ -15,7 +15,6 @@ # along with this program; if not, see . import collections -import json import urllib2 import xml.etree.ElementTree as ET @@ -84,15 +83,18 @@ class DataSourceSelectionWidget(CompositeWidget): r += widget.render_content() return r.getvalue() + def get_items(data_source): structured_items = get_structured_items(data_source) - tupled_items = [(site_encode(x.get('id')), - site_encode(x.get('text')), - site_encode(x.get('key'))) for x in structured_items] - if tupled_items and tupled_items[0][2] is None: # no key - tupled_items = [tuple(x[:2]) + (x[0],) for x in tupled_items] + tupled_items = [] + for item in structured_items: + # skip malformed items + if not item.get('id') or not item.get('text'): + continue + tupled_items.append((str(item['id']), str(item['text']), str(item.get('key', item['id'])))) return tupled_items + def get_structured_items(data_source): data_source = get_real(data_source) if data_source.get('type') == 'formula': @@ -113,6 +115,7 @@ def get_structured_items(data_source): return [] if len(value) == 0: return [] + value = qommon.misc.json_encode_helper(value, get_publisher().site_charset) if isinstance(value[0], list) or isinstance(value[0], tuple): if len(value[0]) >= 3: return [{'id': x[0], 'text': x[1], 'key': x[2]} for x in value] @@ -138,10 +141,8 @@ def get_structured_items(data_source): if '[' in url: vars = get_publisher().substitutions.get_context_variables() url = get_variadic_url(url, vars) - charset = get_publisher().site_charset try: - results = [] - entries = json.load(urllib2.urlopen(url)) + entries = qommon.misc.json_loads(urllib2.urlopen(url).read()) if type(entries) is not dict: raise ValueError('not a json dict') if type(entries.get('data')) is not list: -- 2.1.4