0001-datasources-new-datasource-type-jsonvalue-to-replace.patch
| tests/admin_pages/test_datasource.py | ||
|---|---|---|
| 235 | 235 |
('jsonp', False, 'JSONP URL'),
|
| 236 | 236 |
('geojson', False, 'GeoJSON URL'),
|
| 237 | 237 |
('python', False, 'Python Expression'),
|
| 238 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 238 | 239 |
] |
| 239 | 240 | |
| 240 | 241 |
pub.site_options.set('options', 'disable-python-expressions', 'true')
|
| ... | ... | |
| 247 | 248 |
('json', False, 'JSON URL'),
|
| 248 | 249 |
('jsonp', False, 'JSONP URL'),
|
| 249 | 250 |
('geojson', False, 'GeoJSON URL'),
|
| 251 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 250 | 252 |
] |
| 251 | 253 | |
| 252 | 254 | |
| tests/admin_pages/test_form.py | ||
|---|---|---|
| 2101 | 2101 |
create_superuser(pub) |
| 2102 | 2102 |
create_role(pub) |
| 2103 | 2103 | |
| 2104 |
NamedDataSource.wipe() |
|
| 2104 | 2105 |
FormDef.wipe() |
| 2105 | 2106 |
formdef = FormDef() |
| 2106 | 2107 |
formdef.name = 'form title' |
| ... | ... | |
| 2114 | 2115 |
('json', False, 'JSON URL'),
|
| 2115 | 2116 |
('jsonp', False, 'JSONP URL'),
|
| 2116 | 2117 |
('python', False, 'Python Expression'),
|
| 2118 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 2117 | 2119 |
] |
| 2118 | 2120 |
resp = resp.form.submit('submit').follow()
|
| 2119 | 2121 | |
| ... | ... | |
| 2130 | 2132 |
('json', False, 'JSON URL'),
|
| 2131 | 2133 |
('jsonp', False, 'JSONP URL'),
|
| 2132 | 2134 |
('python', False, 'Python Expression'),
|
| 2135 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 2133 | 2136 |
] |
| 2134 | 2137 |
resp.form['data_mode'].value = 'data-source' |
| 2135 | 2138 |
resp.form['data_source$type'].value = 'foobar' |
| ... | ... | |
| 2149 | 2152 |
('json', False, 'JSON URL'),
|
| 2150 | 2153 |
('jsonp', False, 'JSONP URL'),
|
| 2151 | 2154 |
('python', False, 'Python Expression'),
|
| 2155 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 2152 | 2156 |
] |
| 2153 | 2157 | |
| 2154 | 2158 |
carddef.digest_templates = {'default': 'plop'}
|
| ... | ... | |
| 2168 | 2172 |
('json', False, 'JSON URL'),
|
| 2169 | 2173 |
('jsonp', False, 'JSONP URL'),
|
| 2170 | 2174 |
('python', False, 'Python Expression'),
|
| 2175 |
('jsonvalue', False, 'JSON Expression'),
|
|
| 2171 | 2176 |
] |
| 2172 | 2177 |
assert ( |
| 2173 | 2178 |
resp.pyquery('select#form_data_source__type option')[1].attrib['data-goto-url']
|
| ... | ... | |
| 2231 | 2236 |
'json', |
| 2232 | 2237 |
'jsonp', |
| 2233 | 2238 |
'python', |
| 2239 |
'jsonvalue', |
|
| 2234 | 2240 |
] |
| 2235 | 2241 | |
| 2236 | 2242 |
cat_b = DataSourceCategory(name='Cat B') |
| ... | ... | |
| 2257 | 2263 |
'json', |
| 2258 | 2264 |
'jsonp', |
| 2259 | 2265 |
'python', |
| 2266 |
'jsonvalue', |
|
| 2260 | 2267 |
] |
| 2261 | 2268 | |
| 2262 | 2269 | |
| tests/test_datasource.py | ||
|---|---|---|
| 1 | 1 |
import codecs |
| 2 |
import json |
|
| 2 | 3 |
import os |
| 3 | 4 |
import urllib.parse |
| 4 | 5 |
import xml.etree.ElementTree as ET |
| ... | ... | |
| 216 | 217 |
] |
| 217 | 218 | |
| 218 | 219 | |
| 220 |
def test_item_field_jsonvalue_datasource(requests_pub): |
|
| 221 |
req = get_request() |
|
| 222 |
req.environ['REQUEST_METHOD'] = 'POST' |
|
| 223 |
field = fields.ItemField() |
|
| 224 |
field.id = 1 |
|
| 225 |
field.data_source = {
|
|
| 226 |
'type': 'jsonvalue', |
|
| 227 |
'value': json.dumps([{'id': 1, 'text': 'un'}, {'id': 2, 'text': 'deux'}]),
|
|
| 228 |
} |
|
| 229 |
form = Form() |
|
| 230 |
field.add_to_form(form) |
|
| 231 |
widget = form.get_widget('f1')
|
|
| 232 |
assert widget is not None |
|
| 233 |
assert widget.options == [('1', 'un', '1'), ('2', 'deux', '2')]
|
|
| 234 | ||
| 235 |
form = MockHtmlForm(widget) |
|
| 236 |
mock_form_submission(req, widget, {'f1': ['1']})
|
|
| 237 |
assert widget.parse() == '1' |
|
| 238 | ||
| 239 |
form = Form() |
|
| 240 |
field.add_to_view_form(form, value='1') |
|
| 241 |
widget = form.get_widget('f1')
|
|
| 242 | ||
| 243 |
form = MockHtmlForm(widget) |
|
| 244 |
mock_form_submission(req, widget) |
|
| 245 |
assert widget.parse() == '1' |
|
| 246 | ||
| 247 | ||
| 248 |
def test_jsonvalue_datasource(pub): |
|
| 249 |
plain_list = [{"id": "1", "text": "foo"}, {"id": "2", "text": "bar"}]
|
|
| 250 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list)}
|
|
| 251 |
assert data_sources.get_items(datasource) == [ |
|
| 252 |
('1', 'foo', '1', {'id': '1', 'text': 'foo'}),
|
|
| 253 |
('2', 'bar', '2', {'id': '2', 'text': 'bar'}),
|
|
| 254 |
] |
|
| 255 |
assert data_sources.get_structured_items(datasource) == [ |
|
| 256 |
{'id': '1', 'text': 'foo'},
|
|
| 257 |
{'id': '2', 'text': 'bar'},
|
|
| 258 |
] |
|
| 259 | ||
| 260 |
# with key |
|
| 261 |
plain_list = [{"id": "1", "text": "foo", "key": "a"}, {"id": "2", "text": "bar", "key": "b"}]
|
|
| 262 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list)}
|
|
| 263 |
assert data_sources.get_items(datasource) == [ |
|
| 264 |
('1', 'foo', 'a', {'id': '1', 'key': 'a', 'text': 'foo'}),
|
|
| 265 |
('2', 'bar', 'b', {'id': '2', 'key': 'b', 'text': 'bar'}),
|
|
| 266 |
] |
|
| 267 | ||
| 268 | ||
| 269 |
def test_jsonvalue_datasource_errors(pub): |
|
| 270 |
# not a list |
|
| 271 |
datasource = {'type': 'jsonvalue', 'value': 'foobar', 'record_on_errors': True}
|
|
| 272 |
assert data_sources.get_items(datasource) == [] |
|
| 273 |
assert pub.loggederror_class.count() == 1 |
|
| 274 |
logged_error = pub.loggederror_class.select()[0] |
|
| 275 |
assert logged_error.workflow_id is None |
|
| 276 |
assert logged_error.summary == "[DATASOURCE] JSON data source ('foobar') gave a non usable result"
|
|
| 277 | ||
| 278 |
pub.loggederror_class.wipe() |
|
| 279 |
plain_list = {'foo': 'bar'}
|
|
| 280 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 281 |
assert data_sources.get_items(datasource) == [] |
|
| 282 |
assert pub.loggederror_class.count() == 1 |
|
| 283 |
logged_error = pub.loggederror_class.select()[0] |
|
| 284 |
assert logged_error.workflow_id is None |
|
| 285 |
assert ( |
|
| 286 |
logged_error.summary |
|
| 287 |
== "[DATASOURCE] JSON data source ('{\"foo\": \"bar\"}') gave a non usable result"
|
|
| 288 |
) |
|
| 289 | ||
| 290 |
# not a list of dict |
|
| 291 |
pub.loggederror_class.wipe() |
|
| 292 |
plain_list = ["foobar"] |
|
| 293 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 294 |
assert data_sources.get_items(datasource) == [] |
|
| 295 |
assert pub.loggederror_class.count() == 1 |
|
| 296 |
logged_error = pub.loggederror_class.select()[0] |
|
| 297 |
assert logged_error.workflow_id is None |
|
| 298 |
assert logged_error.summary == "[DATASOURCE] JSON data source ('[\"foobar\"]') gave a non usable result"
|
|
| 299 | ||
| 300 |
pub.loggederror_class.wipe() |
|
| 301 |
plain_list = [{'foo': 'bar'}, "foobar"]
|
|
| 302 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 303 |
assert data_sources.get_items(datasource) == [] |
|
| 304 |
assert pub.loggederror_class.count() == 1 |
|
| 305 |
logged_error = pub.loggederror_class.select()[0] |
|
| 306 |
assert logged_error.workflow_id is None |
|
| 307 |
assert ( |
|
| 308 |
logged_error.summary |
|
| 309 |
== "[DATASOURCE] JSON data source ('[{\"foo\": \"bar\"}, \"foobar\"]') gave a non usable result"
|
|
| 310 |
) |
|
| 311 | ||
| 312 |
# no id found |
|
| 313 |
pub.loggederror_class.wipe() |
|
| 314 |
plain_list = [{"text": "foo"}, {"id": "2", "text": "bar"}]
|
|
| 315 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 316 |
assert data_sources.get_items(datasource) == [] |
|
| 317 |
assert pub.loggederror_class.count() == 1 |
|
| 318 |
logged_error = pub.loggederror_class.select()[0] |
|
| 319 |
assert logged_error.workflow_id is None |
|
| 320 |
assert ( |
|
| 321 |
logged_error.summary |
|
| 322 |
== "[DATASOURCE] JSON data source ('[{\"text\": \"foo\"}, {\"id\": \"2\", \"text\": \"bar\"}]') gave a non usable result"
|
|
| 323 |
) |
|
| 324 | ||
| 325 |
pub.loggederror_class.wipe() |
|
| 326 |
plain_list = [{"id": "1", "text": "foo"}, {"id": "", "text": "bar"}]
|
|
| 327 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 328 |
assert data_sources.get_items(datasource) == [] |
|
| 329 |
assert pub.loggederror_class.count() == 1 |
|
| 330 |
logged_error = pub.loggederror_class.select()[0] |
|
| 331 |
assert logged_error.workflow_id is None |
|
| 332 |
assert ( |
|
| 333 |
logged_error.summary |
|
| 334 |
== "[DATASOURCE] JSON data source ('[{\"id\": \"1\", \"text\": \"foo\"}, {\"id\": \"\", \"text\": \"bar\"}]') gave a non usable result"
|
|
| 335 |
) |
|
| 336 | ||
| 337 |
# no text found |
|
| 338 |
pub.loggederror_class.wipe() |
|
| 339 |
plain_list = [{"id": "1"}, {"id": "2", "text": "bar"}]
|
|
| 340 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 341 |
assert data_sources.get_items(datasource) == [] |
|
| 342 |
assert pub.loggederror_class.count() == 1 |
|
| 343 |
logged_error = pub.loggederror_class.select()[0] |
|
| 344 |
assert logged_error.workflow_id is None |
|
| 345 |
assert ( |
|
| 346 |
logged_error.summary |
|
| 347 |
== "[DATASOURCE] JSON data source ('[{\"id\": \"1\"}, {\"id\": \"2\", \"text\": \"bar\"}]') gave a non usable result"
|
|
| 348 |
) |
|
| 349 | ||
| 350 |
pub.loggederror_class.wipe() |
|
| 351 |
plain_list = [{"id": "1", "text": "foo"}, {"id": "2", "text": ""}]
|
|
| 352 |
datasource = {'type': 'jsonvalue', 'value': json.dumps(plain_list), 'record_on_errors': True}
|
|
| 353 |
assert data_sources.get_items(datasource) == [] |
|
| 354 |
assert pub.loggederror_class.count() == 1 |
|
| 355 |
logged_error = pub.loggederror_class.select()[0] |
|
| 356 |
assert logged_error.workflow_id is None |
|
| 357 |
assert ( |
|
| 358 |
logged_error.summary |
|
| 359 |
== "[DATASOURCE] JSON data source ('[{\"id\": \"1\", \"text\": \"foo\"}, {\"id\": \"2\", \"text\": \"\"}]') gave a non usable result"
|
|
| 360 |
) |
|
| 361 | ||
| 362 | ||
| 219 | 363 |
def test_json_datasource(pub, requests_pub): |
| 220 | 364 |
get_request().datasources_cache = {}
|
| 221 | 365 |
datasource = {'type': 'json', 'value': ''}
|
| wcs/admin/data_sources.py | ||
|---|---|---|
| 88 | 88 |
'data_source', |
| 89 | 89 |
value=self.datasource.data_source, |
| 90 | 90 |
title=_('Data Source'),
|
| 91 |
allowed_source_types={'json', 'jsonp', 'geojson', 'python'},
|
|
| 91 |
allowed_source_types={'json', 'jsonp', 'geojson', 'python', 'jsonvalue'},
|
|
| 92 | 92 |
required=True, |
| 93 | 93 |
) |
| 94 | 94 |
form.add( |
| ... | ... | |
| 371 | 371 |
data_source = self.datasource.extended_data_source |
| 372 | 372 |
finally: |
| 373 | 373 |
get_request().disable_error_notifications = False |
| 374 |
if data_source.get('type') not in ('json', 'geojson', 'formula', 'wcs:users'):
|
|
| 374 |
if data_source.get('type') not in ('json', 'geojson', 'formula', 'jsonvalue', 'wcs:users'):
|
|
| 375 | 375 |
return '' |
| 376 | 376 |
try: |
| 377 | 377 |
items = get_structured_items(data_source) |
| wcs/data_sources.py | ||
|---|---|---|
| 17 | 17 |
import collections |
| 18 | 18 |
import collections.abc |
| 19 | 19 |
import hashlib |
| 20 |
import json |
|
| 20 | 21 |
import urllib.parse |
| 21 | 22 |
import xml.etree.ElementTree as ET |
| 22 | 23 | |
| ... | ... | |
| 57 | 58 |
class DataSourceSelectionWidget(CompositeWidget): |
| 58 | 59 |
def __init__(self, name, value=None, allowed_source_types=None, disallowed_source_types=None, **kwargs): |
| 59 | 60 |
if allowed_source_types is None: |
| 60 |
allowed_source_types = {'json', 'jsonp', 'geojson', 'named', 'cards', 'python'}
|
|
| 61 |
allowed_source_types = {'json', 'jsonp', 'geojson', 'named', 'cards', 'python', 'jsonvalue'}
|
|
| 61 | 62 |
if get_publisher().has_site_option('disable-python-expressions') and 'python' in allowed_source_types:
|
| 62 | 63 |
allowed_source_types.remove('python')
|
| 63 | 64 |
if get_publisher().has_site_option('disable-jsonp-sources') and 'jsonp' in allowed_source_types:
|
| ... | ... | |
| 161 | 162 |
generic_options.append(('formula', _('Python Expression'), 'python'))
|
| 162 | 163 |
elif value.get('type') == 'formula':
|
| 163 | 164 |
generic_options.append(('formula', _('Python Expression (deprecated)'), 'python'))
|
| 165 |
if 'jsonvalue' in allowed_source_types: |
|
| 166 |
generic_options.append(('jsonvalue', _('JSON Expression'), 'jsonvalue'))
|
|
| 164 | 167 | |
| 165 | 168 |
if len(options) > 1 and generic_options: |
| 166 | 169 |
options.append(OptGroup(_('Generic Data Sources')))
|
| ... | ... | |
| 185 | 188 |
size=80, |
| 186 | 189 |
attrs={
|
| 187 | 190 |
'data-dynamic-display-child-of': 'data_source$type', |
| 188 |
'data-dynamic-display-value-in': 'json|jsonp|geojson|python', |
|
| 191 |
'data-dynamic-display-value-in': 'json|jsonp|geojson|python|jsonvalue',
|
|
| 189 | 192 |
}, |
| 190 | 193 |
) |
| 191 | 194 | |
| ... | ... | |
| 368 | 371 | |
| 369 | 372 |
return CardDef.get_data_source_items(data_source['type']) |
| 370 | 373 | |
| 371 |
if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula', 'wcs:users'):
|
|
| 374 |
if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue', 'wcs:users'):
|
|
| 372 | 375 |
# named data source |
| 373 | 376 |
named_data_source = NamedDataSource.get_by_slug(data_source['type']) |
| 374 | 377 |
if named_data_source.cache_duration: |
| ... | ... | |
| 385 | 388 |
include_disabled_users = data_source.get('include_disabled_users')
|
| 386 | 389 |
return [{'id': u.id, 'text': u.name} for u in users if u.is_active or include_disabled_users]
|
| 387 | 390 | |
| 391 |
if data_source.get('type') == 'jsonvalue':
|
|
| 392 |
try: |
|
| 393 |
value = json.loads(data_source.get('value'))
|
|
| 394 |
except json.JSONDecodeError: |
|
| 395 |
get_publisher().record_error( |
|
| 396 |
'JSON data source (%r) gave a non usable result' % data_source.get('value'),
|
|
| 397 |
context='[DATASOURCE]', |
|
| 398 |
notify=data_source.get('notify_on_errors'),
|
|
| 399 |
record=data_source.get('record_on_errors'),
|
|
| 400 |
) |
|
| 401 |
return [] |
|
| 402 |
try: |
|
| 403 |
if not isinstance(value, list): |
|
| 404 |
raise ValueError |
|
| 405 |
for item in value: |
|
| 406 |
if not isinstance(item, dict): |
|
| 407 |
raise ValueError |
|
| 408 |
if all(str(x.get('id', '')) and x.get('text') for x in value):
|
|
| 409 |
return value |
|
| 410 |
raise ValueError |
|
| 411 |
except ValueError: |
|
| 412 |
get_publisher().record_error( |
|
| 413 |
'JSON data source (%r) gave a non usable result' % data_source.get('value'),
|
|
| 414 |
context='[DATASOURCE]', |
|
| 415 |
notify=data_source.get('notify_on_errors'),
|
|
| 416 |
record=data_source.get('record_on_errors'),
|
|
| 417 |
) |
|
| 418 |
return [] |
|
| 388 | 419 |
if data_source.get('type') == 'formula':
|
| 389 | 420 |
# the result of a python expression, it must be a list. |
| 390 | 421 |
# - of strings |
| ... | ... | |
| 510 | 541 |
if not data_source: |
| 511 | 542 |
return None |
| 512 | 543 |
ds_type = data_source.get('type')
|
| 513 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
|
| 544 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue'):
|
|
| 514 | 545 |
return data_source |
| 515 | 546 |
if ds_type and ds_type.startswith('carddef:'):
|
| 516 | 547 |
return data_source |
| ... | ... | |
| 523 | 554 |
ds_type = data_source.get('type')
|
| 524 | 555 |
if ds_type is None: |
| 525 | 556 |
return None |
| 526 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
|
| 557 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula', 'jsonvalue'):
|
|
| 527 | 558 |
named_data_source = NamedDataSource() |
| 528 | 559 |
named_data_source.data_source = data_source |
| 529 | 560 |
return named_data_source |
| ... | ... | |
| 966 | 997 |
'jsonp': _('JSONP'),
|
| 967 | 998 |
'geojson': _('GeoJSON'),
|
| 968 | 999 |
'formula': _('Python Expression'),
|
| 1000 |
'jsonvalue': _('JSON Expression'),
|
|
| 969 | 1001 |
} |
| 970 | 1002 |
data_source_type = self.data_source.get('type')
|
| 971 | 1003 |
return data_source_labels.get(data_source_type) |
| ... | ... | |
| 1011 | 1043 | |
| 1012 | 1044 |
class StubNamedDataSource(NamedDataSource): |
| 1013 | 1045 |
type = 'formula' |
| 1014 |
data_source = {'type': 'formula', 'value': []}
|
|
| 1046 |
data_source = {'type': 'jsonvalue', 'value': []}
|
|
| 1015 | 1047 |
cache_duration = None |
| 1016 | 1048 | |
| 1017 | 1049 |
def __init__(self, name=None): |
| wcs/fields.py | ||
|---|---|---|
| 827 | 827 |
'jsonp': _('JSONP URL'),
|
| 828 | 828 |
'geojson': _('GeoJSON URL'),
|
| 829 | 829 |
'formula': _('Python Expression'),
|
| 830 |
'jsonvalue': _('JSON Expression'),
|
|
| 830 | 831 |
} |
| 831 | 832 |
if value.get('type') in data_source_types:
|
| 832 | 833 |
return '%s - %s' % (data_source_types[value.get('type')], value.get('value'))
|
| wcs/templates/wcs/backoffice/data-source.html | ||
|---|---|---|
| 38 | 38 |
<li>{% trans "URL:" %} <a href="{{ url }}">{{ datasource.data_source.value }}</a></li>
|
| 39 | 39 |
{% elif datasource.data_source.type == 'formula' %}
|
| 40 | 40 |
<li>{% trans "Python Expression:" %} {{ datasource.data_source.value }}</li>
|
| 41 |
{% elif datasource.data_source.type == 'jsonvalue' %}
|
|
| 42 |
<li>{% trans "JSON Expression:" %} {{ datasource.data_source.value }}</li>
|
|
| 41 | 43 |
{% elif datasource.data_source.type == 'wcs:users' %}
|
| 42 | 44 |
{% spaceless %}
|
| 43 | 45 |
<li>{% trans "Users with roles:" %}
|
| 44 |
- |
|