0002-datasources-add-geojson-support-42010.patch
tests/test_datasource.py | ||
---|---|---|
4 | 4 |
import pytest |
5 | 5 |
import os |
6 | 6 |
import json |
7 |
import sys |
|
8 | 7 |
import shutil |
9 | 8 | |
10 |
from django.utils import six |
|
11 | 9 |
from django.utils.six import StringIO |
12 | 10 |
from django.utils.six.moves.urllib import parse as urlparse |
13 | 11 | |
14 | 12 |
from quixote import cleanup |
15 |
from wcs import publisher |
|
16 | 13 |
from wcs.qommon.http_request import HTTPRequest |
17 |
from wcs.qommon.form import *
|
|
14 |
from wcs.qommon.form import get_request, Form
|
|
18 | 15 |
from wcs import fields, data_sources |
19 | 16 |
from wcs.data_sources import NamedDataSource, register_data_source_function |
20 | 17 | |
... | ... | |
140 | 137 |
('foo', 'Foo', 'foo', {'id': 'foo', 'text': 'Foo', 'value': '2017-01-01'})] |
141 | 138 | |
142 | 139 | |
143 |
def test_json_datasource(http_requests): |
|
140 |
def test_json_datasource(requests_pub, http_requests):
|
|
144 | 141 |
req = get_request() |
145 | 142 |
get_request().datasources_cache = {} |
146 | 143 |
datasource = {'type': 'json', 'value': ''} |
... | ... | |
317 | 314 |
assert 'invalid scheme in URL' in caplog.records[-1].message |
318 | 315 | |
319 | 316 | |
317 |
def test_geojson_datasource(requests_pub, http_requests): |
|
318 |
get_request() |
|
319 |
get_request().datasources_cache = {} |
|
320 |
datasource = {'type': 'geojson', 'value': ''} |
|
321 |
assert data_sources.get_items(datasource) == [] |
|
322 | ||
323 |
# missing file |
|
324 |
get_request().datasources_cache = {} |
|
325 |
geojson_file_path = os.path.join(pub.app_dir, 'test.geojson') |
|
326 |
datasource = {'type': 'geojson', 'value': 'file://%s' % geojson_file_path} |
|
327 |
assert data_sources.get_items(datasource) == [] |
|
328 | ||
329 |
# invalid geojson file |
|
330 |
get_request().datasources_cache = {} |
|
331 |
geojson_file = open(geojson_file_path, 'wb') |
|
332 |
geojson_file.write(codecs.encode(b'foobar', 'zlib_codec')) |
|
333 |
geojson_file.close() |
|
334 |
assert data_sources.get_items(datasource) == [] |
|
335 | ||
336 |
# empty geojson file |
|
337 |
get_request().datasources_cache = {} |
|
338 |
geojson_file = open(geojson_file_path, 'w') |
|
339 |
json.dump({}, geojson_file) |
|
340 |
geojson_file.close() |
|
341 |
assert data_sources.get_items(datasource) == [] |
|
342 | ||
343 |
# unrelated geojson file |
|
344 |
get_request().datasources_cache = {} |
|
345 |
geojson_file = open(geojson_file_path, 'w') |
|
346 |
json.dump('foobar', geojson_file) |
|
347 |
geojson_file.close() |
|
348 |
assert data_sources.get_items(datasource) == [] |
|
349 | ||
350 |
# another unrelated geojson file |
|
351 |
get_request().datasources_cache = {} |
|
352 |
geojson_file = open(geojson_file_path, 'w') |
|
353 |
json.dump({'features': 'foobar'}, geojson_file) |
|
354 |
geojson_file.close() |
|
355 |
assert data_sources.get_items(datasource) == [] |
|
356 | ||
357 |
# a good geojson file |
|
358 |
get_request().datasources_cache = {} |
|
359 |
geojson_file = open(geojson_file_path, 'w') |
|
360 |
json.dump({'features': [ |
|
361 |
{'properties': {'id': '1', 'text': 'foo'}}, |
|
362 |
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file) |
|
363 |
geojson_file.close() |
|
364 |
assert data_sources.get_items(datasource) == [ |
|
365 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo'}}), |
|
366 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar'}})] |
|
367 |
assert data_sources.get_structured_items(datasource) == [ |
|
368 |
{'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo'}}, |
|
369 |
{'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar'}}] |
|
370 | ||
371 |
# a geojson file with additional keys |
|
372 |
get_request().datasources_cache = {} |
|
373 |
geojson_file = open(geojson_file_path, 'w') |
|
374 |
json.dump({'features': [ |
|
375 |
{'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}, |
|
376 |
{'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}}]}, geojson_file) |
|
377 |
geojson_file.close() |
|
378 |
assert data_sources.get_items(datasource) == [ |
|
379 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}), |
|
380 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})] |
|
381 |
assert data_sources.get_structured_items(datasource) == [ |
|
382 |
{'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}, |
|
383 |
{'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}}] |
|
384 | ||
385 |
# geojson specified with a variadic url |
|
386 |
get_request().datasources_cache = {} |
|
387 | ||
388 |
class GeoJSONUrlPath(object): |
|
389 |
def get_substitution_variables(self): |
|
390 |
return {'geojson_url': 'file://%s' % geojson_file_path} |
|
391 | ||
392 |
pub.substitutions.feed(GeoJSONUrlPath()) |
|
393 |
datasource = {'type': 'geojson', 'value': '[geojson_url]'} |
|
394 |
assert data_sources.get_items(datasource) == [ |
|
395 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}), |
|
396 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})] |
|
397 | ||
398 |
# same with django templated url |
|
399 |
get_request().datasources_cache = {} |
|
400 | ||
401 |
class GeoJSONUrlPath(object): |
|
402 |
def get_substitution_variables(self): |
|
403 |
return {'geojson_url': 'file://%s' % geojson_file_path} |
|
404 | ||
405 |
pub.substitutions.feed(GeoJSONUrlPath()) |
|
406 |
datasource = {'type': 'geojson', 'value': '{{ geojson_url }}'} |
|
407 |
assert data_sources.get_items(datasource) == [ |
|
408 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}), |
|
409 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})] |
|
410 | ||
411 |
# geojson specified with a variadic url with an erroneous space |
|
412 |
get_request().datasources_cache = {} |
|
413 | ||
414 |
class GeoJSONUrlPath(object): |
|
415 |
def get_substitution_variables(self): |
|
416 |
return {'geojson_url': 'file://%s' % geojson_file_path} |
|
417 | ||
418 |
pub.substitutions.feed(GeoJSONUrlPath()) |
|
419 |
datasource = {'type': 'geojson', 'value': ' [geojson_url]'} |
|
420 |
assert data_sources.get_items(datasource) == [ |
|
421 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}), |
|
422 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})] |
|
423 | ||
424 |
# same with django templated url |
|
425 |
get_request().datasources_cache = {} |
|
426 | ||
427 |
class GeoJSONUrlPath(object): |
|
428 |
def get_substitution_variables(self): |
|
429 |
return {'geojson_url': 'file://%s' % geojson_file_path} |
|
430 | ||
431 |
pub.substitutions.feed(GeoJSONUrlPath()) |
|
432 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}'} |
|
433 |
assert data_sources.get_items(datasource) == [ |
|
434 |
('1', 'foo', '1', {'id': '1', 'text': 'foo', 'properties': {'id': '1', 'text': 'foo', 'more': 'xxx'}}), |
|
435 |
('2', 'bar', '2', {'id': '2', 'text': 'bar', 'properties': {'id': '2', 'text': 'bar', 'more': 'yyy'}})] |
|
436 | ||
437 |
# a geojson file with integer as 'id' |
|
438 |
get_request().datasources_cache = {} |
|
439 |
geojson_file = open(geojson_file_path, 'w') |
|
440 |
json.dump({'features': [ |
|
441 |
{'properties': {'id': 1, 'text': 'foo'}}, |
|
442 |
{'properties': {'id': 2, 'text': 'bar'}}]}, geojson_file) |
|
443 |
geojson_file.close() |
|
444 |
assert data_sources.get_items(datasource) == [ |
|
445 |
('1', 'foo', '1', {'id': 1, 'text': 'foo', 'properties': {'id': 1, 'text': 'foo'}}), |
|
446 |
('2', 'bar', '2', {'id': 2, 'text': 'bar', 'properties': {'id': 2, 'text': 'bar'}})] |
|
447 |
assert data_sources.get_structured_items(datasource) == [ |
|
448 |
{'id': 1, 'text': 'foo', 'properties': {'id': 1, 'text': 'foo'}}, |
|
449 |
{'id': 2, 'text': 'bar', 'properties': {'id': 2, 'text': 'bar'}}] |
|
450 | ||
451 |
# a geojson file with empty or no text values |
|
452 |
get_request().datasources_cache = {} |
|
453 |
geojson_file = open(geojson_file_path, 'w') |
|
454 |
json.dump({'features': [ |
|
455 |
{'properties': {'id': '1', 'text': ''}}, |
|
456 |
{'properties': {'id': '2'}}]}, geojson_file) |
|
457 |
geojson_file.close() |
|
458 |
assert data_sources.get_items(datasource) == [ |
|
459 |
('1', '1', '1', {'id': '1', 'text': '1', 'properties': {'id': '1', 'text': ''}}), |
|
460 |
('2', '2', '2', {'id': '2', 'text': '2', 'properties': {'id': '2'}})] |
|
461 |
assert data_sources.get_structured_items(datasource) == [ |
|
462 |
{'id': '1', 'text': '1', 'properties': {'id': '1', 'text': ''}}, |
|
463 |
{'id': '2', 'text': '2', 'properties': {'id': '2'}}] |
|
464 | ||
465 |
# a geojson file with empty or no id |
|
466 |
get_request().datasources_cache = {} |
|
467 |
geojson_file = open(geojson_file_path, 'w') |
|
468 |
json.dump({'features': [ |
|
469 |
{'properties': {'id': '', 'text': 'foo'}}, |
|
470 |
{'properties': {'text': 'bar'}}, |
|
471 |
{'properties': {'id': None}}]}, geojson_file) |
|
472 |
geojson_file.close() |
|
473 |
assert data_sources.get_items(datasource) == [] |
|
474 |
assert data_sources.get_structured_items(datasource) == [] |
|
475 | ||
476 |
# specify id_property |
|
477 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'id_property': 'gid'} |
|
478 |
get_request().datasources_cache = {} |
|
479 |
geojson_file = open(geojson_file_path, 'w') |
|
480 |
json.dump({'features': [ |
|
481 |
{'properties': {'gid': '1', 'text': 'foo'}}, |
|
482 |
{'properties': {'gid': '2', 'text': 'bar'}}]}, geojson_file) |
|
483 |
geojson_file.close() |
|
484 |
assert data_sources.get_structured_items(datasource) == [ |
|
485 |
{'id': '1', 'text': 'foo', 'properties': {'gid': '1', 'text': 'foo'}}, |
|
486 |
{'id': '2', 'text': 'bar', 'properties': {'gid': '2', 'text': 'bar'}}] |
|
487 | ||
488 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'id_property': 'id'} |
|
489 |
get_request().datasources_cache = {} |
|
490 |
assert data_sources.get_structured_items(datasource) == [] |
|
491 | ||
492 |
# specify label_template_property |
|
493 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ id }}: {{ text }}'} |
|
494 |
get_request().datasources_cache = {} |
|
495 |
geojson_file = open(geojson_file_path, 'w') |
|
496 |
json.dump({'features': [ |
|
497 |
{'properties': {'id': '1', 'text': 'foo'}}, |
|
498 |
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file) |
|
499 |
geojson_file.close() |
|
500 |
assert data_sources.get_structured_items(datasource) == [ |
|
501 |
{'id': '1', 'text': '1: foo', 'properties': {'id': '1', 'text': 'foo'}}, |
|
502 |
{'id': '2', 'text': '2: bar', 'properties': {'id': '2', 'text': 'bar'}}] |
|
503 | ||
504 |
# wrong template |
|
505 |
get_request().datasources_cache = {} |
|
506 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ text }'} |
|
507 |
assert data_sources.get_structured_items(datasource) == [ |
|
508 |
{'id': '1', 'text': '{{ text }', 'properties': {'id': '1', 'text': 'foo'}}, |
|
509 |
{'id': '2', 'text': '{{ text }', 'properties': {'id': '2', 'text': 'bar'}}] |
|
510 |
get_request().datasources_cache = {} |
|
511 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': 'text'} |
|
512 |
assert data_sources.get_structured_items(datasource) == [ |
|
513 |
{'id': '1', 'text': 'text', 'properties': {'id': '1', 'text': 'foo'}}, |
|
514 |
{'id': '2', 'text': 'text', 'properties': {'id': '2', 'text': 'bar'}}] |
|
515 | ||
516 |
# unknown property or empty value |
|
517 |
datasource = {'type': 'geojson', 'value': ' {{ geojson_url }}', 'label_template_property': '{{ label }}'} |
|
518 |
get_request().datasources_cache = {} |
|
519 |
geojson_file = open(geojson_file_path, 'w') |
|
520 |
json.dump({'features': [ |
|
521 |
{'properties': {'id': '1', 'text': 'foo', 'label': ''}}, |
|
522 |
{'properties': {'id': '2', 'text': 'bar'}}]}, geojson_file) |
|
523 |
geojson_file.close() |
|
524 |
assert data_sources.get_structured_items(datasource) == [ |
|
525 |
{'id': '1', 'text': '1', 'properties': {'id': '1', 'text': 'foo', 'label': ''}}, |
|
526 |
{'id': '2', 'text': '2', 'properties': {'id': '2', 'text': 'bar'}}] |
|
527 | ||
528 | ||
529 |
def test_geojson_datasource_bad_url(http_requests, caplog): |
|
530 |
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/404'} |
|
531 |
assert data_sources.get_items(datasource) == [] |
|
532 |
assert 'Error loading GeoJSON data source' in caplog.records[-1].message |
|
533 |
assert 'status: 404' in caplog.records[-1].message |
|
534 | ||
535 |
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/xml'} |
|
536 |
assert data_sources.get_items(datasource) == [] |
|
537 |
assert 'Error reading GeoJSON data source output' in caplog.records[-1].message |
|
538 |
assert 'Expecting value:' in caplog.records[-1].message |
|
539 | ||
540 |
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/connection-error'} |
|
541 |
assert data_sources.get_items(datasource) == [] |
|
542 |
assert 'Error loading GeoJSON data source' in caplog.records[-1].message |
|
543 |
assert 'error' in caplog.records[-1].message |
|
544 | ||
545 |
datasource = {'type': 'geojson', 'value': 'http://remote.example.net/json-list-err1'} |
|
546 |
assert data_sources.get_items(datasource) == [] |
|
547 |
assert 'Error reading GeoJSON data source output (err 1)' in caplog.records[-1].message |
|
548 | ||
549 | ||
550 |
def test_geojson_datasource_bad_url_scheme(caplog): |
|
551 |
datasource = {'type': 'geojson', 'value': ''} |
|
552 |
assert data_sources.get_items(datasource) == [] |
|
553 |
assert caplog.records[-1].message == 'Empty URL in GeoJSON data source' |
|
554 | ||
555 |
datasource = {'type': 'geojson', 'value': 'foo://bar'} |
|
556 |
assert data_sources.get_items(datasource) == [] |
|
557 |
assert 'Error loading GeoJSON data source' in caplog.records[-1].message |
|
558 |
assert 'invalid scheme in URL' in caplog.records[-1].message |
|
559 | ||
560 |
datasource = {'type': 'geojson', 'value': '/bla/blo'} |
|
561 |
assert data_sources.get_items(datasource) == [] |
|
562 |
assert 'Error loading GeoJSON data source' in caplog.records[-1].message |
|
563 |
assert 'invalid scheme in URL' in caplog.records[-1].message |
|
564 | ||
565 | ||
320 | 566 |
def test_item_field_named_python_datasource(): |
321 | 567 |
NamedDataSource.wipe() |
322 | 568 |
data_source = NamedDataSource(name='foobar') |
tests/test_datasources_admin_pages.py | ||
---|---|---|
160 | 160 |
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id) |
161 | 161 |
assert '<a href="http://example.net/foo/bar"' in resp.text |
162 | 162 | |
163 |
# check geojson |
|
164 |
geojson_file_path = os.path.join(pub.app_dir, 'test.geojson') |
|
165 |
geojson_file = open(geojson_file_path, 'w') |
|
166 |
json.dump({'features': [ |
|
167 |
{'properties': {'id': '1', 'text': 'foo', 'label': 'foo'}}, |
|
168 |
{'properties': {'id': '2', 'text': 'bar', 'label': 'bar'}}]}, geojson_file) |
|
169 |
geojson_file.close() |
|
170 |
data_source.data_source = {'type': 'geojson', 'value': 'file://%s' % geojson_file_path} |
|
171 |
data_source.store() |
|
172 |
with HttpRequestsMocking(): |
|
173 |
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id) |
|
174 |
assert 'Preview' in resp.text |
|
175 |
assert 'foo' in resp.text |
|
176 |
assert 'bar' in resp.text |
|
177 |
assert 'Additional keys are available: label' in resp.text |
|
178 | ||
163 | 179 |
data_source.data_source = {'type': 'formula', 'value': '[str(x) for x in range(100)]'} |
164 | 180 |
data_source.store() |
165 | 181 |
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id) |
wcs/admin/data_sources.py | ||
---|---|---|
70 | 70 |
advanced=False, |
71 | 71 |
attrs={ |
72 | 72 |
'data-dynamic-display-child-of': 'data_source$type', |
73 |
'data-dynamic-display-value': 'json',
|
|
73 |
'data-dynamic-display-value-in': 'json|geojson',
|
|
74 | 74 |
}) |
75 | 75 |
form.add(StringWidget, 'query_parameter', |
76 | 76 |
value=self.datasource.query_parameter, |
... | ... | |
92 | 92 |
'data-dynamic-display-child-of': 'data_source$type', |
93 | 93 |
'data-dynamic-display-value': 'json', |
94 | 94 |
}) |
95 |
form.add(StringWidget, 'id_property', |
|
96 |
value=self.datasource.id_property, |
|
97 |
title=_('Id Property'), |
|
98 |
hint=_('Name of the property to use to get a given entry from data source (default: id)'), |
|
99 |
required=False, |
|
100 |
advanced=False, |
|
101 |
attrs={ |
|
102 |
'data-dynamic-display-child-of': 'data_source$type', |
|
103 |
'data-dynamic-display-value': 'geojson', |
|
104 |
}) |
|
105 |
form.add(StringWidget, 'label_template_property', |
|
106 |
value=self.datasource.label_template_property, |
|
107 |
title=_('Label template'), |
|
108 |
hint=_('Django expression to build label of each value (default: {{ text }})'), |
|
109 |
required=False, |
|
110 |
advanced=False, |
|
111 |
size=80, |
|
112 |
attrs={ |
|
113 |
'data-dynamic-display-child-of': 'data_source$type', |
|
114 |
'data-dynamic-display-value': 'geojson', |
|
115 |
}) |
|
95 | 116 |
if self.datasource.slug and not self.is_used(): |
96 | 117 |
form.add(StringWidget, 'slug', |
97 | 118 |
value=self.datasource.slug, |
... | ... | |
124 | 145 |
self.datasource.cache_duration = form.get_widget('cache_duration').parse() |
125 | 146 |
self.datasource.query_parameter = form.get_widget('query_parameter').parse() |
126 | 147 |
self.datasource.id_parameter = form.get_widget('id_parameter').parse() |
148 |
self.datasource.id_property = form.get_widget('id_property').parse() |
|
149 |
self.datasource.label_template_property = form.get_widget('label_template_property').parse() |
|
127 | 150 |
if slug_widget: |
128 | 151 |
self.datasource.slug = slug |
129 | 152 |
self.datasource.store() |
... | ... | |
156 | 179 |
return formdefs |
157 | 180 | |
158 | 181 |
def preview_block(self): |
159 |
if self.datasource.data_source.get('type') not in ('json', 'formula'): |
|
182 |
data_source = self.datasource.extended_data_source |
|
183 |
if data_source.get('type') not in ('json', 'geojson', 'formula'): |
|
160 | 184 |
return '' |
161 |
items = get_structured_items(self.datasource.data_source)
|
|
185 |
items = get_structured_items(data_source) |
|
162 | 186 |
if not items: |
163 | 187 |
return '' |
164 | 188 |
r = TemplateIO(html=True) |
... | ... | |
175 | 199 |
else: |
176 | 200 |
r += htmltext('<li><tt>%s</tt>: %s</li>') % ( |
177 | 201 |
item.get('id'), item.get('text')) |
178 |
additional_keys |= set(item.keys()) |
|
202 |
if data_source.get('type') == 'geojson': |
|
203 |
additional_keys |= set(item.get('properties', {}).keys()) |
|
204 |
else: |
|
205 |
additional_keys |= set(item.keys()) |
|
179 | 206 |
if len(items) > 10: |
180 | 207 |
r += htmltext('<li>...</li>') |
181 | 208 |
r += htmltext('</ul>') |
wcs/data_sources.py | ||
---|---|---|
18 | 18 |
import hashlib |
19 | 19 |
import xml.etree.ElementTree as ET |
20 | 20 | |
21 |
from django.template import TemplateSyntaxError, VariableDoesNotExist |
|
21 | 22 |
from django.utils import six |
22 | 23 |
from django.utils.encoding import force_text, force_bytes |
23 | 24 |
from django.utils.six.moves.urllib import parse as urllib |
... | ... | |
71 | 72 |
options.append(('json', _('JSON URL'), 'json')) |
72 | 73 |
if allow_jsonp: |
73 | 74 |
options.append(('jsonp', _('JSONP URL'), 'jsonp')) |
75 |
options.append(('geojson', _('GeoJSON URL'), 'geojson')) |
|
74 | 76 |
options.append(('formula', _('Python Expression'), 'python')) |
75 | 77 | |
76 | 78 |
self.add(SingleSelectWidget, 'type', options=options, value=value.get('type'), |
... | ... | |
82 | 84 | |
83 | 85 |
self.add(StringWidget, 'value', value=value.get('value'), size=80, |
84 | 86 |
attrs={'data-dynamic-display-child-of': 'data_source$type', |
85 |
'data-dynamic-display-value-in': 'json|jsonp|python'}) |
|
87 |
'data-dynamic-display-value-in': 'json|jsonp|geojson|python'})
|
|
86 | 88 | |
87 | 89 |
self._parsed = False |
88 | 90 | |
... | ... | |
116 | 118 |
return tupled_items |
117 | 119 | |
118 | 120 | |
119 |
def request_json_items(url): |
|
121 |
def request_json_items(url, data_source):
|
|
120 | 122 |
url = sign_url_auto_orig(url) |
123 |
geojson = data_source.get('type') == 'geojson' |
|
121 | 124 |
try: |
122 | 125 |
entries = misc.json_loads(misc.urlopen(url).read()) |
123 | 126 |
if not isinstance(entries, dict): |
124 | 127 |
raise ValueError('not a json dict') |
125 | 128 |
if entries.get('err') not in (None, 0, "0"): |
126 | 129 |
raise ValueError('err %s' % entries['err']) |
127 |
if not isinstance(entries.get('data'), list): |
|
130 |
if not geojson and not isinstance(entries.get('data'), list):
|
|
128 | 131 |
raise ValueError('not a json dict with a data list attribute') |
132 |
if geojson and not isinstance(entries.get('features'), list): |
|
133 |
raise ValueError('bad geojson format') |
|
129 | 134 |
except misc.ConnectionError as e: |
130 |
get_logger().warn('Error loading JSON data source (%s)' % str(e)) |
|
135 |
if geojson: |
|
136 |
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e)) |
|
137 |
else: |
|
138 |
get_logger().warning('Error loading JSON data source (%s)' % str(e)) |
|
131 | 139 |
return None |
132 | 140 |
except (ValueError, TypeError) as e: |
133 |
get_logger().warn('Error reading JSON data source output (%s)' % str(e)) |
|
141 |
if geojson: |
|
142 |
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e)) |
|
143 |
else: |
|
144 |
get_logger().warning('Error reading JSON data source output (%s)' % str(e)) |
|
134 | 145 |
return None |
135 | 146 |
items = [] |
136 |
for item in entries.get('data'): |
|
137 |
# skip malformed items |
|
138 |
if item.get('id') is None or item.get('id') == '': |
|
139 |
continue |
|
140 |
if 'text' not in item: |
|
141 |
item['text'] = item['id'] |
|
142 |
items.append(item) |
|
147 |
if geojson: |
|
148 |
id_property = data_source.get('id_property') |
|
149 |
for item in entries.get('features'): |
|
150 |
if not item.get('properties', {}).get(id_property): |
|
151 |
continue |
|
152 |
item['id'] = item['properties'][id_property] |
|
153 |
try: |
|
154 |
item['text'] = Template(data_source.get('label_template_property')).render(item['properties']) |
|
155 |
except (TemplateSyntaxError, VariableDoesNotExist): |
|
156 |
pass |
|
157 |
if not item.get('text'): |
|
158 |
item['text'] = item['id'] |
|
159 |
items.append(item) |
|
160 |
else: |
|
161 |
for item in entries.get('data'): |
|
162 |
# skip malformed items |
|
163 |
if item.get('id') is None or item.get('id') == '': |
|
164 |
continue |
|
165 |
if 'text' not in item: |
|
166 |
item['text'] = item['id'] |
|
167 |
items.append(item) |
|
143 | 168 |
return items |
144 | 169 | |
145 | 170 | |
... | ... | |
159 | 184 |
items.sort(key=lambda x: misc.simplify(x['text'])) |
160 | 185 |
return items |
161 | 186 | |
162 |
if data_source.get('type') not in ('json', 'jsonp', 'formula'): |
|
187 |
if data_source.get('type') not in ('json', 'jsonp', 'geojson', 'formula'):
|
|
163 | 188 |
# named data source |
164 | 189 |
named_data_source = NamedDataSource.get_by_slug(data_source['type']) |
165 | 190 |
if named_data_source.cache_duration: |
166 | 191 |
cache_duration = int(named_data_source.cache_duration) |
167 |
data_source = named_data_source.data_source |
|
192 |
data_source = named_data_source.extended_data_source
|
|
168 | 193 | |
169 | 194 |
if data_source.get('type') == 'formula': |
170 | 195 |
# the result of a python expression, it must be a list. |
... | ... | |
181 | 206 |
try: |
182 | 207 |
value = eval(data_source.get('value'), global_eval_dict, variables) |
183 | 208 |
if not isinstance(value, collections.Iterable): |
184 |
get_logger().warn('Python data source (%r) gave a non-iterable result' % \
|
|
185 |
data_source.get('value')) |
|
209 |
get_logger().warn('Python data source (%r) gave a non-iterable result' % |
|
210 |
data_source.get('value'))
|
|
186 | 211 |
return [] |
187 | 212 |
if len(value) == 0: |
188 | 213 |
return [] |
... | ... | |
201 | 226 |
except: |
202 | 227 |
get_logger().warn('Failed to eval() Python data source (%r)' % data_source.get('value')) |
203 | 228 |
return [] |
204 |
elif data_source.get('type') == 'json':
|
|
229 |
elif data_source.get('type') in ['json', 'geojson']:
|
|
205 | 230 |
# the content available at a json URL, it must answer with a dict with |
206 | 231 |
# a 'data' key holding the list of items, each of them being a dict |
207 | 232 |
# with at least both an "id" and a "text" key. |
233 |
geojson = data_source.get('type') == 'geojson' |
|
208 | 234 |
url = data_source.get('value') |
209 | 235 |
if not url: |
210 |
get_logger().warn('Empty URL in JSON data source') |
|
236 |
if geojson: |
|
237 |
get_logger().warning('Empty URL in GeoJSON data source') |
|
238 |
else: |
|
239 |
get_logger().warning('Empty URL in JSON data source') |
|
211 | 240 |
return [] |
212 | 241 |
url = url.strip() |
213 | 242 |
if Template.is_template_string(url): |
... | ... | |
225 | 254 |
if items is not None: |
226 | 255 |
return items |
227 | 256 | |
228 |
items = request_json_items(url) |
|
257 |
items = request_json_items(url, data_source)
|
|
229 | 258 |
if items is None: |
230 | 259 |
return [] |
231 | 260 |
if hasattr(request, 'datasources_cache'): |
... | ... | |
240 | 269 |
if not data_source: |
241 | 270 |
return None |
242 | 271 |
ds_type = data_source.get('type') |
243 |
if ds_type in ('json', 'jsonp', 'formula'): |
|
272 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
|
244 | 273 |
return data_source |
245 | 274 |
if ds_type and ds_type.startswith('carddef:'): |
246 | 275 |
return data_source |
... | ... | |
251 | 280 |
if not data_source: |
252 | 281 |
return None |
253 | 282 |
ds_type = data_source.get('type') |
254 |
if ds_type in ('json', 'jsonp', 'formula'): |
|
283 |
if ds_type in ('json', 'jsonp', 'geojson', 'formula'):
|
|
255 | 284 |
named_data_source = NamedDataSource() |
256 | 285 |
named_data_source.data_source = data_source |
257 | 286 |
return named_data_source |
... | ... | |
274 | 303 |
cache_duration = None |
275 | 304 |
query_parameter = None |
276 | 305 |
id_parameter = None |
306 |
id_property = None |
|
307 |
label_template_property = None |
|
277 | 308 | |
278 | 309 |
# declarations for serialization |
279 | 310 |
XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'), |
280 | 311 |
('cache_duration', 'str'), |
281 | 312 |
('query_parameter', 'str'), |
282 | 313 |
('id_parameter', 'str'), |
314 |
('id_property', 'str'), |
|
315 |
('label_template_property', 'str'), |
|
283 | 316 |
('data_source', 'data_source'), |
284 | 317 |
] |
285 | 318 | |
... | ... | |
291 | 324 |
def type(self): |
292 | 325 |
return self.data_source.get('type') |
293 | 326 | |
327 |
@property |
|
328 |
def extended_data_source(self): |
|
329 |
if self.type != 'geojson': |
|
330 |
return self.data_source |
|
331 |
data_source = self.data_source.copy() |
|
332 |
data_source.update({ |
|
333 |
'id_property': self.id_property or 'id', |
|
334 |
'label_template_property': self.label_template_property or '{{ text }}', |
|
335 |
}) |
|
336 |
return data_source |
|
337 | ||
294 | 338 |
def can_jsonp(self): |
295 | 339 |
if self.type == 'jsonp': |
296 | 340 |
return True |
... | ... | |
387 | 431 |
return None |
388 | 432 |
return items[0] |
389 | 433 | |
390 |
items = request_json_items(url) |
|
434 |
items = request_json_items(url, self.data_source)
|
|
391 | 435 |
if not items: # None or empty list are not valid |
392 | 436 |
return None |
393 | 437 |
if hasattr(request, 'datasources_cache'): |
... | ... | |
428 | 472 |
data_source_labels = { |
429 | 473 |
'json': _('JSON'), |
430 | 474 |
'jsonp': _('JSONP'), |
475 |
'geojson': _('GeoJSON'), |
|
431 | 476 |
'formula': _('Python Expression'), |
432 | 477 |
} |
433 | 478 |
data_source_type = self.data_source.get('type') |
434 |
- |