0001-datasources-json-datasource-attributes-47218.patch
| tests/test_datasource.py | ||
|---|---|---|
| 283 | 283 |
assert data_sources.get_items(datasource) == [] |
| 284 | 284 |
assert data_sources.get_structured_items(datasource) == [] |
| 285 | 285 | |
| 286 |
# specify data_attribute |
|
| 287 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'results'}
|
|
| 288 |
get_request().datasources_cache = {}
|
|
| 289 |
json_file = open(json_file_path, 'w') |
|
| 290 |
json.dump({'results': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file)
|
|
| 291 |
json_file.close() |
|
| 292 |
assert data_sources.get_structured_items(datasource) == [ |
|
| 293 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
|
| 294 | ||
| 295 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'data_attribute': 'data'}
|
|
| 296 |
get_request().datasources_cache = {}
|
|
| 297 |
assert data_sources.get_structured_items(datasource) == [] |
|
| 298 | ||
| 299 |
# specify id_attribute |
|
| 300 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'pk'}
|
|
| 301 |
get_request().datasources_cache = {}
|
|
| 302 |
json_file = open(json_file_path, 'w') |
|
| 303 |
json.dump({'data': [{'pk': '1', 'text': 'foo'}, {'pk': '2', 'text': 'bar'}]}, json_file)
|
|
| 304 |
json_file.close() |
|
| 305 |
assert data_sources.get_structured_items(datasource) == [ |
|
| 306 |
{'id': '1', 'text': 'foo', 'pk': '1'}, {'id': '2', 'text': 'bar', 'pk': '2'}]
|
|
| 307 | ||
| 308 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'id_attribute': 'id'}
|
|
| 309 |
get_request().datasources_cache = {}
|
|
| 310 |
assert data_sources.get_structured_items(datasource) == [] |
|
| 311 | ||
| 312 |
# specify text_attribute |
|
| 313 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'label'}
|
|
| 314 |
get_request().datasources_cache = {}
|
|
| 315 |
json_file = open(json_file_path, 'w') |
|
| 316 |
json.dump({'data': [{'id': '1', 'label': 'foo'}, {'id': '2', 'label': 'bar'}]}, json_file)
|
|
| 317 |
json_file.close() |
|
| 318 |
assert data_sources.get_structured_items(datasource) == [ |
|
| 319 |
{'id': '1', 'text': 'foo', 'label': 'foo'}, {'id': '2', 'text': 'bar', 'label': 'bar'}]
|
|
| 320 | ||
| 321 |
datasource = {'type': 'json', 'value': ' {{ json_url }}', 'text_attribute': 'text'}
|
|
| 322 |
get_request().datasources_cache = {}
|
|
| 323 |
assert data_sources.get_structured_items(datasource) == [ |
|
| 324 |
{'id': '1', 'text': '1', 'label': 'foo'}, {'id': '2', 'text': '2', 'label': 'bar'}]
|
|
| 325 | ||
| 286 | 326 | |
| 287 | 327 |
def test_json_datasource_bad_url(http_requests, caplog): |
| 288 | 328 |
datasource = {'type': 'json', 'value': 'http://remote.example.net/404'}
|
| tests/test_datasources_admin_pages.py | ||
|---|---|---|
| 161 | 161 |
assert 'Preview' in resp.text |
| 162 | 162 |
assert 'foo' in resp.text |
| 163 | 163 | |
| 164 |
# with other attributes |
|
| 165 |
json_file = open(json_file_path, 'w') |
|
| 166 |
json.dump({'results': [{'pk': '1', 'label': 'foo'}, {'pk': '2'}]}, json_file)
|
|
| 167 |
json_file.close() |
|
| 168 | ||
| 169 |
data_source.data_attribute = 'results' |
|
| 170 |
data_source.id_attribute = 'pk' |
|
| 171 |
data_source.text_attribute = 'label' |
|
| 172 |
data_source.store() |
|
| 173 |
with HttpRequestsMocking(): |
|
| 174 |
resp = app.get('/backoffice/settings/data-sources/%s/' % data_source.id)
|
|
| 175 |
assert 'Preview' in resp.text |
|
| 176 |
assert '<tt>1</tt>: foo</li>' in resp.text |
|
| 177 |
assert '<tt>2</tt>: 2</li>' in resp.text |
|
| 178 |
assert '<p>Additional keys are available: label, pk</p>' in resp.text |
|
| 179 | ||
| 164 | 180 |
# variadic url |
| 181 |
data_source.data_attribute = None |
|
| 165 | 182 |
data_source.data_source = {'type': 'json', 'value': '{{ site_url }}/foo/bar'}
|
| 166 | 183 |
data_source.store() |
| 167 | 184 |
with HttpRequestsMocking(): |
| wcs/admin/data_sources.py | ||
|---|---|---|
| 104 | 104 |
'data-dynamic-display-child-of': 'data_source$type', |
| 105 | 105 |
'data-dynamic-display-value': 'geojson', |
| 106 | 106 |
}) |
| 107 |
form.add(StringWidget, 'data_attribute', |
|
| 108 |
value=self.datasource.data_attribute, |
|
| 109 |
title=_('Data Attribute'),
|
|
| 110 |
hint=_('Name of the attribute containing the list of results (default: data)'),
|
|
| 111 |
required=False, |
|
| 112 |
advanced=False, |
|
| 113 |
attrs={
|
|
| 114 |
'data-dynamic-display-child-of': 'data_source$type', |
|
| 115 |
'data-dynamic-display-value': 'json', |
|
| 116 |
}) |
|
| 117 |
form.add(StringWidget, 'id_attribute', |
|
| 118 |
value=self.datasource.id_attribute, |
|
| 119 |
title=_('Id Attribute'),
|
|
| 120 |
hint=_('Name of the attribute containing the identifier of an entry (default: id)'),
|
|
| 121 |
required=False, |
|
| 122 |
advanced=False, |
|
| 123 |
attrs={
|
|
| 124 |
'data-dynamic-display-child-of': 'data_source$type', |
|
| 125 |
'data-dynamic-display-value-in': 'json|geojson', |
|
| 126 |
}) |
|
| 127 |
form.add(StringWidget, 'text_attribute', |
|
| 128 |
value=self.datasource.text_attribute, |
|
| 129 |
title=_('Text Attribute'),
|
|
| 130 |
hint=_('Name of the attribute containing the label of an entry (default: text)'),
|
|
| 131 |
required=False, |
|
| 132 |
advanced=False, |
|
| 133 |
attrs={
|
|
| 134 |
'data-dynamic-display-child-of': 'data_source$type', |
|
| 135 |
'data-dynamic-display-value': 'json', |
|
| 136 |
}) |
|
| 107 | 137 |
form.add(StringWidget, 'label_template_property', |
| 108 | 138 |
value=self.datasource.label_template_property, |
| 109 | 139 |
title=_('Label template'),
|
| ... | ... | |
| 148 | 178 |
self.datasource.cache_duration = form.get_widget('cache_duration').parse()
|
| 149 | 179 |
self.datasource.query_parameter = form.get_widget('query_parameter').parse()
|
| 150 | 180 |
self.datasource.id_parameter = form.get_widget('id_parameter').parse()
|
| 181 |
self.datasource.data_attribute = form.get_widget('data_attribute').parse()
|
|
| 182 |
self.datasource.id_attribute = form.get_widget('id_attribute').parse()
|
|
| 183 |
self.datasource.text_attribute = form.get_widget('text_attribute').parse()
|
|
| 151 | 184 |
self.datasource.id_property = form.get_widget('id_property').parse()
|
| 152 | 185 |
self.datasource.label_template_property = form.get_widget('label_template_property').parse()
|
| 153 | 186 |
if slug_widget: |
| wcs/data_sources.py | ||
|---|---|---|
| 118 | 118 | |
| 119 | 119 |
def request_json_items(url, data_source): |
| 120 | 120 |
url = sign_url_auto_orig(url) |
| 121 |
geojson = data_source.get('type') == 'geojson'
|
|
| 121 |
data_key = data_source.get('data_attribute') or 'data'
|
|
| 122 | 122 |
try: |
| 123 | 123 |
entries = misc.json_loads(misc.urlopen(url).read()) |
| 124 | 124 |
if not isinstance(entries, dict): |
| 125 | 125 |
raise ValueError('not a json dict')
|
| 126 | 126 |
if entries.get('err') not in (None, 0, "0"):
|
| 127 | 127 |
raise ValueError('err %s' % entries['err'])
|
| 128 |
if not geojson and not isinstance(entries.get('data'), list):
|
|
| 129 |
raise ValueError('not a json dict with a data list attribute')
|
|
| 130 |
if geojson and not isinstance(entries.get('features'), list):
|
|
| 131 |
raise ValueError('bad geojson format')
|
|
| 128 |
if not isinstance(entries.get(data_key), list): |
|
| 129 |
raise ValueError('not a json dict with a %s list attribute' % data_key)
|
|
| 132 | 130 |
except misc.ConnectionError as e: |
| 133 |
if geojson: |
|
| 134 |
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
|
|
| 135 |
else: |
|
| 136 |
get_logger().warning('Error loading JSON data source (%s)' % str(e))
|
|
| 131 |
get_logger().warning('Error loading JSON data source (%s)' % str(e))
|
|
| 137 | 132 |
return None |
| 138 | 133 |
except (ValueError, TypeError) as e: |
| 139 |
if geojson: |
|
| 140 |
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
|
|
| 134 |
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
|
|
| 135 |
return None |
|
| 136 |
items = [] |
|
| 137 |
id_attribute = data_source.get('id_attribute') or 'id'
|
|
| 138 |
text_attribute = data_source.get('text_attribute') or 'text'
|
|
| 139 |
for item in entries.get(data_key): |
|
| 140 |
# skip malformed items |
|
| 141 |
if not isinstance(item, dict): |
|
| 142 |
continue |
|
| 143 |
if item.get(id_attribute) is None or item.get(id_attribute) == '': |
|
| 144 |
continue |
|
| 145 |
item['id'] = item[id_attribute] |
|
| 146 |
if text_attribute not in item: |
|
| 147 |
item['text'] = item['id'] |
|
| 141 | 148 |
else: |
| 142 |
get_logger().warning('Error reading JSON data source output (%s)' % str(e))
|
|
| 149 |
item['text'] = item[text_attribute] |
|
| 150 |
items.append(item) |
|
| 151 |
return items |
|
| 152 | ||
| 153 | ||
| 154 |
def request_geojson_items(url, data_source): |
|
| 155 |
url = sign_url_auto_orig(url) |
|
| 156 |
try: |
|
| 157 |
entries = misc.json_loads(misc.urlopen(url).read()) |
|
| 158 |
if not isinstance(entries, dict): |
|
| 159 |
raise ValueError('not a json dict')
|
|
| 160 |
if entries.get('err') not in (None, 0, "0"):
|
|
| 161 |
raise ValueError('err %s' % entries['err'])
|
|
| 162 |
if not isinstance(entries.get('features'), list):
|
|
| 163 |
raise ValueError('bad geojson format')
|
|
| 164 |
except misc.ConnectionError as e: |
|
| 165 |
get_logger().warning('Error loading GeoJSON data source (%s)' % str(e))
|
|
| 166 |
return None |
|
| 167 |
except (ValueError, TypeError) as e: |
|
| 168 |
get_logger().warning('Error reading GeoJSON data source output (%s)' % str(e))
|
|
| 143 | 169 |
return None |
| 144 | 170 |
items = [] |
| 145 |
if geojson: |
|
| 146 |
id_property = data_source.get('id_property') or 'id'
|
|
| 147 |
for item in entries.get('features'):
|
|
| 148 |
if not item.get('properties', {}).get(id_property):
|
|
| 149 |
continue |
|
| 150 |
item['id'] = item['properties'][id_property] |
|
| 151 |
try: |
|
| 152 |
item['text'] = Template( |
|
| 153 |
data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
|
|
| 154 |
except (TemplateSyntaxError, VariableDoesNotExist): |
|
| 155 |
pass |
|
| 156 |
if not item.get('text'):
|
|
| 157 |
item['text'] = item['id'] |
|
| 158 |
items.append(item) |
|
| 159 |
else: |
|
| 160 |
for item in entries.get('data'):
|
|
| 161 |
# skip malformed items |
|
| 162 |
if not isinstance(item, dict): |
|
| 163 |
continue |
|
| 164 |
if item.get('id') is None or item.get('id') == '':
|
|
| 165 |
continue |
|
| 166 |
if 'text' not in item: |
|
| 167 |
item['text'] = item['id'] |
|
| 168 |
items.append(item) |
|
| 171 |
id_property = data_source.get('id_property') or 'id'
|
|
| 172 |
for item in entries.get('features'):
|
|
| 173 |
if not item.get('properties', {}).get(id_property):
|
|
| 174 |
continue |
|
| 175 |
item['id'] = item['properties'][id_property] |
|
| 176 |
try: |
|
| 177 |
item['text'] = Template( |
|
| 178 |
data_source.get('label_template_property') or '{{ text }}').render(item['properties'])
|
|
| 179 |
except (TemplateSyntaxError, VariableDoesNotExist): |
|
| 180 |
pass |
|
| 181 |
if not item.get('text'):
|
|
| 182 |
item['text'] = item['id'] |
|
| 183 |
items.append(item) |
|
| 169 | 184 |
return items |
| 170 | 185 | |
| 171 | 186 | |
| ... | ... | |
| 247 | 262 |
if items is not None: |
| 248 | 263 |
return items |
| 249 | 264 | |
| 250 |
items = request_json_items(url, data_source) |
|
| 265 |
if geojson: |
|
| 266 |
items = request_geojson_items(url, data_source) |
|
| 267 |
else: |
|
| 268 |
items = request_json_items(url, data_source) |
|
| 251 | 269 |
if items is None: |
| 252 | 270 |
return [] |
| 253 | 271 |
if hasattr(request, 'datasources_cache'): |
| ... | ... | |
| 298 | 316 |
cache_duration = None |
| 299 | 317 |
query_parameter = None |
| 300 | 318 |
id_parameter = None |
| 319 |
data_attribute = None |
|
| 320 |
id_attribute = None |
|
| 321 |
text_attribute = None |
|
| 301 | 322 |
id_property = None |
| 302 | 323 |
label_template_property = None |
| 303 | 324 | |
| ... | ... | |
| 306 | 327 |
('cache_duration', 'str'),
|
| 307 | 328 |
('query_parameter', 'str'),
|
| 308 | 329 |
('id_parameter', 'str'),
|
| 330 |
('data_attribute', 'str'),
|
|
| 331 |
('id_attribute', 'str'),
|
|
| 332 |
('text_attribute', 'str'),
|
|
| 309 | 333 |
('id_property', 'str'),
|
| 310 | 334 |
('label_template_property', 'str'),
|
| 311 | 335 |
('data_source', 'data_source'),
|
| ... | ... | |
| 321 | 345 | |
| 322 | 346 |
@property |
| 323 | 347 |
def extended_data_source(self): |
| 324 |
if self.type != 'geojson': |
|
| 325 |
return self.data_source |
|
| 326 |
data_source = self.data_source.copy() |
|
| 327 |
data_source.update({
|
|
| 328 |
'id_property': self.id_property, |
|
| 329 |
'label_template_property': self.label_template_property, |
|
| 330 |
}) |
|
| 331 |
return data_source |
|
| 348 |
if self.type == 'geojson': |
|
| 349 |
data_source = self.data_source.copy() |
|
| 350 |
data_source.update({
|
|
| 351 |
'id_property': self.id_property, |
|
| 352 |
'label_template_property': self.label_template_property, |
|
| 353 |
}) |
|
| 354 |
return data_source |
|
| 355 |
if self.type == 'json': |
|
| 356 |
data_source = self.data_source.copy() |
|
| 357 |
data_source.update({
|
|
| 358 |
'data_attribute': self.data_attribute, |
|
| 359 |
'id_attribute': self.id_attribute, |
|
| 360 |
'text_attribute': self.text_attribute, |
|
| 361 |
}) |
|
| 362 |
return data_source |
|
| 363 |
return self.data_source |
|
| 332 | 364 | |
| 333 | 365 |
def can_jsonp(self): |
| 334 | 366 |
if self.type == 'jsonp': |
| 335 |
- |
|