0001-misc-add-cache-duration-option-to-named-data-sources.patch
tests/test_datasource.py | ||
---|---|---|
439 | 439 |
assert len(data_sources.get_items({'type': 'foobar'})) == 1 |
440 | 440 |
unsigned_url = urlopen.call_args[0][0] |
441 | 441 |
assert unsigned_url == 'https://no-secret.example.com/json' |
442 | ||
443 |
def test_named_datasource_json_cache(): |
|
444 |
NamedDataSource.wipe() |
|
445 |
datasource = NamedDataSource(name='foobar') |
|
446 |
datasource.data_source = {'type': 'json', 'value': 'http://whatever/'} |
|
447 |
datasource.store() |
|
448 | ||
449 |
with mock.patch('qommon.misc.urlopen') as urlopen: |
|
450 |
urlopen.side_effect = lambda *args: StringIO( |
|
451 |
json.dumps({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]})) |
|
452 | ||
453 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [ |
|
454 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] |
|
455 |
assert urlopen.call_count == 1 |
|
456 | ||
457 |
get_request().datasources_cache = {} |
|
458 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [ |
|
459 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] |
|
460 |
assert urlopen.call_count == 2 |
|
461 | ||
462 |
datasource.cache_duration = '60' |
|
463 |
datasource.store() |
|
464 | ||
465 |
# will cache |
|
466 |
get_request().datasources_cache = {} |
|
467 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [ |
|
468 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] |
|
469 |
assert urlopen.call_count == 3 |
|
470 | ||
471 |
# will get from cache |
|
472 |
get_request().datasources_cache = {} |
|
473 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [ |
|
474 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] |
|
475 |
assert urlopen.call_count == 3 |
wcs/admin/data_sources.py | ||
---|---|---|
20 | 20 | |
21 | 21 |
from qommon import _ |
22 | 22 |
from qommon.form import * |
23 |
from qommon.humantime import seconds2humanduration |
|
23 | 24 |
from qommon.backoffice.menu import html_top |
24 | 25 |
from wcs.data_sources import (NamedDataSource, DataSourceSelectionWidget, |
25 | 26 |
get_structured_items) |
... | ... | |
43 | 44 |
title=_('Data Source'), |
44 | 45 |
allow_named_sources=False, |
45 | 46 |
required=True) |
47 |
form.add(DurationWidget, 'cache_duration', |
|
48 |
value=self.datasource.cache_duration, |
|
49 |
title=_('Cache Duration'), |
|
50 |
required=False, |
|
51 |
advanced=False, |
|
52 |
attrs={ |
|
53 |
'data-dynamic-display-child-of': 'data_source$type', |
|
54 |
'data-dynamic-display-value': _('JSON URL'), |
|
55 |
}) |
|
46 | 56 |
if self.datasource.slug: |
47 | 57 |
form.add(StringWidget, 'slug', |
48 | 58 |
value=self.datasource.slug, |
... | ... | |
74 | 84 |
self.datasource.name = name |
75 | 85 |
self.datasource.description = form.get_widget('description').parse() |
76 | 86 |
self.datasource.data_source = form.get_widget('data_source') |
87 |
self.datasource.cache_duration = form.get_widget('cache_duration').parse() |
|
77 | 88 |
if self.datasource.slug: |
78 | 89 |
self.datasource.slug = slug |
79 | 90 |
self.datasource.store() |
... | ... | |
127 | 138 |
r += htmltext('<li>%s<tt>%s</tt></li>') % ( |
128 | 139 |
_('Python Expression: '), |
129 | 140 |
self.datasource.data_source.get('value')) |
141 |
if self.datasource.cache_duration: |
|
142 |
r += htmltext('<li>%s %s</li>') % ( |
|
143 |
_('Cache Duration:'), |
|
144 |
seconds2humanduration(int(self.datasource.cache_duration))) |
|
130 | 145 |
r += htmltext('</ul>') |
131 | 146 | |
132 | 147 |
if data_source_type in ('json', 'formula'): |
wcs/data_sources.py | ||
---|---|---|
15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
16 | 16 | |
17 | 17 |
import collections |
18 |
import hashlib |
|
18 | 19 |
import urllib |
19 | 20 |
import urlparse |
20 | 21 |
import xml.etree.ElementTree as ET |
... | ... | |
108 | 109 | |
109 | 110 | |
110 | 111 |
def get_structured_items(data_source): |
111 |
data_source = get_real(data_source) |
|
112 |
cache_duration = 0 |
|
113 |
if data_source.get('type') not in ('json', 'jsonp', 'formula'): |
|
114 |
# named data source |
|
115 |
named_data_source = NamedDataSource.get_by_slug(data_source['type']) |
|
116 |
if named_data_source.cache_duration: |
|
117 |
cache_duration = int(named_data_source.cache_duration) |
|
118 |
data_source = named_data_source.data_source |
|
119 | ||
112 | 120 |
if data_source.get('type') == 'formula': |
113 | 121 |
# the result of a python expression, it must be a list. |
114 | 122 |
# - of strings |
... | ... | |
161 | 169 |
if hasattr(request, 'datasources_cache') and url in request.datasources_cache: |
162 | 170 |
return request.datasources_cache[url] |
163 | 171 | |
172 |
if cache_duration: |
|
173 |
cache_key = 'data-source-%s' % hashlib.md5(url).hexdigest() |
|
174 |
from django.core.cache import cache |
|
175 |
items = cache.get(cache_key) |
|
176 |
if items is not None: |
|
177 |
return items |
|
178 | ||
164 | 179 |
try: |
165 | 180 |
signature_key, orig = get_secret_and_orig(url) |
166 | 181 |
except MissingSecret: |
... | ... | |
188 | 203 |
items.append(item) |
189 | 204 |
if hasattr(request, 'datasources_cache'): |
190 | 205 |
request.datasources_cache[url] = items |
206 | ||
207 |
if cache_duration: |
|
208 |
cache.set(cache_key, items, cache_duration) |
|
209 | ||
191 | 210 |
return items |
192 | 211 |
except qommon.misc.ConnectionError as e: |
193 | 212 |
get_logger().warn('Error loading JSON data source (%s)' % str(e)) |
... | ... | |
213 | 232 |
slug = None |
214 | 233 |
description = None |
215 | 234 |
data_source = None |
235 |
cache_duration = None |
|
216 | 236 | |
217 | 237 |
# declarations for serialization |
218 | 238 |
XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'), |
219 |
('data_source', 'data_source')] |
|
239 |
('cache_duration', 'str'), |
|
240 |
('data_source', 'data_source'), |
|
241 |
] |
|
220 | 242 | |
221 | 243 |
def __init__(self, name=None): |
222 | 244 |
StorableObject.__init__(self) |
wcs/qommon/form.py | ||
---|---|---|
70 | 70 | |
71 | 71 |
from qommon import _, ngettext |
72 | 72 |
import misc |
73 |
from .humantime import humanduration2seconds, seconds2humanduration, timewords |
|
73 | 74 |
from .misc import strftime, C_ |
74 | 75 |
from publisher import get_cfg |
75 | 76 |
from .template_utils import render_block_to_string |
... | ... | |
480 | 481 |
self.error = str(e) |
481 | 482 | |
482 | 483 | |
484 |
class DurationWidget(StringWidget): |
|
485 |
def __init__(self, name, value=None, **kwargs): |
|
486 |
if value: |
|
487 |
value = seconds2humanduration(int(value)) |
|
488 |
if not 'hint' in kwargs: |
|
489 |
kwargs['hint'] = htmltext( |
|
490 |
_('ex.: 10 minutes. ' |
|
491 |
'Usable units of time: %s.')) % ', '.join(timewords()) |
|
492 |
super(DurationWidget, self).__init__(name, value=value, **kwargs) |
|
493 | ||
494 |
def parse(self, request=None): |
|
495 |
value = super(DurationWidget, self).parse(request) |
|
496 |
return str(humanduration2seconds(self.value)) if value else None |
|
497 | ||
498 | ||
483 | 499 |
class TextWidget(quixote.form.TextWidget): |
484 | 500 |
def __init__(self, name, *args, **kwargs): |
485 | 501 |
self.validation_function = kwargs.pop('validation_function', None) |
486 |
- |