0001-misc-add-cache-duration-option-to-named-data-sources.patch
| tests/test_datasource.py | ||
|---|---|---|
| 439 | 439 |
assert len(data_sources.get_items({'type': 'foobar'})) == 1
|
| 440 | 440 |
unsigned_url = urlopen.call_args[0][0] |
| 441 | 441 |
assert unsigned_url == 'https://no-secret.example.com/json' |
| 442 | ||
| 443 |
def test_named_datasource_json_cache(): |
|
| 444 |
NamedDataSource.wipe() |
|
| 445 |
datasource = NamedDataSource(name='foobar') |
|
| 446 |
datasource.data_source = {'type': 'json', 'value': 'http://whatever/'}
|
|
| 447 |
datasource.store() |
|
| 448 | ||
| 449 |
with mock.patch('qommon.misc.urlopen') as urlopen:
|
|
| 450 |
urlopen.side_effect = lambda *args: StringIO( |
|
| 451 |
json.dumps({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}))
|
|
| 452 | ||
| 453 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [
|
|
| 454 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
|
| 455 |
assert urlopen.call_count == 1 |
|
| 456 | ||
| 457 |
get_request().datasources_cache = {}
|
|
| 458 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [
|
|
| 459 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
|
| 460 |
assert urlopen.call_count == 2 |
|
| 461 | ||
| 462 |
datasource.cache_duration = '60' |
|
| 463 |
datasource.store() |
|
| 464 | ||
| 465 |
# will cache |
|
| 466 |
get_request().datasources_cache = {}
|
|
| 467 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [
|
|
| 468 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
|
| 469 |
assert urlopen.call_count == 3 |
|
| 470 | ||
| 471 |
# will get from cache |
|
| 472 |
get_request().datasources_cache = {}
|
|
| 473 |
assert data_sources.get_structured_items({'type': 'foobar'}) == [
|
|
| 474 |
{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]
|
|
| 475 |
assert urlopen.call_count == 3 |
|
| wcs/admin/data_sources.py | ||
|---|---|---|
| 20 | 20 | |
| 21 | 21 |
from qommon import _ |
| 22 | 22 |
from qommon.form import * |
| 23 |
from qommon.humantime import seconds2humanduration |
|
| 23 | 24 |
from qommon.backoffice.menu import html_top |
| 24 | 25 |
from wcs.data_sources import (NamedDataSource, DataSourceSelectionWidget, |
| 25 | 26 |
get_structured_items) |
| ... | ... | |
| 43 | 44 |
title=_('Data Source'),
|
| 44 | 45 |
allow_named_sources=False, |
| 45 | 46 |
required=True) |
| 47 |
form.add(DurationWidget, 'cache_duration', |
|
| 48 |
value=self.datasource.cache_duration, |
|
| 49 |
title=_('Cache Duration'),
|
|
| 50 |
hint=_('Caching data will improve performances but will keep changes '
|
|
| 51 |
'from being visible immediately. You should keep this duration ' |
|
| 52 |
'reasonably short.'), |
|
| 53 |
required=False, |
|
| 54 |
advanced=False, |
|
| 55 |
attrs={
|
|
| 56 |
'data-dynamic-display-child-of': 'data_source$type', |
|
| 57 |
'data-dynamic-display-value': _('JSON URL'),
|
|
| 58 |
}) |
|
| 46 | 59 |
if self.datasource.slug: |
| 47 | 60 |
form.add(StringWidget, 'slug', |
| 48 | 61 |
value=self.datasource.slug, |
| ... | ... | |
| 74 | 87 |
self.datasource.name = name |
| 75 | 88 |
self.datasource.description = form.get_widget('description').parse()
|
| 76 | 89 |
self.datasource.data_source = form.get_widget('data_source')
|
| 90 |
self.datasource.cache_duration = form.get_widget('cache_duration').parse()
|
|
| 77 | 91 |
if self.datasource.slug: |
| 78 | 92 |
self.datasource.slug = slug |
| 79 | 93 |
self.datasource.store() |
| ... | ... | |
| 127 | 141 |
r += htmltext('<li>%s<tt>%s</tt></li>') % (
|
| 128 | 142 |
_('Python Expression: '),
|
| 129 | 143 |
self.datasource.data_source.get('value'))
|
| 144 |
if self.datasource.cache_duration: |
|
| 145 |
r += htmltext('<li>%s %s</li>') % (
|
|
| 146 |
_('Cache Duration:'),
|
|
| 147 |
seconds2humanduration(int(self.datasource.cache_duration))) |
|
| 130 | 148 |
r += htmltext('</ul>')
|
| 131 | 149 | |
| 132 | 150 |
if data_source_type in ('json', 'formula'):
|
| wcs/data_sources.py | ||
|---|---|---|
| 15 | 15 |
# along with this program; if not, see <http://www.gnu.org/licenses/>. |
| 16 | 16 | |
| 17 | 17 |
import collections |
| 18 |
import hashlib |
|
| 18 | 19 |
import urllib |
| 19 | 20 |
import urlparse |
| 20 | 21 |
import xml.etree.ElementTree as ET |
| ... | ... | |
| 108 | 109 | |
| 109 | 110 | |
| 110 | 111 |
def get_structured_items(data_source): |
| 111 |
data_source = get_real(data_source) |
|
| 112 |
cache_duration = 0 |
|
| 113 |
if data_source.get('type') not in ('json', 'jsonp', 'formula'):
|
|
| 114 |
# named data source |
|
| 115 |
named_data_source = NamedDataSource.get_by_slug(data_source['type']) |
|
| 116 |
if named_data_source.cache_duration: |
|
| 117 |
cache_duration = int(named_data_source.cache_duration) |
|
| 118 |
data_source = named_data_source.data_source |
|
| 119 | ||
| 112 | 120 |
if data_source.get('type') == 'formula':
|
| 113 | 121 |
# the result of a python expression, it must be a list. |
| 114 | 122 |
# - of strings |
| ... | ... | |
| 161 | 169 |
if hasattr(request, 'datasources_cache') and url in request.datasources_cache: |
| 162 | 170 |
return request.datasources_cache[url] |
| 163 | 171 | |
| 172 |
if cache_duration: |
|
| 173 |
cache_key = 'data-source-%s' % hashlib.md5(url).hexdigest() |
|
| 174 |
from django.core.cache import cache |
|
| 175 |
items = cache.get(cache_key) |
|
| 176 |
if items is not None: |
|
| 177 |
return items |
|
| 178 | ||
| 164 | 179 |
try: |
| 165 | 180 |
signature_key, orig = get_secret_and_orig(url) |
| 166 | 181 |
except MissingSecret: |
| ... | ... | |
| 188 | 203 |
items.append(item) |
| 189 | 204 |
if hasattr(request, 'datasources_cache'): |
| 190 | 205 |
request.datasources_cache[url] = items |
| 206 | ||
| 207 |
if cache_duration: |
|
| 208 |
cache.set(cache_key, items, cache_duration) |
|
| 209 | ||
| 191 | 210 |
return items |
| 192 | 211 |
except qommon.misc.ConnectionError as e: |
| 193 | 212 |
get_logger().warn('Error loading JSON data source (%s)' % str(e))
|
| ... | ... | |
| 213 | 232 |
slug = None |
| 214 | 233 |
description = None |
| 215 | 234 |
data_source = None |
| 235 |
cache_duration = None |
|
| 216 | 236 | |
| 217 | 237 |
# declarations for serialization |
| 218 | 238 |
XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'),
|
| 219 |
('data_source', 'data_source')]
|
|
| 239 |
('cache_duration', 'str'),
|
|
| 240 |
('data_source', 'data_source'),
|
|
| 241 |
] |
|
| 220 | 242 | |
| 221 | 243 |
def __init__(self, name=None): |
| 222 | 244 |
StorableObject.__init__(self) |
| wcs/qommon/form.py | ||
|---|---|---|
| 70 | 70 | |
| 71 | 71 |
from qommon import _, ngettext |
| 72 | 72 |
import misc |
| 73 |
from .humantime import humanduration2seconds, seconds2humanduration, timewords |
|
| 73 | 74 |
from .misc import strftime, C_ |
| 74 | 75 |
from publisher import get_cfg |
| 75 | 76 |
from .template_utils import render_block_to_string |
| ... | ... | |
| 480 | 481 |
self.error = str(e) |
| 481 | 482 | |
| 482 | 483 | |
| 484 |
class DurationWidget(StringWidget): |
|
| 485 |
def __init__(self, name, value=None, **kwargs): |
|
| 486 |
if value: |
|
| 487 |
value = seconds2humanduration(int(value)) |
|
| 488 |
if 'hint' in kwargs: |
|
| 489 |
kwargs['hint'] += htmltext('<br>')
|
|
| 490 |
else: |
|
| 491 |
kwargs['hint'] = '' |
|
| 492 |
kwargs['hint'] += htmltext( |
|
| 493 |
_('Usable units of time: %s.')) % ', '.join(timewords())
|
|
| 494 |
super(DurationWidget, self).__init__(name, value=value, **kwargs) |
|
| 495 | ||
| 496 |
def parse(self, request=None): |
|
| 497 |
value = super(DurationWidget, self).parse(request) |
|
| 498 |
return str(humanduration2seconds(self.value)) if value else None |
|
| 499 | ||
| 500 | ||
| 483 | 501 |
class TextWidget(quixote.form.TextWidget): |
| 484 | 502 |
def __init__(self, name, *args, **kwargs): |
| 485 | 503 |
self.validation_function = kwargs.pop('validation_function', None)
|
| 486 |
- |
|