From 361b1be0870d72e685e7f39089ddd451791ed5c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Sat, 22 Sep 2018 11:24:43 +0200 Subject: [PATCH 1/2] misc: add cache duration option to named data sources (#26620) --- tests/test_datasource.py | 34 ++++++++++++++++++++++++++++++++++ wcs/admin/data_sources.py | 18 ++++++++++++++++++ wcs/data_sources.py | 26 ++++++++++++++++++++++++-- wcs/qommon/form.py | 18 ++++++++++++++++++ 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/tests/test_datasource.py b/tests/test_datasource.py index b746e159b..57f677630 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -439,3 +439,37 @@ def test_data_source_signed(no_request_pub): assert len(data_sources.get_items({'type': 'foobar'})) == 1 unsigned_url = urlopen.call_args[0][0] assert unsigned_url == 'https://no-secret.example.com/json' + +def test_named_datasource_json_cache(): + NamedDataSource.wipe() + datasource = NamedDataSource(name='foobar') + datasource.data_source = {'type': 'json', 'value': 'http://whatever/'} + datasource.store() + + with mock.patch('qommon.misc.urlopen') as urlopen: + urlopen.side_effect = lambda *args: StringIO( + json.dumps({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]})) + + assert data_sources.get_structured_items({'type': 'foobar'}) == [ + {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] + assert urlopen.call_count == 1 + + get_request().datasources_cache = {} + assert data_sources.get_structured_items({'type': 'foobar'}) == [ + {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] + assert urlopen.call_count == 2 + + datasource.cache_duration = '60' + datasource.store() + + # will cache + get_request().datasources_cache = {} + assert data_sources.get_structured_items({'type': 'foobar'}) == [ + {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] + assert urlopen.call_count == 3 + + # will get from cache + get_request().datasources_cache = {} + assert data_sources.get_structured_items({'type': 'foobar'}) == [ + {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] + assert urlopen.call_count == 3 diff --git a/wcs/admin/data_sources.py b/wcs/admin/data_sources.py index d12d89bc2..e6f3da0bd 100644 --- a/wcs/admin/data_sources.py +++ b/wcs/admin/data_sources.py @@ -20,6 +20,7 @@ from quixote.html import TemplateIO, htmltext from qommon import _ from qommon.form import * +from qommon.humantime import seconds2humanduration from qommon.backoffice.menu import html_top from wcs.data_sources import (NamedDataSource, DataSourceSelectionWidget, get_structured_items) @@ -43,6 +44,18 @@ class NamedDataSourceUI(object): title=_('Data Source'), allow_named_sources=False, required=True) + form.add(DurationWidget, 'cache_duration', + value=self.datasource.cache_duration, + title=_('Cache Duration'), + hint=_('Caching data will improve performances but will keep changes ' + 'from being visible immediately. You should keep this duration ' + 'reasonably short.'), + required=False, + advanced=False, + attrs={ + 'data-dynamic-display-child-of': 'data_source$type', + 'data-dynamic-display-value': _('JSON URL'), + }) if self.datasource.slug: form.add(StringWidget, 'slug', value=self.datasource.slug, @@ -74,6 +87,7 @@ class NamedDataSourceUI(object): self.datasource.name = name self.datasource.description = form.get_widget('description').parse() self.datasource.data_source = form.get_widget('data_source') + self.datasource.cache_duration = form.get_widget('cache_duration').parse() if self.datasource.slug: self.datasource.slug = slug self.datasource.store() @@ -127,6 +141,10 @@ class NamedDataSourcePage(Directory): r += htmltext('
  • %s%s
    • ') % ( _('Python Expression: '), self.datasource.data_source.get('value')) + if self.datasource.cache_duration: + r += htmltext('
  • %s %s
    • ') % ( + _('Cache Duration:'), + seconds2humanduration(int(self.datasource.cache_duration))) r += htmltext('') if data_source_type in ('json', 'formula'): diff --git a/wcs/data_sources.py b/wcs/data_sources.py index af89cedc5..e889120d6 100644 --- a/wcs/data_sources.py +++ b/wcs/data_sources.py @@ -15,6 +15,7 @@ # along with this program; if not, see . import collections +import hashlib import urllib import urlparse import xml.etree.ElementTree as ET @@ -108,7 +109,14 @@ def get_items(data_source, include_disabled=False): def get_structured_items(data_source): - data_source = get_real(data_source) + cache_duration = 0 + if data_source.get('type') not in ('json', 'jsonp', 'formula'): + # named data source + named_data_source = NamedDataSource.get_by_slug(data_source['type']) + if named_data_source.cache_duration: + cache_duration = int(named_data_source.cache_duration) + data_source = named_data_source.data_source + if data_source.get('type') == 'formula': # the result of a python expression, it must be a list. # - of strings @@ -161,6 +169,13 @@ def get_structured_items(data_source): if hasattr(request, 'datasources_cache') and url in request.datasources_cache: return request.datasources_cache[url] + if cache_duration: + cache_key = 'data-source-%s' % hashlib.md5(url).hexdigest() + from django.core.cache import cache + items = cache.get(cache_key) + if items is not None: + return items + try: signature_key, orig = get_secret_and_orig(url) except MissingSecret: @@ -188,6 +203,10 @@ def get_structured_items(data_source): items.append(item) if hasattr(request, 'datasources_cache'): request.datasources_cache[url] = items + + if cache_duration: + cache.set(cache_key, items, cache_duration) + return items except qommon.misc.ConnectionError as e: get_logger().warn('Error loading JSON data source (%s)' % str(e)) @@ -213,10 +232,13 @@ class NamedDataSource(XmlStorableObject): slug = None description = None data_source = None + cache_duration = None # declarations for serialization XML_NODES = [('name', 'str'), ('slug', 'str'), ('description', 'str'), - ('data_source', 'data_source')] + ('cache_duration', 'str'), + ('data_source', 'data_source'), + ] def __init__(self, name=None): StorableObject.__init__(self) diff --git a/wcs/qommon/form.py b/wcs/qommon/form.py index 8fa2cd633..8976bd84e 100644 --- a/wcs/qommon/form.py +++ b/wcs/qommon/form.py @@ -70,6 +70,7 @@ from wcs.conditions import Condition, ValidationError from qommon import _, ngettext import misc +from .humantime import humanduration2seconds, seconds2humanduration, timewords from .misc import strftime, C_ from publisher import get_cfg from .template_utils import render_block_to_string @@ -480,6 +481,23 @@ class StringWidget(quixote.form.StringWidget): self.error = str(e) +class DurationWidget(StringWidget): + def __init__(self, name, value=None, **kwargs): + if value: + value = seconds2humanduration(int(value)) + if 'hint' in kwargs: + kwargs['hint'] += htmltext('
    ') + else: + kwargs['hint'] = '' + kwargs['hint'] += htmltext( + _('Usable units of time: %s.')) % ', '.join(timewords()) + super(DurationWidget, self).__init__(name, value=value, **kwargs) + + def parse(self, request=None): + value = super(DurationWidget, self).parse(request) + return str(humanduration2seconds(self.value)) if value else None + + class TextWidget(quixote.form.TextWidget): def __init__(self, name, *args, **kwargs): self.validation_function = kwargs.pop('validation_function', None) -- 2.19.0