From db7d9e5bea03629a6bfac79dd96443890b92573f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9ters?= Date: Fri, 21 Sep 2018 12:39:20 +0200 Subject: [PATCH] misc: keep data source calls in request cache (#26612) --- tests/test_datasource.py | 15 +++++++++++++++ wcs/data_sources.py | 9 ++++++++- wcs/qommon/http_request.py | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_datasource.py b/tests/test_datasource.py index f0c561248..b746e159b 100644 --- a/tests/test_datasource.py +++ b/tests/test_datasource.py @@ -117,39 +117,46 @@ def test_python_datasource_with_evalutils(): ('foo', 'Foo', 'foo', {'id': 'foo', 'text': 'Foo', 'value': '2017-01-01'})] def test_json_datasource(http_requests): + get_request().datasources_cache = {} datasource = {'type': 'json', 'value': ''} assert data_sources.get_items(datasource) == [] # missing file + get_request().datasources_cache = {} json_file_path = os.path.join(pub.app_dir, 'test.json') datasource = {'type': 'json', 'value': 'file://%s' % json_file_path} assert data_sources.get_items(datasource) == [] # invalid json file + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json_file.write(u'foobar'.encode('zlib_codec')) json_file.close() assert data_sources.get_items(datasource) == [] # empty json file + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({}, json_file) json_file.close() assert data_sources.get_items(datasource) == [] # unrelated json file + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump('foobar', json_file) json_file.close() assert data_sources.get_items(datasource) == [] # another unrelated json file + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': 'foobar'}, json_file) json_file.close() assert data_sources.get_items(datasource) == [] # a good json file + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': [{'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}]}, json_file) json_file.close() @@ -160,6 +167,7 @@ def test_json_datasource(http_requests): {'id': '1', 'text': 'foo'}, {'id': '2', 'text': 'bar'}] # a json file with additional keys + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': [{'id': '1', 'text': 'foo', 'more': 'xxx'}, {'id': '2', 'text': 'bar', 'more': 'yyy'}]}, json_file) @@ -172,6 +180,7 @@ def test_json_datasource(http_requests): {'id': '2', 'text': 'bar', 'more': 'yyy'}] # json specified with a variadic url + get_request().datasources_cache = {} class JsonUrlPath(object): def get_substitution_variables(self): return {'json_url': 'file://%s' % json_file_path} @@ -182,6 +191,7 @@ def test_json_datasource(http_requests): ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})] # same with django templated url + get_request().datasources_cache = {} class JsonUrlPath(object): def get_substitution_variables(self): return {'json_url': 'file://%s' % json_file_path} @@ -192,6 +202,7 @@ def test_json_datasource(http_requests): ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})] # json specified with a variadic url with an erroneous space + get_request().datasources_cache = {} class JsonUrlPath(object): def get_substitution_variables(self): return {'json_url': 'file://%s' % json_file_path} @@ -202,6 +213,7 @@ def test_json_datasource(http_requests): ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})] # same with django templated url + get_request().datasources_cache = {} class JsonUrlPath(object): def get_substitution_variables(self): return {'json_url': 'file://%s' % json_file_path} @@ -212,6 +224,7 @@ def test_json_datasource(http_requests): ('2', 'bar', '2', {'id': '2', 'text': 'bar', 'more': 'yyy'})] # a json file with integer as 'id' + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': [{'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}]}, json_file) json_file.close() @@ -222,6 +235,7 @@ def test_json_datasource(http_requests): {'id': 1, 'text': 'foo'}, {'id': 2, 'text': 'bar'}] # a json file with empty or no text values + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': [{'id': '1', 'text': ''}, {'id': '2'}]}, json_file) json_file.close() @@ -233,6 +247,7 @@ def test_json_datasource(http_requests): {'id': '2', 'text': '2'}] # a json file with empty or no id + get_request().datasources_cache = {} json_file = open(json_file_path, 'w') json.dump({'data': [{'id': '', 'text': 'foo'}, {'text': 'bar'}, {'id': None}]}, json_file) json_file.close() diff --git a/wcs/data_sources.py b/wcs/data_sources.py index a49d7eb3a..af89cedc5 100644 --- a/wcs/data_sources.py +++ b/wcs/data_sources.py @@ -19,7 +19,7 @@ import urllib import urlparse import xml.etree.ElementTree as ET -from quixote import get_publisher +from quixote import get_publisher, get_request from quixote.html import TemplateIO from qommon import _ @@ -156,6 +156,11 @@ def get_structured_items(data_source): if Template.is_template_string(url): vars = get_publisher().substitutions.get_context_variables() url = get_variadic_url(url, vars) + + request = get_request() + if hasattr(request, 'datasources_cache') and url in request.datasources_cache: + return request.datasources_cache[url] + try: signature_key, orig = get_secret_and_orig(url) except MissingSecret: @@ -181,6 +186,8 @@ def get_structured_items(data_source): if 'text' not in item: item['text'] = item['id'] items.append(item) + if hasattr(request, 'datasources_cache'): + request.datasources_cache[url] = items return items except qommon.misc.ConnectionError as e: get_logger().warn('Error loading JSON data source (%s)' % str(e)) diff --git a/wcs/qommon/http_request.py b/wcs/qommon/http_request.py index 76ad42797..35d1b90b1 100644 --- a/wcs/qommon/http_request.py +++ b/wcs/qommon/http_request.py @@ -37,6 +37,7 @@ class HTTPRequest(quixote.http_request.HTTPRequest): self.is_json_marker = None self.ignore_session = False self.wscalls_cache = {} + self.datasources_cache = {} # keep a copy of environment to make sure it's not reused along # uwsgi/gunicorn processes. self.environ = copy.copy(self.environ) -- 2.19.0