0001-utils-add-cache-support-to-requests-wrapper-17192.patch
| passerelle/utils/__init__.py | ||
|---|---|---|
| 1 |
from cStringIO import StringIO |
|
| 1 | 2 |
from functools import wraps |
| 3 |
import hashlib |
|
| 2 | 4 |
import json |
| 3 | 5 |
import re |
| 4 | 6 |
import logging |
| 5 | 7 | |
| 6 |
from requests import Session as RequestSession |
|
| 8 |
from requests import Session as RequestSession, Response as RequestResponse |
|
| 9 |
from requests.structures import CaseInsensitiveDict |
|
| 7 | 10 | |
| 8 | 11 |
from django.conf import settings |
| 12 |
from django.core.cache import cache |
|
| 9 | 13 |
from django.core.exceptions import PermissionDenied, ObjectDoesNotExist |
| 10 | 14 |
from django.core.serializers.json import DjangoJSONEncoder |
| 11 | 15 |
from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest |
| ... | ... | |
| 165 | 169 |
super(Request, self).__init__(*args, **kwargs) |
| 166 | 170 | |
| 167 | 171 |
def request(self, method, url, **kwargs): |
| 172 |
cache_duration = kwargs.pop('cache_duration', None)
|
|
| 173 |
invalidate_cache = kwargs.pop('invalidate_cache', False)
|
|
| 168 | 174 |
params = kwargs.get('params', '')
|
| 169 | 175 |
self.logger.info('%s %s %s' % (method, url, params),
|
| 170 | 176 |
extra={'requests_url': url}
|
| ... | ... | |
| 192 | 198 |
if proxy: |
| 193 | 199 |
kwargs['proxies'] = {'http': proxy, 'https': proxy}
|
| 194 | 200 | |
| 201 |
if method == 'GET' and cache_duration: |
|
| 202 |
cache_key = hashlib.md5('%r;%r' % (url, kwargs)).hexdigest()
|
|
| 203 |
cache_content = cache.get(cache_key) |
|
| 204 |
if cache_content and not invalidate_cache: |
|
| 205 |
response = RequestResponse() |
|
| 206 |
response.raw = StringIO(cache_content.get('content'))
|
|
| 207 |
response.headers = CaseInsensitiveDict(cache_content.get('headers', {}))
|
|
| 208 |
response.status_code = cache_content.get('status_code')
|
|
| 209 |
print 'cached response' |
|
| 210 |
return response |
|
| 211 | ||
| 195 | 212 |
if settings.REQUESTS_PROXIES and 'proxies' not in kwargs: |
| 196 | 213 |
kwargs['proxies'] = settings.REQUESTS_PROXIES |
| 197 | 214 | |
| 198 | 215 |
response = super(Request, self).request(method, url, **kwargs) |
| 199 | 216 | |
| 217 |
if method == 'GET' and cache_duration and (response.status_code // 100 == 2): |
|
| 218 |
cache.set(cache_key, {
|
|
| 219 |
'content': response.content, |
|
| 220 |
'headers': response.headers, |
|
| 221 |
'statux_code': response.status_code, |
|
| 222 |
}, cache_duration) |
|
| 223 | ||
| 200 | 224 |
self.logger.debug('Request Headers: {}'.format(''.join([
|
| 201 | 225 |
'%s: %s | ' % (k,v) for k,v in response.request.headers.items() |
| 202 | 226 |
]))) |
| tests/test_requests.py | ||
|---|---|---|
| 6 | 6 | |
| 7 | 7 |
from django.test import override_settings |
| 8 | 8 | |
| 9 |
from passerelle.utils import Request |
|
| 9 |
from passerelle.utils import Request, CaseInsensitiveDict
|
|
| 10 | 10 |
import utils |
| 11 | 11 |
from utils import FakedResponse |
| 12 | 12 | |
| ... | ... | |
| 217 | 217 |
request.get('http://example.net/whatever', cert='/local.pem', verify=False)
|
| 218 | 218 |
assert mocked_get.call_args[1].get('cert') == '/local.pem'
|
| 219 | 219 |
assert mocked_get.call_args[1].get('verify') is False
|
| 220 | ||
| 221 |
@mock.patch('passerelle.utils.RequestSession.request')
|
|
| 222 |
def test_requests_cache(mocked_get, caplog): |
|
| 223 |
resource = MockResource() |
|
| 224 |
logger = logging.getLogger('requests')
|
|
| 225 |
request = Request(resource=resource, logger=logger) |
|
| 226 | ||
| 227 |
response_request = mock.Mock(headers={'Accept': '*/*'}, body=None)
|
|
| 228 |
mocked_get.return_value = FakedResponse( |
|
| 229 |
headers={'Content-Type': 'text/plain; charset=charset=utf-8'},
|
|
| 230 |
request=response_request, |
|
| 231 |
content='hello world', status_code=200) |
|
| 232 | ||
| 233 |
# by default there is no cache |
|
| 234 |
assert request.get('http://cache.example.org/').content == 'hello world'
|
|
| 235 |
assert request.get('http://cache.example.org/').content == 'hello world'
|
|
| 236 |
assert mocked_get.call_count == 2 |
|
| 237 | ||
| 238 |
# add some cache |
|
| 239 |
mocked_get.reset_mock() |
|
| 240 |
assert request.get('http://cache.example.org/', cache_duration=15).content == 'hello world'
|
|
| 241 |
assert mocked_get.call_count == 1 |
|
| 242 |
assert request.get('http://cache.example.org/', cache_duration=15).content == 'hello world'
|
|
| 243 |
assert mocked_get.call_count == 1 # got a cached response |
|
| 244 | ||
| 245 |
# value changed |
|
| 246 |
mocked_get.return_value = FakedResponse( |
|
| 247 |
headers={'Content-Type': 'text/plain; charset=charset=utf-8'},
|
|
| 248 |
request=response_request, |
|
| 249 |
content='hello second world', status_code=200) |
|
| 250 |
assert request.get('http://cache.example.org/', cache_duration=15).content == 'hello world'
|
|
| 251 |
assert mocked_get.call_count == 1 |
|
| 252 | ||
| 253 |
# force cache invalidation |
|
| 254 |
assert request.get('http://cache.example.org/', invalidate_cache=True).content == 'hello second world'
|
|
| 255 |
assert mocked_get.call_count == 2 |
|
| 256 | ||
| 257 |
# do not cache errors |
|
| 258 |
mocked_get.return_value = FakedResponse( |
|
| 259 |
headers={'Content-Type': 'text/plain; charset=charset=utf-8'},
|
|
| 260 |
request=response_request, |
|
| 261 |
content='no such world', status_code=404) |
|
| 262 |
mocked_get.reset_mock() |
|
| 263 |
response = request.get('http://cache.example.org/404', cache_duration=15)
|
|
| 264 |
assert response.content == 'no such world' |
|
| 265 |
assert response.status_code == 404 |
|
| 266 |
assert mocked_get.call_count == 1 |
|
| 267 |
response = request.get('http://cache.example.org/404', cache_duration=15)
|
|
| 268 |
assert mocked_get.call_count == 2 |
|
| 269 | ||
| 270 |
# check response headers |
|
| 271 |
mocked_get.reset_mock() |
|
| 272 |
mocked_get.return_value = FakedResponse( |
|
| 273 |
headers=CaseInsensitiveDict({'Content-Type': 'image/png'}),
|
|
| 274 |
request=response_request, |
|
| 275 |
content='hello world', status_code=200) |
|
| 276 |
assert request.get('http://cache.example.org/img', cache_duration=15).headers.get('content-type') == 'image/png'
|
|
| 277 |
assert mocked_get.call_count == 1 |
|
| 278 |
assert request.get('http://cache.example.org/img', cache_duration=15).headers.get('content-type') == 'image/png'
|
|
| 279 |
assert mocked_get.call_count == 1 # got a cached response |
|
| 220 |
- |
|