0003-allow-ressources-to-disable-error-logging-in-request.patch
passerelle/base/models.py | ||
---|---|---|
18 | 18 |
from django.db.models import Q |
19 | 19 |
from django.test import override_settings |
20 | 20 |
from django.utils.text import slugify |
21 |
from django.utils import timezone |
|
21 |
from django.utils import timezone, six
|
|
22 | 22 |
from django.utils.translation import ugettext_lazy as _ |
23 | 23 |
from django.core.files.base import ContentFile |
24 | 24 | |
... | ... | |
665 | 665 | |
666 | 666 |
attr['appname'] = self.appname |
667 | 667 |
attr['slug'] = self.slug |
668 |
attr['extra'] = kwargs.get('extra', {}) |
|
669 |
request = kwargs.pop('request', None) |
|
668 | ||
669 |
extra = kwargs.get('extra', {}) |
|
670 |
request = extra.get('request') |
|
671 | ||
672 |
def is_json_serializable(value): |
|
673 |
return isinstance(value, (list, dict) + six.integer_types + six.string_types) |
|
674 |
attr['extra'] = {key: value for key, value in extra.items() if is_json_serializable(value)} |
|
670 | 675 | |
671 | 676 |
if getattr(request, 'META', None): |
672 | 677 |
if 'HTTP_X_FORWARDED_FOR' in request.META: |
passerelle/contrib/mdph13/models.py | ||
---|---|---|
15 | 15 |
# You should have received a copy of the GNU Affero General Public License |
16 | 16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
17 | 17 | |
18 |
import sys |
|
18 | 19 |
import base64 |
19 | 20 |
import urlparse |
20 | 21 |
import datetime |
... | ... | |
44 | 45 |
else: |
45 | 46 |
func(value, path) |
46 | 47 | |
48 |
ERROR_MAPPING = { |
|
49 |
'dossier-inconnu': 'dossier-inconnu', |
|
50 |
'secret-invalide': 'secret-invalide', |
|
51 |
'dateNaissance-erronee': 'date-de-naissance-erronee', |
|
52 |
} |
|
53 | ||
47 | 54 | |
48 | 55 |
class MDPH13Resource(BaseResource, HTTPResource): |
49 | 56 |
category = _('Business Process Connectors') |
... | ... | |
53 | 60 |
EMAIL_RE = re.compile(r'^[^@\s]+@[^@\s]+\.[^@\s]+$') |
54 | 61 |
DATE_RE = re.compile(r'^\d{4}-\d{2}-\d{2}$') |
55 | 62 | |
63 |
log_requests_errors = False |
|
64 | ||
56 | 65 |
class Meta: |
57 | 66 |
verbose_name = _('MDPH CD13') |
58 | 67 | |
... | ... | |
60 | 69 |
return urlparse.urljoin(self.webservice_base_url, 'situation/dossier/%s' % file_number) |
61 | 70 | |
62 | 71 |
def url_get(self, *args, **kwargs): |
63 |
try: |
|
64 |
response = None |
|
65 |
response = self.requests.get(*args, **kwargs) |
|
66 |
response.raise_for_status() |
|
67 |
except requests.RequestException as e: |
|
68 |
data = {'exception': six.text_type(e)} |
|
69 |
if response is not None: |
|
70 |
try: |
|
71 |
if response.json(): |
|
72 |
data['body'] = response.json() |
|
73 |
except ValueError: |
|
74 |
body = response.content[:1000] |
|
75 |
if len(response.content) > 1000: |
|
76 |
body += b'<SNIPPED>' |
|
77 |
data['body'] = repr(body) |
|
78 |
data['status_code'] = response.status_code |
|
79 |
raise APIError('HTTP request failed', data=data) |
|
72 |
response = self.requests.get(*args, **kwargs) |
|
80 | 73 |
try: |
81 | 74 |
content = response.json() |
82 |
except ValueError as e: |
|
83 |
data = {'exception': six.text_type(e)} |
|
84 |
data['body'] = '%r<SNIPPED>' % response.content[:1000] |
|
85 |
raise APIError('HTTP request did not respond with JSON', data=data) |
|
75 |
except ValueError: |
|
76 |
response.raise_for_status() |
|
77 |
raise requests.RequestException('JSON expected', response=response) |
|
78 | ||
79 |
if content.get('err') != 0: |
|
80 |
err_desc = ERROR_MAPPING.get( |
|
81 |
content.get('err_code'), |
|
82 |
'err != 0: missing or unknown error code') |
|
83 |
raise APIError(err_desc, data=content) |
|
84 |
response.raise_for_status() |
|
86 | 85 |
return content |
87 | 86 | |
88 | 87 |
def call_situation_dossier(self, file_number, secret, dob, email=None): |
89 | 88 |
url = self.situation_dossier_url(file_number) |
90 | 89 |
email = email or 'appel-sans-utilisateur@cd13.fr' |
91 |
error_mapping = { |
|
92 |
'dossier-inconnu': 'dossier-inconnu', |
|
93 |
'secret-invalide': 'secret-invalide', |
|
94 |
'dateNaissance-erronee': 'date-de-naissance-erronee', |
|
95 |
} |
|
96 |
try: |
|
97 |
content = self.url_get( |
|
98 |
url, |
|
99 |
headers={ |
|
100 |
'X-CD13-Secret': base64.b64encode(secret.encode('utf-8')).decode('ascii'), |
|
101 |
'X-CD13-Email': email, |
|
102 |
'X-CD13-DateNaissBenef': dob.isoformat(), |
|
103 |
}) |
|
104 |
except APIError as e: |
|
105 |
if e.data and e.data.get('status_code') == 404 and isinstance(e.data.get('body'), dict): |
|
106 |
content = e.data['body'] |
|
107 |
if content.get('err') != 0: |
|
108 |
error_code = content.get('err_code') |
|
109 |
err_desc = error_mapping.get(error_code, 'err != 0') |
|
110 |
raise APIError(err_desc, data=e.data) |
|
111 |
raise |
|
112 | ||
113 |
if content.get('err') != 0: |
|
114 |
error_code = content.get('err_code') |
|
115 |
err_desc = error_mapping.get(error_code, 'err != 0') |
|
116 |
raise APIError(err_desc, data=content) |
|
90 |
content = self.url_get( |
|
91 |
url, |
|
92 |
headers={ |
|
93 |
'X-CD13-Secret': base64.b64encode(secret.encode('utf-8')).decode('ascii'), |
|
94 |
'X-CD13-Email': email, |
|
95 |
'X-CD13-DateNaissBenef': dob.isoformat(), |
|
96 |
}) |
|
117 | 97 | |
118 | 98 |
data = content.get('data') |
119 | 99 |
if not isinstance(data, dict): |
passerelle/utils/__init__.py | ||
---|---|---|
42 | 42 |
from passerelle.base.models import ApiUser, AccessRight, BaseResource |
43 | 43 |
from passerelle.base.signature import check_query, check_url |
44 | 44 | |
45 |
# legacy import, other modules keep importing to_json from passerelle.utils |
|
46 |
from .jsonresponse import to_json |
|
47 | ||
48 | 45 | |
49 | 46 |
def get_template_vars(): |
50 | 47 |
""" |
... | ... | |
173 | 170 |
return False |
174 | 171 | |
175 | 172 | |
173 |
def log_http_request(logger, request, response=None, exception=None, error_log=True, extra=None): |
|
174 |
log_function = logger.info |
|
175 |
message = '%s %s' % (request.method, request.url) |
|
176 |
extra = extra or {} |
|
177 |
extra['request_url'] = request.url |
|
178 |
if logger.level == 10: # DEBUG |
|
179 |
extra['request_headers'] = dict(request.headers.items()) |
|
180 |
if request.body: |
|
181 |
extra['request_payload'] = repr(request.body[:5000]) |
|
182 |
if response is not None: |
|
183 |
message = message + ' (=> %s)' % response.status_code |
|
184 |
extra['response_status'] = response.status_code |
|
185 |
if logger.level == 10: # DEBUG |
|
186 |
extra['response_headers'] = dict(response.headers.items()) |
|
187 |
# log body only if content type is allowed |
|
188 |
if content_type_match(response.headers.get('Content-Type')): |
|
189 |
content = response.content[:settings.LOGGED_RESPONSES_MAX_SIZE] |
|
190 |
extra['response_content'] = repr(content) |
|
191 |
if response.status_code // 100 == 3: |
|
192 |
log_function = logger.warning |
|
193 |
elif response.status_code // 100 >= 4: |
|
194 |
log_function = logger.error |
|
195 |
elif exception: |
|
196 |
message = message + ' (=> %s)' % repr(exception) |
|
197 |
extra['response_exception'] = repr(exception) |
|
198 |
log_function = logger.error |
|
199 | ||
200 |
# allow resources to disable any error log at requests level |
|
201 |
if not error_log: |
|
202 |
log_function = logger.info |
|
203 |
log_function(message, extra=extra) |
|
204 | ||
205 | ||
176 | 206 |
# Wrapper around requests.Session |
177 | 207 |
# - log input and output data |
178 | 208 |
# - use HTTP Basic auth if resource.basic_auth_username and resource.basic_auth_password exist |
... | ... | |
182 | 212 |
# - use a proxy for HTTP and HTTPS if resource.http_proxy exists |
183 | 213 | |
184 | 214 |
class Request(RequestSession): |
185 | ||
186 | 215 |
def __init__(self, *args, **kwargs): |
187 | 216 |
self.logger = kwargs.pop('logger') |
188 | 217 |
self.resource = kwargs.pop('resource', None) |
... | ... | |
251 | 280 |
except Exception as exc: |
252 | 281 |
self.log_http_request(request, exception=exc) |
253 | 282 |
raise |
254 |
self.log_http_request(request, response) |
|
283 |
self.log_http_request(request, response=response)
|
|
255 | 284 |
return response |
256 | 285 | |
257 | 286 |
def log_http_request(self, request, response=None, exception=None): |
258 |
message = '%s %s' % (request.method, request.url) |
|
259 |
extra = {} |
|
260 |
extra['request_url'] = request.url |
|
261 |
if self.logger.level == 10: # DEBUG |
|
262 |
extra['request_headers'] = dict(request.headers.items()) |
|
263 |
if request.body: |
|
264 |
extra['request_payload'] = repr(request.body[:5000]) |
|
265 |
log_function = self.logger.info |
|
266 |
if response is not None: |
|
267 |
message = message + ' (=> %s)' % response.status_code |
|
268 |
extra['response_status'] = response.status_code |
|
269 |
if self.logger.level == 10: # DEBUG |
|
270 |
extra['response_headers'] = dict(response.headers.items()) |
|
271 |
# log body only if content type is allowed |
|
272 |
if content_type_match(response.headers.get('Content-Type')): |
|
273 |
content = response.content[:settings.LOGGED_RESPONSES_MAX_SIZE] |
|
274 |
extra['response_content'] = repr(content) |
|
275 |
if response.status_code // 100 == 3: |
|
276 |
log_function = self.logger.warning |
|
277 |
elif response.status_code // 100 >= 4: |
|
278 |
log_function = self.logger.error |
|
279 |
elif exception: |
|
280 |
message = message + ' (=> %s)' % repr(exception) |
|
281 |
extra['response_exception'] = repr(exception) |
|
282 |
log_function = self.logger.error |
|
283 | ||
284 |
log_function(message, extra=extra) |
|
287 |
error_log = getattr(self.resource, 'log_requests_errors', True) |
|
288 |
log_http_request(self.logger, request=request, response=response, exception=exception, error_log=error_log) |
|
285 | 289 | |
286 | 290 | |
287 | 291 |
class SOAPClient(Client): |
... | ... | |
359 | 363 |
# call next() at least one time to advance, if the caller does not |
360 | 364 |
# consume the returned iterators, sourceiter will never be exhausted. |
361 | 365 |
yield chain([batchiter.next()], batchiter) |
366 | ||
367 |
# legacy import, other modules keep importing to_json from passerelle.utils |
|
368 |
from .jsonresponse import to_json |
passerelle/utils/jsonresponse.py | ||
---|---|---|
16 | 16 |
from django.utils.translation import force_text |
17 | 17 |
from django.utils import six |
18 | 18 | |
19 |
from requests.exceptions import RequestException |
|
19 |
from requests import RequestException, HTTPError |
|
20 | ||
21 |
from passerelle.utils import log_http_request |
|
20 | 22 | |
21 | 23 | |
22 | 24 |
DEFAULT_DEBUG = getattr(settings, 'JSONRESPONSE_DEFAULT_DEBUG', False) |
... | ... | |
148 | 150 |
data = self.obj_to_response(req, resp) |
149 | 151 |
status = 200 |
150 | 152 |
except Exception as e: |
151 |
extras = {'method': req.method, 'exception': exception_to_text(e)} |
|
152 |
if not self.logger: |
|
153 |
extras['request'] = req |
|
153 |
extras = {'method': req.method, 'exception': exception_to_text(e), 'request': req} |
|
154 | 154 |
if req.method == 'POST': |
155 | 155 |
extras.update({'body': repr(req.body[:5000])}) |
156 | 156 |
if (not isinstance(e, (Http404, PermissionDenied, ObjectDoesNotExist, RequestException)) |
... | ... | |
160 | 160 |
logger.warning('object not found: %r', e, extra=extras) |
161 | 161 |
elif isinstance(e, PermissionDenied): |
162 | 162 |
logger.warning('Permission denied', extra=extras) |
163 |
elif isinstance(e, HTTPError): |
|
164 |
log_http_request(logger, |
|
165 |
request=e.request, |
|
166 |
response=e.response, |
|
167 |
extra=extras) |
|
163 | 168 |
elif isinstance(e, RequestException): |
164 |
logger.warning("RequestException occurred while processing request", extra=extras) |
|
169 |
log_http_request(logger, |
|
170 |
request=e.request, |
|
171 |
exception=e, |
|
172 |
extra=extras) |
|
165 | 173 |
elif isinstance(e, Http404): |
166 | 174 |
# Http404 is for silent object not found exceptions |
167 | 175 |
pass |
passerelle/views.py | ||
---|---|---|
376 | 376 |
payload = '<BINARY PAYLOAD>' |
377 | 377 |
connector.logger.info('endpoint %s %s (%r) ' % |
378 | 378 |
(request.method, url, payload), |
379 |
request=request, |
|
380 | 379 |
extra={ |
380 |
'request': request, |
|
381 | 381 |
'connector': connector_name, |
382 | 382 |
'connector_endpoint': endpoint_name, |
383 | 383 |
'connector_endpoint_method': request.method, |
tests/test_mdph13.py | ||
---|---|---|
18 | 18 |
import json |
19 | 19 |
import base64 |
20 | 20 |
import datetime |
21 |
import logging |
|
22 | ||
23 |
import requests |
|
24 |
import requests.exceptions |
|
21 | 25 | |
22 | 26 |
import httmock |
23 | 27 |
import pytest |
... | ... | |
216 | 220 |
def request_handler(self, url, request): |
217 | 221 |
idx = len(self.requests) |
218 | 222 |
self.requests.append(request) |
219 |
return self.responses[idx] |
|
223 |
response = self.responses[idx] |
|
224 |
if isinstance(response, Exception): |
|
225 |
raise response |
|
226 |
elif hasattr(response, '__call__'): |
|
227 |
response = response(url, request) |
|
228 |
return response |
|
220 | 229 | |
221 | 230 |
mock_http = MockHttp() |
222 | 231 |
with httmock.HTTMock(httmock.urlmatch()(mock_http.request_handler)): |
... | ... | |
488 | 497 |
with pytest.raises(APIError) as exc_info: |
489 | 498 |
mdph13.dossiers(None, NAME_ID, EMAIL, link_id=str(link.pk)) |
490 | 499 |
assert str(exc_info.value) == 'invalid-response-format' |
500 | ||
501 | ||
502 |
def test_dossier_http_error(app, mdph13, mock_http, caplog): |
|
503 |
url = '/mdph13/test1/link/?NameID=%s&numero_dossier=%s&date_de_naissance=%s&secret=%s&email=%s' |
|
504 |
url = url % (NAME_ID, FILE_NUMBER, DOB, SECRET, EMAIL) |
|
505 |
mock_http.add_response({'status_code': 401, 'content': 'wtf', 'reason': 'Authentication required'}) |
|
506 |
response = app.post(url, status=500) |
|
507 |
assert response.json['err_class'] == 'requests.exceptions.HTTPError' |
|
508 |
assert caplog.records[-1].levelno == logging.ERROR |
|
509 |
assert caplog.records[-1].getMessage() == 'GET http://cd13.fr/situation/dossier/1234 (=> 401)' |
|
510 |
assert hasattr(caplog.records[-1].request, 'META') |
|
511 | ||
512 |
def raise_ssl_error(url, request): |
|
513 |
raise requests.exceptions.SSLError(request=request) |
|
514 |
mock_http.add_response(raise_ssl_error) |
|
515 |
response = app.post(url, status=500) |
|
516 |
assert response.json['err_class'] == 'requests.exceptions.SSLError' |
|
517 |
assert caplog.records[-1].levelno == logging.ERROR |
|
518 |
assert caplog.records[-1].getMessage() == 'GET http://cd13.fr/situation/dossier/1234 (=> SSLError())' |
|
519 |
assert hasattr(caplog.records[-1].request, 'META') |
|
491 |
- |