0001-logging-add-requests-and-responses-max-size-to-conne.patch

Nicolas Roche, 02 octobre 2019 17:18

Voir les différences: en ligne côte à côte

Subject: [PATCH] logging: add requests and responses max size to connector log
 parameters (#36596)

 .../migrations/0016_auto_20191002_1443.py     | 25 +++++++++++++++++++
 passerelle/base/models.py                     | 10 ++++++++
 passerelle/base/views.py                      |  6 ++++-
 passerelle/settings.py                        |  2 +-
 passerelle/utils/__init__.py                  | 12 +++++++--
 passerelle/utils/jsonresponse.py              |  6 ++++-
 passerelle/views.py                           |  2 +-
 tests/settings.py                             |  2 ++
 tests/test_api_access.py                      |  9 +++++--
 tests/test_requests.py                        |  9 ++++---
 10 files changed, 72 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/base/migrations/0016_auto_20191002_1443.py

     # -*- coding: utf-8 -*-
     # Generated by Django 1.11.18 on 2019-10-02 12:43
     from __future__ import unicode_literals
     from django.db import migrations, models
     class Migration(migrations.Migration):
         dependencies = [
             ('base', '0015_auto_20190921_0347'),
+        ]
         operations = [
             migrations.AddField(
                 model_name='loggingparameters',
                 name='requests_max_size',
                 field=models.PositiveIntegerField(default=5000, help_text='Maximum HTTP request size to log', verbose_name='Requests maximum size'),
             ),
             migrations.AddField(
                 model_name='loggingparameters',
                 name='responses_max_size',
                 field=models.PositiveIntegerField(default=4096, help_text='Maximum HTTP reponse size to log', verbose_name='Responses maximum size'),
             ),
+        ]

             help_text=_('One address per line (empty for site administrators)'),
             blank=True
+        )
         requests_max_size = models.PositiveIntegerField(
             verbose_name=_('Requests maximum size'),
             help_text=_('Maximum HTTP request size to log'),
             default=settings.LOGGED_REQUESTS_MAX_SIZE
+        )
         responses_max_size = models.PositiveIntegerField(
             verbose_name=_('Responses maximum size'),
             help_text=_('Maximum HTTP reponse size to log'),
             default=settings.LOGGED_RESPONSES_MAX_SIZE
+        )
         class Meta:
             unique_together = (('resource_type', 'resource_pk'))

         def get_form_class(self):
             form_class = model_forms.modelform_factory(
                 LoggingParameters,
                 fields=['log_level', 'trace_emails'])
                 fields=['log_level', 'trace_emails', 'requests_max_size', 'responses_max_size'])
             form_class.base_fields['trace_emails'].widget.attrs['rows'] = '3'
             return form_class
-...
             parameters = self.get_resource().logging_parameters
             d['log_level'] = parameters.log_level
             d['trace_emails'] = parameters.trace_emails
             d['requests_max_size'] = parameters.requests_max_size
             d['responses_max_size'] = parameters.responses_max_size
             return d
         def get_resource(self):
-...
             parameters = self.get_resource().logging_parameters
             parameters.log_level = form.cleaned_data['log_level']
             parameters.trace_emails = form.cleaned_data['trace_emails']
             parameters.requests_max_size = form.cleaned_data['requests_max_size']
             parameters.responses_max_size = form.cleaned_data['responses_max_size']
             parameters.save()
             return super(LoggingParametersUpdateView, self).form_valid(form)

     LOGGED_RESPONSES_MAX_SIZE = 4096
     # Max size of the request to log
     LOGGED_REQUEST_MAX_SIZE = 5000
     LOGGED_REQUESTS_MAX_SIZE = 5000
     # Number of days to keep logs
     LOG_RETENTION_DAYS = 7

         if logger.level == 10:  # DEBUG
             extra['request_headers'] = dict(request.headers.items())
             if request.body:
                 extra['request_payload'] = repr(request.body[:settings.LOGGED_REQUEST_MAX_SIZE])
                 if hasattr(logger, 'connector'):
                     max_size = logger.connector.logging_parameters.requests_max_size
                 else:
                     max_size = settings.LOGGED_REQUESTS_MAX_SIZE
                 extra['request_payload'] = repr(request.body[:max_size])
         if response is not None:
             message = message + ' (=> %s)' % response.status_code
             extra['response_status'] = response.status_code
-...
                 extra['response_headers'] = dict(response.headers.items())
                 # log body only if content type is allowed
                 if content_type_match(response.headers.get('Content-Type')):
                     content = response.content[:settings.LOGGED_RESPONSES_MAX_SIZE]
                     if hasattr(logger, 'connector'):
                         max_size = logger.connector.logging_parameters.responses_max_size
                     else:
                         max_size = settings.LOGGED_RESPONSES_MAX_SIZE
                     content = response.content[:max_size]
                     extra['response_content'] = repr(content)
             if response.status_code // 100 == 3:
                 log_function = logger.warning

             except Exception as e:
                 extras = {'method': req.method, 'exception': exception_to_text(e), 'request': req}
                 if req.method == 'POST':
                     extras.update({'body': repr(req.body[:settings.LOGGED_REQUEST_MAX_SIZE])})
                     if hasattr(logger, 'connector'):
                         max_size = logger.connector.logging_parameters.requests_max_size
                     else:
                         max_size = settings.LOGGED_REQUESTS_MAX_SIZE
                     extras.update({'body': repr(req.body[:max_size])})
                 if (not isinstance(e, (Http404, PermissionDenied, ObjectDoesNotExist, RequestException))
                         and getattr(e, 'log_error', True)):
                     logger.exception("Error occurred while processing request", extra=extras)

             connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
             connector = self.get_object()
             url = request.get_full_path()
             payload = request.body[:settings.LOGGED_REQUEST_MAX_SIZE]
             payload = request.body[:connector.logging_parameters.requests_max_size]
             try:
                 payload.decode('utf-8')
             except UnicodeDecodeError:

tests/settings.py
67	67	},
68	68	}
69	69	}
	70
	71	LOGGED_REQUESTS_MAX_SIZE = 4999

         assert resp.json['err'] == 1
         assert resp.json['err_desc'] == 'Payload error: missing "message" in JSON payload'
     def test_logged_request_max_size(app, oxyd, settings):
     def test_logged_requests_max_size(app, oxyd, settings):
         assert oxyd.logging_parameters.requests_max_size == 4999
         endpoint_url = reverse('generic-endpoint',
                 kwargs={'connector': 'oxyd', 'slug': oxyd.slug, 'endpoint': 'send'})
         api = ApiUser.objects.create(username='public',
-...
         # for empty payload the connector returns an APIError with
         assert 'Error occurred' in ResourceLog.objects.all()[1].message
         assert ResourceLog.objects.all()[1].extra['body'] == '\'{"foo": "bar"}\''
         # troncate logs
         settings.LOGGED_REQUEST_MAX_SIZE = 6
         parameters = oxyd.logging_parameters
         parameters.requests_max_size = 6
         parameters.save()
         resp = app.post_json(endpoint_url, params={'foo': 'bar'})
         assert ResourceLog.objects.all()[2].message[-8:] == '"foo"\') '
         assert ResourceLog.objects.all()[3].extra['body'] == '\'{"foo"\''

                 assert not hasattr(record, 'response_content')
                 assert not hasattr(record, 'response_headers')
     def test_log_error_request_max_size(caplog, log_level, settings):
     def test_log_error_http_max_sizes(caplog, log_level, settings):
         url = 'https://httperror.org/plop'
         logger = logging.getLogger('requests')
         logger.setLevel(log_level)
         settings.LOGGED_REQUEST_MAX_SIZE = 8
         assert settings.LOGGED_REQUESTS_MAX_SIZE == 4999
         assert settings.LOGGED_RESPONSES_MAX_SIZE == 4096
         settings.LOGGED_REQUESTS_MAX_SIZE = 8
         settings.LOGGED_RESPONSES_MAX_SIZE = 7
         with HTTMock(http400_mock):
             requests = Request(logger=logger)
             response = requests.post(url, json={'name':'josh'})
         print logger.level
         if logger.level == 10:  # DEBUG
             records = [record for record in caplog.records if record.name == 'requests']
             assert records[0].request_payload == '\'{"name":\''
             assert records[0].response_content == '\'{"foo":\''
     @pytest.fixture(params=['xml', 'whatever', 'jpeg', 'pdf'])
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0001-logging-add-requests-and-responses-max-size-to-conne.patch