0002-logging-add-max-sizes-to-connector-log-parameters-36.patch

Nicolas Roche, 25 octobre 2019 14:28

Voir les différences: en ligne côte à côte

Subject: [PATCH 2/2] logging: add max sizes to connector log parameters
 (#36596)

 .../migrations/0016_auto_20191002_1443.py     | 25 +++++++++++++++++++
 passerelle/base/models.py                     | 10 ++++++++
 passerelle/base/views.py                      |  6 ++++-
 passerelle/settings.py                        |  2 +-
 passerelle/utils/__init__.py                  | 12 +++++++--
 passerelle/utils/jsonresponse.py              |  6 ++++-
 passerelle/views.py                           |  2 +-
 tests/settings.py                             |  2 ++
 tests/test_api_access.py                      | 12 ++++++---
 tests/test_requests.py                        |  7 ++++--
 10 files changed, 72 insertions(+), 12 deletions(-)
 create mode 100644 passerelle/base/migrations/0016_auto_20191002_1443.py

     # -*- coding: utf-8 -*-
     # Generated by Django 1.11.18 on 2019-10-02 12:43
     from __future__ import unicode_literals
     from django.db import migrations, models
     class Migration(migrations.Migration):
         dependencies = [
             ('base', '0015_auto_20190921_0347'),
+        ]
         operations = [
             migrations.AddField(
                 model_name='loggingparameters',
                 name='requests_max_size',
                 field=models.PositiveIntegerField(default=5000, help_text='Maximum HTTP request size to log', verbose_name='Requests maximum size'),
             ),
             migrations.AddField(
                 model_name='loggingparameters',
                 name='responses_max_size',
                 field=models.PositiveIntegerField(default=5000, help_text='Maximum HTTP reponse size to log', verbose_name='Responses maximum size'),
             ),
+        ]

             help_text=_('One address per line (empty for site administrators)'),
             blank=True
+        )
         requests_max_size = models.PositiveIntegerField(
             verbose_name=_('Requests maximum size'),
             help_text=_('Maximum HTTP request size to log'),
             default=settings.LOGGED_REQUESTS_MAX_SIZE
+        )
         responses_max_size = models.PositiveIntegerField(
             verbose_name=_('Responses maximum size'),
             help_text=_('Maximum HTTP reponse size to log'),
             default=settings.LOGGED_RESPONSES_MAX_SIZE
+        )
         class Meta:
             unique_together = (('resource_type', 'resource_pk'))

         def get_form_class(self):
             form_class = model_forms.modelform_factory(
                 LoggingParameters,
                 fields=['log_level', 'trace_emails'])
                 fields=['log_level', 'trace_emails', 'requests_max_size', 'responses_max_size'])
             form_class.base_fields['trace_emails'].widget.attrs['rows'] = '3'
             return form_class
-...
             parameters = self.get_resource().logging_parameters
             d['log_level'] = parameters.log_level
             d['trace_emails'] = parameters.trace_emails
             d['requests_max_size'] = parameters.requests_max_size
             d['responses_max_size'] = parameters.responses_max_size
             return d
         def get_resource(self):
-...
             parameters = self.get_resource().logging_parameters
             parameters.log_level = form.cleaned_data['log_level']
             parameters.trace_emails = form.cleaned_data['trace_emails']
             parameters.requests_max_size = form.cleaned_data['requests_max_size']
             parameters.responses_max_size = form.cleaned_data['responses_max_size']
             parameters.save()
             return super(LoggingParametersUpdateView, self).form_valid(form)

+    )
     # Max size of the response to log
     LOGGED_RESPONSES_MAX_SIZE = 4096
     LOGGED_RESPONSES_MAX_SIZE = 5000
     # Max size of the request to log
     LOGGED_REQUESTS_MAX_SIZE = 5000

         if logger.level == 10:  # DEBUG
             extra['request_headers'] = dict(request.headers.items())
             if request.body:
                 extra['request_payload'] = repr(request.body[:settings.LOGGED_REQUESTS_MAX_SIZE])
                 if hasattr(logger, 'connector'):
                     max_size = logger.connector.logging_parameters.requests_max_size
                 else:
                     max_size = settings.LOGGED_REQUESTS_MAX_SIZE
                 extra['request_payload'] = repr(request.body[:max_size])
         if response is not None:
             message = message + ' (=> %s)' % response.status_code
             extra['response_status'] = response.status_code
-...
                 extra['response_headers'] = dict(response.headers.items())
                 # log body only if content type is allowed
                 if content_type_match(response.headers.get('Content-Type')):
                     content = response.content[:settings.LOGGED_RESPONSES_MAX_SIZE]
                     if hasattr(logger, 'connector'):
                         max_size = logger.connector.logging_parameters.responses_max_size
                     else:
                         max_size = settings.LOGGED_RESPONSES_MAX_SIZE
                     content = response.content[:max_size]
                     extra['response_content'] = repr(content)
             if response.status_code // 100 == 3:
                 log_function = logger.warning

             except Exception as e:
                 extras = {'method': req.method, 'exception': exception_to_text(e), 'request': req}
                 if req.method == 'POST':
                     extras.update({'body': repr(req.body[:settings.LOGGED_REQUESTS_MAX_SIZE])})
                     if hasattr(logger, 'connector'):
                         max_size = logger.connector.logging_parameters.requests_max_size
                     else:
                         max_size = settings.LOGGED_REQUESTS_MAX_SIZE
                     extras.update({'body': repr(req.body[:max_size])})
                 if (not isinstance(e, (Http404, PermissionDenied, ObjectDoesNotExist, RequestException))
                         and getattr(e, 'log_error', True)):
                     logger.exception("Error occurred while processing request", extra=extras)

             connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
             connector = self.get_object()
             url = request.get_full_path()
             payload = request.body[:settings.LOGGED_REQUESTS_MAX_SIZE]
             payload = request.body[:connector.logging_parameters.requests_max_size]
             try:
                 payload.decode('utf-8')
             except UnicodeDecodeError:

tests/settings.py
67	67	},
68	68	}
69	69	}
	70
	71	LOGGED_REQUESTS_MAX_SIZE = 4999

         oxyd.set_log_level('DEBUG')  # log request payload and response headers/content
         settings.LOGGED_CONTENT_TYPES_MESSAGES = 'foo/bar'  # response content to log
         assert settings.LOGGED_REQUEST_MAX_SIZE == 5000
         assert settings.LOGGED_RESPONSES_MAX_SIZE == 4096
         assert oxyd.logging_parameters.requests_max_size == 4999
         assert oxyd.logging_parameters.responses_max_size == 5000
         with utils.mock_url(oxyd.URL, response, headers=headers):
             result = app.post_json(endpoint_url, params=payload)
         assert len(ResourceLog.objects.all()) == 4
-...
         assert len(ResourceLog.objects.all()[3].extra['body']) == 86
         # troncate logs
         settings.LOGGED_REQUESTS_MAX_SIZE = 10
         settings.LOGGED_RESPONSES_MAX_SIZE = 20
         parameters = oxyd.logging_parameters
         parameters.requests_max_size = 10
         parameters.save()
         parameters = oxyd.logging_parameters
         parameters.responses_max_size = 20
         parameters.save()
         with utils.mock_url(oxyd.URL, response, headers=headers):
             result = app.post_json(endpoint_url, params=payload)
         assert len(ResourceLog.objects.all()) == 8

                 assert not hasattr(record, 'response_content')
                 assert not hasattr(record, 'response_headers')
     def test_log_error_request_max_size(caplog, log_level, settings):
     def test_log_error_http_max_sizes(caplog, log_level, settings):
         url = 'https://httperror.org/plop'
         logger = logging.getLogger('requests')
         logger.setLevel(log_level)
         assert settings.LOGGED_REQUESTS_MAX_SIZE == 4999
         assert settings.LOGGED_RESPONSES_MAX_SIZE == 5000
         settings.LOGGED_REQUESTS_MAX_SIZE = 8
         settings.LOGGED_RESPONSES_MAX_SIZE = 7
         with HTTMock(http400_mock):
             requests = Request(logger=logger)
             response = requests.post(url, json={'name':'josh'})
         print logger.level
         if logger.level == 10:  # DEBUG
             records = [record for record in caplog.records if record.name == 'requests']
             assert records[0].request_payload == '\'{"name":\''
             assert records[0].response_content == '\'{"foo":\''
     @pytest.fixture(params=['xml', 'whatever', 'jpeg', 'pdf'])
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0002-logging-add-max-sizes-to-connector-log-parameters-36.patch