From 6edc86f11251dc98c58a994ef8a3b0084e62ef8d Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Thu, 3 Oct 2019 15:59:59 +0200
Subject: [PATCH 2/2] logging: add max sizes to connector log parameters
 (#36596)

---
 .../migrations/0016_auto_20191002_1443.py     | 25 +++++++++++++++++++
 passerelle/base/models.py                     | 10 ++++++++
 passerelle/base/views.py                      |  6 ++++-
 passerelle/settings.py                        |  2 +-
 passerelle/utils/__init__.py                  | 12 +++++++--
 passerelle/utils/jsonresponse.py              |  6 ++++-
 passerelle/views.py                           |  2 +-
 tests/settings.py                             |  2 ++
 tests/test_api_access.py                      | 12 ++++++---
 tests/test_requests.py                        |  7 ++++--
 10 files changed, 72 insertions(+), 12 deletions(-)
 create mode 100644 passerelle/base/migrations/0016_auto_20191002_1443.py

diff --git a/passerelle/base/migrations/0016_auto_20191002_1443.py b/passerelle/base/migrations/0016_auto_20191002_1443.py
new file mode 100644
index 00000000..ec1c24de
--- /dev/null
+++ b/passerelle/base/migrations/0016_auto_20191002_1443.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-10-02 12:43
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0015_auto_20190921_0347'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='loggingparameters',
+            name='requests_max_size',
+            field=models.PositiveIntegerField(default=5000, help_text='Maximum HTTP request size to log', verbose_name='Requests maximum size'),
+        ),
+        migrations.AddField(
+            model_name='loggingparameters',
+            name='responses_max_size',
+            field=models.PositiveIntegerField(default=5000, help_text='Maximum HTTP reponse size to log', verbose_name='Responses maximum size'),
+        ),
+    ]
diff --git a/passerelle/base/models.py b/passerelle/base/models.py
index 09263f19..60fa9402 100644
--- a/passerelle/base/models.py
+++ b/passerelle/base/models.py
@@ -605,6 +605,16 @@ class LoggingParameters(models.Model):
         help_text=_('One address per line (empty for site administrators)'),
         blank=True
     )
+    requests_max_size = models.PositiveIntegerField(
+        verbose_name=_('Requests maximum size'),
+        help_text=_('Maximum HTTP request size to log'),
+        default=settings.LOGGED_REQUESTS_MAX_SIZE
+    )
+    responses_max_size = models.PositiveIntegerField(
+        verbose_name=_('Responses maximum size'),
+        help_text=_('Maximum HTTP reponse size to log'),
+        default=settings.LOGGED_RESPONSES_MAX_SIZE
+    )
 
     class Meta:
         unique_together = (('resource_type', 'resource_pk'))
diff --git a/passerelle/base/views.py b/passerelle/base/views.py
index 2ee44946..d2630076 100644
--- a/passerelle/base/views.py
+++ b/passerelle/base/views.py
@@ -136,7 +136,7 @@ class LoggingParametersUpdateView(FormView):
     def get_form_class(self):
         form_class = model_forms.modelform_factory(
             LoggingParameters,
-            fields=['log_level', 'trace_emails'])
+            fields=['log_level', 'trace_emails', 'requests_max_size', 'responses_max_size'])
         form_class.base_fields['trace_emails'].widget.attrs['rows'] = '3'
         return form_class
 
@@ -147,6 +147,8 @@ class LoggingParametersUpdateView(FormView):
         parameters = self.get_resource().logging_parameters
         d['log_level'] = parameters.log_level
         d['trace_emails'] = parameters.trace_emails
+        d['requests_max_size'] = parameters.requests_max_size
+        d['responses_max_size'] = parameters.responses_max_size
         return d
 
     def get_resource(self):
@@ -160,6 +162,8 @@ class LoggingParametersUpdateView(FormView):
         parameters = self.get_resource().logging_parameters
         parameters.log_level = form.cleaned_data['log_level']
         parameters.trace_emails = form.cleaned_data['trace_emails']
+        parameters.requests_max_size = form.cleaned_data['requests_max_size']
+        parameters.responses_max_size = form.cleaned_data['responses_max_size']
         parameters.save()
         return super(LoggingParametersUpdateView, self).form_valid(form)
 
diff --git a/passerelle/settings.py b/passerelle/settings.py
index 607719a4..0190fa18 100644
--- a/passerelle/settings.py
+++ b/passerelle/settings.py
@@ -209,7 +209,7 @@ LOGGED_CONTENT_TYPES_MESSAGES = (
 )
 
 # Max size of the response to log
-LOGGED_RESPONSES_MAX_SIZE = 4096
+LOGGED_RESPONSES_MAX_SIZE = 5000
 
 # Max size of the request to log
 LOGGED_REQUESTS_MAX_SIZE = 5000
diff --git a/passerelle/utils/__init__.py b/passerelle/utils/__init__.py
index 4710a9f3..e17ebba1 100644
--- a/passerelle/utils/__init__.py
+++ b/passerelle/utils/__init__.py
@@ -161,7 +161,11 @@ def log_http_request(logger, request, response=None, exception=None, error_log=T
     if logger.level == 10:  # DEBUG
         extra['request_headers'] = dict(request.headers.items())
         if request.body:
-            extra['request_payload'] = repr(request.body[:settings.LOGGED_REQUESTS_MAX_SIZE])
+            if hasattr(logger, 'connector'):
+                max_size = logger.connector.logging_parameters.requests_max_size
+            else:
+                max_size = settings.LOGGED_REQUESTS_MAX_SIZE
+            extra['request_payload'] = repr(request.body[:max_size])
     if response is not None:
         message = message + ' (=> %s)' % response.status_code
         extra['response_status'] = response.status_code
@@ -169,7 +173,11 @@ def log_http_request(logger, request, response=None, exception=None, error_log=T
             extra['response_headers'] = dict(response.headers.items())
             # log body only if content type is allowed
             if content_type_match(response.headers.get('Content-Type')):
-                content = response.content[:settings.LOGGED_RESPONSES_MAX_SIZE]
+                if hasattr(logger, 'connector'):
+                    max_size = logger.connector.logging_parameters.responses_max_size
+                else:
+                    max_size = settings.LOGGED_RESPONSES_MAX_SIZE
+                content = response.content[:max_size]
                 extra['response_content'] = repr(content)
         if response.status_code // 100 == 3:
             log_function = logger.warning
diff --git a/passerelle/utils/jsonresponse.py b/passerelle/utils/jsonresponse.py
index 4e25e932..71c03875 100644
--- a/passerelle/utils/jsonresponse.py
+++ b/passerelle/utils/jsonresponse.py
@@ -132,7 +132,11 @@ class to_json(object):
         except Exception as e:
             extras = {'method': req.method, 'exception': exception_to_text(e), 'request': req}
             if req.method == 'POST':
-                extras.update({'body': repr(req.body[:settings.LOGGED_REQUESTS_MAX_SIZE])})
+                if hasattr(logger, 'connector'):
+                    max_size = logger.connector.logging_parameters.requests_max_size
+                else:
+                    max_size = settings.LOGGED_REQUESTS_MAX_SIZE
+                extras.update({'body': repr(req.body[:max_size])})
             if (not isinstance(e, (Http404, PermissionDenied, ObjectDoesNotExist, RequestException))
                     and getattr(e, 'log_error', True)):
                 logger.exception("Error occurred while processing request", extra=extras)
diff --git a/passerelle/views.py b/passerelle/views.py
index b330e611..ea474353 100644
--- a/passerelle/views.py
+++ b/passerelle/views.py
@@ -408,7 +408,7 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
         connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
         connector = self.get_object()
         url = request.get_full_path()
-        payload = request.body[:settings.LOGGED_REQUESTS_MAX_SIZE]
+        payload = request.body[:connector.logging_parameters.requests_max_size]
         try:
             payload.decode('utf-8')
         except UnicodeDecodeError:
diff --git a/tests/settings.py b/tests/settings.py
index cff87ca9..5486ea6c 100644
--- a/tests/settings.py
+++ b/tests/settings.py
@@ -67,3 +67,5 @@ DATABASES = {
         },
     }
 }
+
+LOGGED_REQUESTS_MAX_SIZE = 4999
diff --git a/tests/test_api_access.py b/tests/test_api_access.py
index bfd73827..cc525fd4 100644
--- a/tests/test_api_access.py
+++ b/tests/test_api_access.py
@@ -208,8 +208,8 @@ def test_logged_requests_and_responses_max_size(app, oxyd, settings):
     oxyd.set_log_level('DEBUG')  # log request payload and response headers/content
     settings.LOGGED_CONTENT_TYPES_MESSAGES = 'foo/bar'  # response content to log
 
-    assert settings.LOGGED_REQUEST_MAX_SIZE == 5000
-    assert settings.LOGGED_RESPONSES_MAX_SIZE == 4096
+    assert oxyd.logging_parameters.requests_max_size == 4999
+    assert oxyd.logging_parameters.responses_max_size == 5000
     with utils.mock_url(oxyd.URL, response, headers=headers):
         result = app.post_json(endpoint_url, params=payload)
     assert len(ResourceLog.objects.all()) == 4
@@ -227,8 +227,12 @@ def test_logged_requests_and_responses_max_size(app, oxyd, settings):
     assert len(ResourceLog.objects.all()[3].extra['body']) == 86
 
     # troncate logs
-    settings.LOGGED_REQUESTS_MAX_SIZE = 10
-    settings.LOGGED_RESPONSES_MAX_SIZE = 20
+    parameters = oxyd.logging_parameters
+    parameters.requests_max_size = 10
+    parameters.save()
+    parameters = oxyd.logging_parameters
+    parameters.responses_max_size = 20
+    parameters.save()
     with utils.mock_url(oxyd.URL, response, headers=headers):
         result = app.post_json(endpoint_url, params=payload)
     assert len(ResourceLog.objects.all()) == 8
diff --git a/tests/test_requests.py b/tests/test_requests.py
index b0c91cb4..f5f63d22 100644
--- a/tests/test_requests.py
+++ b/tests/test_requests.py
@@ -105,21 +105,24 @@ def test_log_error(caplog, log_level):
             assert not hasattr(record, 'response_content')
             assert not hasattr(record, 'response_headers')
 
-def test_log_error_request_max_size(caplog, log_level, settings):
+def test_log_error_http_max_sizes(caplog, log_level, settings):
     url = 'https://httperror.org/plop'
 
     logger = logging.getLogger('requests')
     logger.setLevel(log_level)
 
+    assert settings.LOGGED_REQUESTS_MAX_SIZE == 4999
+    assert settings.LOGGED_RESPONSES_MAX_SIZE == 5000
     settings.LOGGED_REQUESTS_MAX_SIZE = 8
+    settings.LOGGED_RESPONSES_MAX_SIZE = 7
     with HTTMock(http400_mock):
         requests = Request(logger=logger)
         response = requests.post(url, json={'name':'josh'})
 
-    print logger.level
     if logger.level == 10:  # DEBUG
         records = [record for record in caplog.records if record.name == 'requests']
         assert records[0].request_payload == '\'{"name":\''
+        assert records[0].response_content == '\'{"foo":\''
 
 
 @pytest.fixture(params=['xml', 'whatever', 'jpeg', 'pdf'])
-- 
2.23.0

