From c47cbbcdc9fdd8907c7b8c4c9220d167128be17d Mon Sep 17 00:00:00 2001
From: Nicolas ROCHE <nroche@entrouvert.com>
Date: Wed, 13 Nov 2019 15:05:34 +0100
Subject: [PATCH 2/2] logging: append connector response to ressource log
 (#31774)

---
 passerelle/base/models.py      | 40 ++++++++++++++++++++++++++-------
 passerelle/views.py            |  5 ++++-
 tests/test_generic_endpoint.py | 41 +++++++++++++++++++++++++++++++++-
 3 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/passerelle/base/models.py b/passerelle/base/models.py
index 19eded49..c6bb70c2 100644
--- a/passerelle/base/models.py
+++ b/passerelle/base/models.py
@@ -11,16 +11,18 @@ import base64
 import itertools
 
 from django.apps import apps
 from django.conf import settings
 from django.core.exceptions import ValidationError, ObjectDoesNotExist, PermissionDenied
 from django.core.urlresolvers import reverse
 from django.db import connection, models, transaction
 from django.db.models import Q
+from django.db.utils import DataError
+from django.http.response import HttpResponse
 from django.test import override_settings
 from django.utils.text import slugify
 from django.utils import timezone, six
 from django.utils import six
 from django.utils.encoding import force_text
 from django.utils.six.moves.urllib.parse import urlparse
 from django.utils.translation import ugettext_lazy as _
 from django.utils.timezone import now
@@ -731,16 +733,36 @@ class ResourceLog(models.Model):
 
     @property
     def level(self):
         return slugify(logging.getLevelName(self.levelno))
 
     def __str__(self):
         return '%s %s %s %s' % (self.timestamp, self.levelno, self.appname, self.slug)
 
+    def log_response(self, response, from_cache=False,
+                     responses_max_size=settings.LOGGED_RESPONSES_MAX_SIZE):
+        if from_cache:
+            self.extra['from_cache'] = True
+        if isinstance(response, HttpResponse):
+            self.extra['response_status'] = response.status_code
+            self.extra['response_headers'] = force_text(
+                    response.serialize_headers(), errors='ignore')
+            self.extra['response_content'] = force_text(
+                    response.content[:responses_max_size], errors='ignore')
+        else:
+            self.extra['response_content'] = force_text(
+                    str(response)[:responses_max_size], errors='ignore')
+        try:
+            with transaction.atomic():
+                self.save()
+        except (DataError, TypeError):  # py2, py3
+            self.extra['response_content'] = '<binary>'
+            self.save()
+
 
 STATUS_CHOICES = (
     ('unknown', _('Unknown')),
     ('up', _('Up')),
     ('down', _('Down')),
 )
 
 class ResourceStatus(models.Model):
@@ -775,16 +797,17 @@ class ProxyLogger(object):
         return self.__class__(self.connector, extra=dict(self.extra or {}, **kwargs))
 
     @property
     def level(self):
         return self._logger.getEffectiveLevel()
 
     def _log(self, levelname, message, *args, **kwargs):
         force = kwargs.pop('force', False)
+        ressource_log = None
         if self.connector.down() and not force:
             # don't log if the connector is known to be down
             return
 
         # hide credentials in urls
         for url in re.findall(r'(https?://\S+)', message):
             try:
                 parsed = urlparse(url)
@@ -831,46 +854,47 @@ class ProxyLogger(object):
             else:
                 sourceip = None
             attr['sourceip'] = sourceip
 
             if kwargs.get('exc_info'):
                 (exc_type, exc_value, tb) = sys.exc_info()
                 attr['extra']['error_summary'] = traceback.format_exception_only(exc_type, exc_value)
 
-            ResourceLog.objects.create(**attr)
+            ressource_log = ResourceLog.objects.create(**attr)
 
         admins = settings.ADMINS
         logging_parameters = self.connector.logging_parameters
         if logging_parameters.trace_emails:
             admins = [('', x) for x in logging_parameters.trace_emails.splitlines()]
         with override_settings(ADMINS=admins):
             getattr(self._logger, levelname.lower())(message, *args, **kwargs)
+        return ressource_log
 
     def exception(self, message, *args, **kwargs):
         kwargs['exc_info'] = 1
-        self._log('ERROR', message, *args, **kwargs)
+        return self._log('ERROR', message, *args, **kwargs)
 
     def debug(self, message, *args, **kwargs):
-        self._log('DEBUG', message, *args, **kwargs)
+        return self._log('DEBUG', message, *args, **kwargs)
 
     def info(self, message, *args, **kwargs):
-        self._log('INFO', message, *args, **kwargs)
+        return self._log('INFO', message, *args, **kwargs)
 
     def warning(self, message, *args, **kwargs):
-        self._log('WARNING', message, *args, **kwargs)
+        return self._log('WARNING', message, *args, **kwargs)
 
     def critical(self, message, *args, **kwargs):
-        self._log('CRITICAL', message, *args, **kwargs)
+        return self._log('CRITICAL', message, *args, **kwargs)
 
     def error(self, message, *args, **kwargs):
-        self._log('ERROR', message, *args, **kwargs)
+        return self._log('ERROR', message, *args, **kwargs)
 
     def fatal(self, message, *args, **kwargs):
-        self._log('FATAL', message, *args, **kwargs)
+        return self._log('FATAL', message, *args, **kwargs)
 
 
 class HTTPResource(models.Model):
     '''Mixin to add basic TLS/Basic HTTP authentication fields to any
        resource.'''
     basic_auth_username = models.CharField(
         max_length=128,
         verbose_name=_('Basic authentication username'),
diff --git a/passerelle/views.py b/passerelle/views.py
index 0fbdf379..e33382ff 100644
--- a/passerelle/views.py
+++ b/passerelle/views.py
@@ -439,17 +439,17 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
         connector_name, endpoint_name = kwargs['connector'], kwargs['endpoint']
         connector = self.get_object()
         url = request.get_full_path()
         payload = request.body[:connector.logging_parameters.requests_max_size]
         try:
             payload = payload.decode('utf-8')
         except UnicodeDecodeError:
             payload = '<BINARY PAYLOAD>'
-        connector.logger.info('endpoint %s %s (%r) ' %
+        resource_log = connector.logger.info('endpoint %s %s (%r) ' %
                               (request.method, url, payload),
                               extra={
                                   'request': request,
                                   'connector': connector_name,
                                   'connector_endpoint': endpoint_name,
                                   'connector_endpoint_method': request.method,
                                   'connector_endpoint_url': url,
                                   'connector_payload': payload
@@ -457,19 +457,22 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View):
 
         params = self.get_params(request, *args, **kwargs)
         if request.method == 'GET' and self.endpoint.endpoint_info.cache_duration:
             cache_key = hashlib.md5(
                 force_bytes(repr(self.get_object().slug) + repr(self.endpoint) + repr(params))
             ).hexdigest()
             result = cache.get(cache_key)
             if result is not None:
+                resource_log.log_response(result, from_cache=True,
+                                          responses_max_size=connector.logging_parameters.responses_max_size)
                 return result
 
         result = self.endpoint(request, **params)
+        resource_log.log_response(result, responses_max_size=connector.logging_parameters.responses_max_size)
         if request.method == 'GET' and self.endpoint.endpoint_info.cache_duration:
             cache.set(cache_key, result, self.endpoint.endpoint_info.cache_duration)
         return result
 
     def get(self, request, *args, **kwargs):
         if self.endpoint.endpoint_info.pattern:
             pattern = url(self.endpoint.endpoint_info.pattern, self.endpoint)
             match = pattern.resolve(kwargs.get('rest') or '')
diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py
index 13e1440c..8ca483fe 100644
--- a/tests/test_generic_endpoint.py
+++ b/tests/test_generic_endpoint.py
@@ -27,16 +27,17 @@ import mock
 import pytest
 
 import utils
 
 from django.contrib.contenttypes.models import ContentType
 from django.core.urlresolvers import reverse
 
 from passerelle.apps.arcgis.models import ArcGIS
+from passerelle.apps.opengis.models import OpenGIS
 from passerelle.base.models import ResourceLog, ProxyLogger, BaseResource, HTTPResource, LoggingParameters
 from passerelle.base.models import ResourceStatus
 from passerelle.apps.mdel.models import MDEL
 from passerelle.contrib.stub_invoices.models import StubInvoicesConnector
 from passerelle.utils.api import endpoint
 
 from test_manager import login, admin_user, simple_user
 
@@ -48,16 +49,23 @@ def mdel(db):
 
 @pytest.fixture
 def arcgis(db):
     instance = utils.setup_access_rights(ArcGIS.objects.create(slug='test'))
     instance.set_log_level('DEBUG')
     return instance
 
 
+@pytest.fixture
+def opengis(db):
+    instance = utils.setup_access_rights(OpenGIS.objects.create(slug='test'))
+    instance.set_log_level('DEBUG')
+    return instance
+
+
 DEMAND_STATUS = {
     'closed': True,
     'status': 'accepted',
     'comment': 'dossier trait\xe9.'
 }
 
 
 @mock.patch('passerelle.apps.mdel.models.Demand.get_status', lambda x: DEMAND_STATUS)
@@ -89,17 +97,16 @@ def test_generic_payload_logging(caplog, app, mdel):
 
 
 @mock.patch('passerelle.utils.Request.get')
 def test_proxy_logger(mocked_get, caplog, app, arcgis):
     payload = open(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read()
     mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200)
 
     # simple logger
-    arcgis.log_evel = 'DEBUG'
     logger = ProxyLogger(connector=arcgis)
     logger.debug('this is a debug test')
     logger.info('this is an info test')
 
     assert ResourceLog.objects.count() == 2
     for log in ResourceLog.objects.all():
         if log.levelno == 10:
             assert log.message == 'this is a debug test'
@@ -114,16 +121,17 @@ def test_proxy_logger(mocked_get, caplog, app, arcgis):
     log = ResourceLog.objects.filter(appname='arcgis', slug='test').first()
     assert log.appname == 'arcgis'
     assert log.slug == 'test'
     assert log.levelno == 20
     assert log.sourceip == '127.0.0.1'
     assert log.extra['connector'] == 'arcgis'
     assert log.extra['connector_endpoint'] == 'district'
     assert log.extra['connector_endpoint_method'] == 'GET'
+    assert eval(log.extra['response_content'])['data']['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT'
     assert '/arcgis/test/district?' in log.extra['connector_endpoint_url']
 
     # Resource Generic Logger
     for record in caplog.records:
         if record.name != 'passerelle.resource.arcgis.test':
             continue
         assert record.levelno == 20
         assert record.levelname == 'INFO'
@@ -141,16 +149,47 @@ def test_proxy_logger(mocked_get, caplog, app, arcgis):
     app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200)
     assert ResourceLog.objects.count() == 1
 
     arcgis.logger.warning('first warning')
     assert ResourceLog.objects.count() == 2
     assert ResourceLog.objects.last().message == 'first warning'
     assert ResourceLog.objects.last().levelno == 30
 
+    # truncate connector response on Resource Custom DB Logger
+    ResourceLog.objects.all().delete()
+    parameters = arcgis.logging_parameters
+    parameters.responses_max_size = 20
+    parameters.save()
+    app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200)
+    assert ResourceLog.objects.count() == 1
+    log = ResourceLog.objects.filter(appname='arcgis', slug='test').first()
+    assert len(log.extra['response_content']) == 20
+    assert log.extra.get('from_cache') == None
+
+    # check connector response returned by cache
+    ResourceLog.objects.all().delete()
+    arcgis.district.endpoint_info.cache_duration = 5
+    app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200)
+    app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200)
+    assert ResourceLog.objects.count() == 2
+    log = ResourceLog.objects.filter(appname='arcgis', slug='test').all()[1]
+    assert len(log.extra['response_content']) == 20
+    assert log.extra['from_cache'] == True
+
+@mock.patch('passerelle.utils.Request.get')
+def test_proxy_logger_receiving_binary(mocked_get, app, opengis):
+    mocked_get.return_value = utils.FakedResponse(content=b'\x89PNG\r\n\x1a\n\x00\x00...', status_code=200)
+    app.get('/opengis/test/tile' + '/16/33650/23378.png', status=200)
+    # Resource Custom DB Logger
+    assert ResourceLog.objects.count() == 1
+    log = ResourceLog.objects.filter(appname='opengis', slug='test').first()
+    assert log.extra['response_status'] == 200
+    assert log.extra['response_headers'] == 'Content-Type: image/png'
+    assert log.extra['response_content'] == '<binary>'
 
 class FakeConnectorBase(object):
     slug = 'connector'
 
     def get_connector_slug(self):
         return 'fake'
 
     @endpoint()
-- 
2.25.0

