From 206058d8a038eb22177fea060902056b12bcf6f4 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 5 Dec 2016 10:14:27 +0100 Subject: [PATCH] add integrated log system (#14191) --- passerelle/base/migrations/0003_resourcelog.py | 27 ++++++++++++++++++ passerelle/base/models.py | 39 ++++++++++++++++++++++++++ passerelle/settings.py | 4 +-- passerelle/views.py | 3 +- tests/test_generic_endpoint.py | 38 ++++++++++++++++++++++++- 5 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 passerelle/base/migrations/0003_resourcelog.py diff --git a/passerelle/base/migrations/0003_resourcelog.py b/passerelle/base/migrations/0003_resourcelog.py new file mode 100644 index 0000000..2613cd0 --- /dev/null +++ b/passerelle/base/migrations/0003_resourcelog.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0002_auto_20151009_0326'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('connector', models.CharField(max_length=128, verbose_name=b'connector')), + ('loglevel', models.CharField(max_length=16, verbose_name=b'log level')), + ('ipsource', models.GenericIPAddressField(unpack_ipv4=True, null=True, verbose_name='IP Address', blank=True)), + ('message', models.TextField(max_length=2048, verbose_name=b'message')), + ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), + ], + ), + ] diff --git a/passerelle/base/models.py b/passerelle/base/models.py index fe6ac3f..f456a71 100644 --- a/passerelle/base/models.py +++ b/passerelle/base/models.py @@ -13,6 +13,8 @@ from django.contrib.contenttypes import fields from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager +import jsonfield + import passerelle KEYTYPE_CHOICES = ( @@ -151,6 +153,33 @@ class BaseResource(models.Model): return [(x, getattr(self, x.name, None)) for x in self._meta.fields if x.name not in ( 'id', 'title', 'slug', 'description', 'log_level', 'users')] + # LOG Helpers + def _log(self, levelname, message, request=None, **extra): + connector = '%s-%s' % (slugify(unicode(self.__class__.__name__)), self.slug) + + if 'HTTP_X_FORWARDED_FOR' in request.META: + ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() + else: + ipsource = request.META.get('REMOTE_ADDR') + + # Resource Custom DB Loggger + ResourceLog.objects.create(connector=connector, loglevel=levelname, + ipsource=ipsource, message=message, extra=extra.get('extra', {})) + # Default Resource Logger + getattr(self.logger, levelname)(message, extra=extra.get('extra', {})) + + def log_debug(self, message, request=None, **extra): + self._log('debug', message, request, **extra) + + def log_info(self, message, request=None, **extra): + self._log('info', message, request, **extra) + + def log_warn(self, message, request=None, **extra): + self._log('warning', message, request, **extra) + + def log_error(self, message, request=None, **extra): + self._log('error', message, request, **extra) + class AccessRight(models.Model): codename = models.CharField(max_length=100, verbose_name='codename') @@ -166,3 +195,13 @@ class AccessRight(models.Model): def __unicode__(self): return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) + + +class ResourceLog(models.Model): + timestamp = models.DateTimeField(auto_now_add=True) + connector = models.CharField(max_length=128, verbose_name='connector') + loglevel = models.CharField(max_length=16, verbose_name='log level') + ipsource = models.GenericIPAddressField(blank=True, null=True, unpack_ipv4=True, + verbose_name=_('IP Address')) + message = models.TextField(max_length=2048, verbose_name='message') + extra = jsonfield.JSONField(verbose_name='extras', default={}) diff --git a/passerelle/settings.py b/passerelle/settings.py index 2830f08..f14d69f 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -173,8 +173,8 @@ LOGGING = { 'handlers': { 'console': { 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - }, + 'class': 'logging.StreamHandler' + }, }, 'loggers': { 'django.request': { diff --git a/passerelle/views.py b/passerelle/views.py index ddbc511..8d17406 100644 --- a/passerelle/views.py +++ b/passerelle/views.py @@ -226,8 +226,9 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): connector = self.get_object() url = request.get_full_path() payload = request.body[:5000] - connector.logger.debug('endpoint %s %s (%r) ' % + connector.log_debug('endpoint %s %s (%r) ' % (request.method, url, payload), + request=request, extra={ 'connector': connector_name, 'connector_endpoint': endpoint_name, diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py index 45e1460..2d805be 100644 --- a/tests/test_generic_endpoint.py +++ b/tests/test_generic_endpoint.py @@ -25,7 +25,9 @@ import pytest import utils +from passerelle.base.models import ResourceLog from passerelle.contrib.mdel.models import MDEL +from passerelle.contrib.arcgis.models import Arcgis @pytest.fixture @@ -33,6 +35,11 @@ def mdel(db): return utils.setup_access_rights(MDEL.objects.create(slug='test')) +@pytest.fixture +def arcgis(db): + return utils.setup_access_rights(Arcgis.objects.create(slug='test')) + + DEMAND_STATUS = { 'closed': True, 'status': 'accepted', @@ -57,7 +64,7 @@ def test_generic_payload_logging(caplog, app, mdel): records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] for record in records: - assert record.module == 'views' + # assert record.module == 'views' assert record.levelname == 'DEBUG' assert record.connector == 'mdel' if record.connector_endpoint_method == 'POST': @@ -67,3 +74,32 @@ def test_generic_payload_logging(caplog, app, mdel): assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message assert record.connector_endpoint == 'status' assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' + + +@mock.patch('passerelle.utils.LoggedRequest.get') +def test_get_district(mocked_get, caplog, app, arcgis): + payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() + mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) + resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + + # Resource Custom DB Logger + log = ResourceLog.objects.first() + assert log.connector == 'arcgis-test' + assert log.loglevel == 'debug' + assert log.ipsource == '127.0.0.1' + assert log.extra['connector'] == 'arcgis' + assert log.extra['connector_endpoint'] == 'district' + assert log.extra['connector_endpoint_method'] == 'GET' + assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' + + # Resource Generic Logger + assert len(caplog.records()) == 1 + for record in caplog.records(): + assert record.levelno == 10 + assert record.levelname == 'DEBUG' + assert record.name == 'passerelle.resource.arcgis.test' + assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " + + data = resp.json['data'] + assert data['id'] == 4 + assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' -- 2.11.0