From 2e4c26fbcad17d161ddfaefc8d0c5648bf4e42c1 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 5 Dec 2016 10:14:27 +0100 Subject: [PATCH] add integrated log system (#14191) --- passerelle/base/migrations/0003_resourcelog.py | 35 ++++++++++ passerelle/base/models.py | 97 ++++++++++++++++++++++++-- passerelle/settings.py | 4 +- passerelle/views.py | 1 + tests/test_generic_endpoint.py | 71 ++++++++++++++++++- 5 files changed, 201 insertions(+), 7 deletions(-) create mode 100644 passerelle/base/migrations/0003_resourcelog.py diff --git a/passerelle/base/migrations/0003_resourcelog.py b/passerelle/base/migrations/0003_resourcelog.py new file mode 100644 index 0000000..09473ac --- /dev/null +++ b/passerelle/base/migrations/0003_resourcelog.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '0002_auto_20151009_0326'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('resource_pk', models.PositiveIntegerField(null=True, blank=True)), + ('connector', models.CharField(max_length=128, null=True, verbose_name=b'connector')), + ('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), + ('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), + ('loglevel', models.CharField(max_length=16, verbose_name=b'log level')), + ('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Address', blank=True)), + ('message', models.TextField(max_length=2048, verbose_name=b'message')), + ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), + ('resource_type', models.ForeignKey(blank=True, to='contenttypes.ContentType', null=True)), + ], + options={ + 'permissions': (('view_resourcelog', 'Can view resource logs'),), + }, + ), + ] diff --git a/passerelle/base/models.py b/passerelle/base/models.py index fe6ac3f..67be10a 100644 --- a/passerelle/base/models.py +++ b/passerelle/base/models.py @@ -13,6 +13,8 @@ from django.contrib.contenttypes import fields from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager +import jsonfield + import passerelle KEYTYPE_CHOICES = ( @@ -97,10 +99,8 @@ class BaseResource(models.Model): def __init__(self, *args, **kwargs): super(BaseResource, self).__init__(*args, **kwargs) - self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( - slugify(unicode(self.__class__.__name__)), self.slug) - ) - self.logger.setLevel(getattr(logging, self.log_level)) + self.logger = ProxyLogger(self.log_level, self.__class__.__name__, + self.get_connector_slug(), self.slug) def __unicode__(self): return self.title @@ -166,3 +166,92 @@ class AccessRight(models.Model): def __unicode__(self): return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) + + +class ResourceLog(models.Model): + timestamp = models.DateTimeField(auto_now_add=True) + resource_type = models.ForeignKey(ContentType, blank=True, null=True) + resource_pk = models.PositiveIntegerField(blank=True, null=True) + resource = fields.GenericForeignKey('resource_type', 'resource_pk') + connector = models.CharField(max_length=128, verbose_name='connector', null=True) + appname = models.CharField(max_length=128, verbose_name='appname', null=True) + slug = models.CharField(max_length=128, verbose_name='slug', null=True) + loglevel = models.CharField(max_length=16, verbose_name='log level') + ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Address')) + message = models.TextField(max_length=2048, verbose_name='message') + extra = jsonfield.JSONField(verbose_name='extras', default={}) + + class Meta: + permissions = ( + ('view_resourcelog', 'Can view resource logs'), + ) + + +class ProxyLogger(object): + + def __init__(self, level, classname=None, appname=None, slug=None): + self.classname = classname + self.appname = appname + self.slug = slug + if classname: + logger_name = 'passerelle.resource.%s.%s' % ( + slugify(unicode(self.classname.lower())), self.slug) + else: + logger_name = 'passerelle.resource' + + self._logger = logging.getLogger(logger_name) + self._logger.setLevel(level) + + def _log(self, levelname, message, request=None, **extra): + attr = {} + attr['loglevel'] = levelname + attr['message'] = message + attr['extra'] = extra.get('extra', {}) + + if getattr(request, 'META', None): + if 'HTTP_X_FORWARDED_FOR' in request.META: + ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() + else: + ipsource = request.META.get('REMOTE_ADDR') + else: + ipsource = None + + if self.classname and self.slug: + connector = '%s-%s' % (self.classname.lower(), self.slug) + klass = ContentType.objects.get(model=self.classname.lower()) + klass_pk = klass.pk + # instance = ContentType.get_object_for_this_type(klass, slug=self.slug) + else: + connector = None + klass = None + klass_pk = None + + attr['ipsource'] = ipsource + attr['connector'] = connector + attr['resource_type'] = klass + attr['resource_pk'] = klass_pk + + # Resource Custom DB Loggger + if self._logger.level <= getattr(logging, levelname.upper()): + ResourceLog.objects.create(**attr) + + # Default Resource Logger + getattr(self._logger, levelname)(message, extra=extra.get('extra', {})) + + def debug(self, message, request=None, **extra): + self._log('debug', message, request, **extra) + + def info(self, message, request=None, **extra): + self._log('info', message, request, **extra) + + def warning(self, message, request=None, **extra): + self._log('warning', message, request, **extra) + + def error(self, message, request=None, **extra): + self._log('error', message, request, **extra) + + def critical(self, message, request=None, **extra): + self._log('critical', message, request, **extra) + + def fatal(self, message, request=None, **extra): + self._log('fatal', message, request, **extra) diff --git a/passerelle/settings.py b/passerelle/settings.py index 2830f08..f14d69f 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -173,8 +173,8 @@ LOGGING = { 'handlers': { 'console': { 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - }, + 'class': 'logging.StreamHandler' + }, }, 'loggers': { 'django.request': { diff --git a/passerelle/views.py b/passerelle/views.py index ddbc511..53d9793 100644 --- a/passerelle/views.py +++ b/passerelle/views.py @@ -228,6 +228,7 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): payload = request.body[:5000] connector.logger.debug('endpoint %s %s (%r) ' % (request.method, url, payload), + request=request, extra={ 'connector': connector_name, 'connector_endpoint': endpoint_name, diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py index 45e1460..ff6e4d7 100644 --- a/tests/test_generic_endpoint.py +++ b/tests/test_generic_endpoint.py @@ -25,7 +25,9 @@ import pytest import utils +from passerelle.base.models import ResourceLog, ProxyLogger from passerelle.contrib.mdel.models import MDEL +from passerelle.contrib.arcgis.models import Arcgis @pytest.fixture @@ -33,6 +35,11 @@ def mdel(db): return utils.setup_access_rights(MDEL.objects.create(slug='test')) +@pytest.fixture +def arcgis(db): + return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) + + DEMAND_STATUS = { 'closed': True, 'status': 'accepted', @@ -57,7 +64,7 @@ def test_generic_payload_logging(caplog, app, mdel): records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] for record in records: - assert record.module == 'views' + # assert record.module == 'views' assert record.levelname == 'DEBUG' assert record.connector == 'mdel' if record.connector_endpoint_method == 'POST': @@ -67,3 +74,65 @@ def test_generic_payload_logging(caplog, app, mdel): assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message assert record.connector_endpoint == 'status' assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' + + +@mock.patch('passerelle.utils.LoggedRequest.get') +def test_proxy_logger(mocked_get, caplog, app, arcgis): + payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() + mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) + + # simple logger + logger = ProxyLogger('DEBUG') + logger.debug('this is a debug test') + logger.info('this is an info test') + + assert ResourceLog.objects.count() == 2 + for log in ResourceLog.objects.all(): + if log.loglevel == 'debug': + assert log.message == 'this is a debug test' + else: + assert log.message == 'this is an info test' + + resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + + # Resource Custom DB Logger + log = ResourceLog.objects.filter(connector='arcgis-test')[0] + assert log.connector == 'arcgis-test' + assert log.loglevel == 'debug' + assert log.ipsource == '127.0.0.1' + assert log.extra['connector'] == 'arcgis' + assert log.extra['connector_endpoint'] == 'district' + assert log.extra['connector_endpoint_method'] == 'GET' + assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' + + # Resource Generic Logger + for record in caplog.records(): + if record.name != 'passerelle.resource.arcgis.test': + continue + assert record.levelno == 10 + assert record.levelname == 'DEBUG' + assert record.name == 'passerelle.resource.arcgis.test' + assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " + + data = resp.json['data'] + assert data['id'] == 4 + assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' + + # when changing log level + ResourceLog.objects.all().delete() + arcgis.log_level = 'INFO' + arcgis.save() + app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + assert ResourceLog.objects.count() == 0 + + arcgis.logger.info('testing info log message') + assert ResourceLog.objects.count() == 1 + log = ResourceLog.objects.first() + assert log.connector == 'arcgis-test' + assert log.loglevel == 'info' + assert log.message == 'testing info log message' + + arcgis.logger.warning('first warning') + assert ResourceLog.objects.count() == 2 + assert ResourceLog.objects.last().message == 'first warning' + assert ResourceLog.objects.last().loglevel == 'warning' -- 2.11.0