From 8fba0919c8c575f76cd4e602c7da6fa4d3d0b84a Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 5 Dec 2016 10:14:27 +0100 Subject: [PATCH] add integrated log system (#14191) --- passerelle/base/migrations/0003_resourcelog.py | 31 +++++++ passerelle/base/models.py | 98 ++++++++++++++++++++-- passerelle/base/templatetags/passerelle.py | 25 +++++- .../passerelle/includes/resource-logs-table.html | 50 +++++++++++ .../templates/passerelle/manage/service_view.html | 9 ++ passerelle/views.py | 1 + tests/test_generic_endpoint.py | 82 +++++++++++++++++- 7 files changed, 289 insertions(+), 7 deletions(-) create mode 100644 passerelle/base/migrations/0003_resourcelog.py create mode 100644 passerelle/templates/passerelle/includes/resource-logs-table.html diff --git a/passerelle/base/migrations/0003_resourcelog.py b/passerelle/base/migrations/0003_resourcelog.py new file mode 100644 index 0000000..998765e --- /dev/null +++ b/passerelle/base/migrations/0003_resourcelog.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0002_auto_20151009_0326'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), + ('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), + ('levelno', models.IntegerField(verbose_name=b'log level')), + ('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Source', blank=True)), + ('_message', models.TextField(max_length=2048, verbose_name=b'message')), + ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), + ], + options={ + 'permissions': (('view_resourcelog', 'Can view resource logs'),), + }, + ), + ] diff --git a/passerelle/base/models.py b/passerelle/base/models.py index 12dccaf..3cf6041 100644 --- a/passerelle/base/models.py +++ b/passerelle/base/models.py @@ -9,7 +9,6 @@ from django.core.urlresolvers import reverse from django.db import models, transaction from django.db.models import Q from django.utils.translation import ugettext_lazy as _ -from django.utils.text import slugify from django.core.files.base import ContentFile from django.contrib.contenttypes.models import ContentType @@ -19,6 +18,8 @@ from jsonfield import JSONField from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager +import jsonfield + import passerelle KEYTYPE_CHOICES = ( @@ -126,10 +127,7 @@ class BaseResource(models.Model): def __init__(self, *args, **kwargs): super(BaseResource, self).__init__(*args, **kwargs) - self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( - slugify(unicode(self.__class__.__name__)), self.slug) - ) - self.logger.setLevel(getattr(logging, self.log_level)) + self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) def __unicode__(self): return self.title @@ -304,3 +302,93 @@ class AccessRight(models.Model): def __unicode__(self): return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) + + +class ResourceLog(models.Model): + timestamp = models.DateTimeField(auto_now_add=True) + appname = models.CharField(max_length=128, verbose_name='appname', null=True) + slug = models.CharField(max_length=128, verbose_name='slug', null=True) + levelno = models.IntegerField(verbose_name='log level') + ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Source')) + _message = models.TextField(max_length=2048, verbose_name='message') + extra = jsonfield.JSONField(verbose_name='extras', default={}) + + class Meta: + permissions = ( + ('view_resourcelog', 'Can view resource logs'), + ) + + def __unicode__(self): + return '%s %s %s %s' % (self.timestamp, self.levelno, self.appname, self.slug) + + @property + def message(self): + return base64.b64decode(self._message) + + @message.setter + def message(self, value): + self._message = base64.b64encode(value) + + def save(self, *args, **kwargs): + try: + base64.b64decode(self._message) + except (TypeError,): + self._message = base64.encode(self._message) + + return super(ResourceLog, self).save(*args, **kwargs) + + +class ProxyLogger(object): + + def __init__(self, level, appname=None, slug=None): + self.appname = appname + self.slug = slug + if appname: + logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) + else: + logger_name = 'passerelle.resource' + + self._logger = logging.getLogger(logger_name) + self._logger.setLevel(level) + + def _log(self, levelname, message, *args, **kwargs): + levelno = getattr(logging, levelname) + attr = {} + attr['levelno'] = levelno + attr['message'] = message + attr['appname'] = self.appname + attr['slug'] = self.slug + attr['extra'] = kwargs.get('extra', {}) + request = kwargs.pop('request', None) + + if getattr(request, 'META', None): + if 'HTTP_X_FORWARDED_FOR' in request.META: + ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() + else: + ipsource = request.META.get('REMOTE_ADDR') + else: + ipsource = None + attr['ipsource'] = ipsource + + if self._logger.level <= levelno: + ResourceLog.objects.create(**attr) + + getattr(self._logger, levelname.lower())(message, *args, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log('DEBUG', message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log('INFO', message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log('WARNING', message, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log('CRITICAL', message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log('ERROR', message, *args, **kwargs) + + def fatal(self, message, *args, **kwargs): + self._log('FATAL', message, *args, **kwargs) diff --git a/passerelle/base/templatetags/passerelle.py b/passerelle/base/templatetags/passerelle.py index a6e94b8..2e51b37 100644 --- a/passerelle/base/templatetags/passerelle.py +++ b/passerelle/base/templatetags/passerelle.py @@ -3,9 +3,10 @@ from __future__ import absolute_import from django import template from django.contrib.contenttypes.models import ContentType from django.contrib.auth import get_permission_codename +from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from passerelle.utils import get_trusted_services -from ..models import AccessRight +from ..models import AccessRight, ResourceLog register = template.Library() @@ -22,6 +23,28 @@ def access_rights_table(context, resource, permission): return context +@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) +def resource_logs_table(context, resource): + request = context.get('request') + page = request.GET.get('page', 1) + + connector = resource.get_connector_slug() + context['connector'] = connector + context['slug'] = resource.slug + qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') + + paginator = Paginator(qs, 10) + try: + logrecords = paginator.page(page) + except PageNotAnInteger: + logrecords = paginator.page(1) + except (EmptyPage,): + logrecords = paginator.page(paginator.num_pages) + + context['logrecords'] = logrecords + return context + + @register.filter def can_edit(obj, user): return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) diff --git a/passerelle/templates/passerelle/includes/resource-logs-table.html b/passerelle/templates/passerelle/includes/resource-logs-table.html new file mode 100644 index 0000000..3194e33 --- /dev/null +++ b/passerelle/templates/passerelle/includes/resource-logs-table.html @@ -0,0 +1,50 @@ +{% load i18n passerelle %} +{% load tz %} + +{% block content %} +{% if logrecords %} + + + + + + + + + + {% for record in logrecords %} + + + + + + + + {% endfor %} + +
{% trans 'Id' %}{% trans 'Timestamp' %}{% trans 'Level' %}{% trans 'Ip Source' %}{% trans 'Message' %}
{{ record.id }}{{ record.timestamp|localtime }}{{ record.loglevel }}{{ record.ipsource }}{{ record.message}}
+ +{% if logrecords.has_other_pages %} +

+ {% if logrecords.has_previous %} + << + {% else %} + << + {% endif %} +   + + {{ logrecords.number }} / {{ logrecords.paginator.num_pages }} + +   + {% if logrecords.has_next %} + >> + {% else %} + >> + {% endif %} + + {% endif %} + +{% else %} +

{% trans 'No records found' %}

+{% endif %} +{% endblock %} diff --git a/passerelle/templates/passerelle/manage/service_view.html b/passerelle/templates/passerelle/manage/service_view.html index d39af39..9d2abb7 100644 --- a/passerelle/templates/passerelle/manage/service_view.html +++ b/passerelle/templates/passerelle/manage/service_view.html @@ -48,4 +48,13 @@ {% endif %} +{% if perms.base.view_resourcelog %} +
+

{% trans "Logs" %}

+ {% block logs %} + {% resource_logs_table resource=object %} + {% endblock %} +
+{% endif %} + {% endblock %} diff --git a/passerelle/views.py b/passerelle/views.py index 0620640..57eec23 100644 --- a/passerelle/views.py +++ b/passerelle/views.py @@ -246,6 +246,7 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): payload = request.body[:5000] connector.logger.debug('endpoint %s %s (%r) ' % (request.method, url, payload), + request=request, extra={ 'connector': connector_name, 'connector_endpoint': endpoint_name, diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py index 45e1460..0b5a739 100644 --- a/tests/test_generic_endpoint.py +++ b/tests/test_generic_endpoint.py @@ -19,13 +19,16 @@ from __future__ import unicode_literals import os import json +import base64 import mock import pytest import utils +from passerelle.base.models import ResourceLog, ProxyLogger from passerelle.contrib.mdel.models import MDEL +from passerelle.contrib.arcgis.models import Arcgis @pytest.fixture @@ -33,6 +36,11 @@ def mdel(db): return utils.setup_access_rights(MDEL.objects.create(slug='test')) +@pytest.fixture +def arcgis(db): + return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) + + DEMAND_STATUS = { 'closed': True, 'status': 'accepted', @@ -57,7 +65,6 @@ def test_generic_payload_logging(caplog, app, mdel): records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] for record in records: - assert record.module == 'views' assert record.levelname == 'DEBUG' assert record.connector == 'mdel' if record.connector_endpoint_method == 'POST': @@ -67,3 +74,76 @@ def test_generic_payload_logging(caplog, app, mdel): assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message assert record.connector_endpoint == 'status' assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' + + +@mock.patch('passerelle.utils.LoggedRequest.get') +def test_proxy_logger(mocked_get, caplog, app, arcgis): + payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() + mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) + + # simple logger + logger = ProxyLogger('DEBUG') + logger.debug('this is a debug test') + logger.info('this is an info test') + + assert ResourceLog.objects.count() == 2 + for log in ResourceLog.objects.all(): + if log.levelno == 10: + assert log.message == 'this is a debug test' + else: + assert log.message == 'this is an info test' + + resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + + logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) + + # Resource Custom DB Logger + log = ResourceLog.objects.filter(appname='arcgis', slug='test').first() + assert log.appname == 'arcgis' + assert log.slug == 'test' + assert log.levelno == 10 + assert log.ipsource == '127.0.0.1' + assert log.extra['connector'] == 'arcgis' + assert log.extra['connector_endpoint'] == 'district' + assert log.extra['connector_endpoint_method'] == 'GET' + assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' + + # Resource Generic Logger + for record in caplog.records(): + if record.name != 'passerelle.resource.arcgis.test': + continue + assert record.levelno == 10 + assert record.levelname == 'DEBUG' + assert record.name == 'passerelle.resource.arcgis.test' + assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " + + data = resp.json['data'] + assert data['id'] == 4 + assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' + + # when changing log level + ResourceLog.objects.all().delete() + arcgis.log_level = 'INFO' + arcgis.save() + app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + assert ResourceLog.objects.count() == 0 + + arcgis.logger.info('testing info log message') + assert ResourceLog.objects.count() == 1 + log = ResourceLog.objects.first() + assert log.levelno == 20 + assert log.message == 'testing info log message' + + arcgis.logger.warning('first warning') + assert ResourceLog.objects.count() == 2 + assert ResourceLog.objects.last().message == 'first warning' + assert ResourceLog.objects.last().levelno == 30 + + # when logging a file + ResourceLog.objects.all().delete() + filename = os.path.join(os.path.dirname(__file__), 'data', 'iparapheur_test.pdf') + message = '(%r)' % file(filename).read() + logger.debug(message) + assert ResourceLog.objects.count() == 1 + assert ResourceLog.objects.first().levelno == 10 + assert ResourceLog.objects.first()._message == base64.b64encode(message) -- 2.11.0