From bbb7e430625d2f3f7adb509780b5d95d89da21b2 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 5 Dec 2016 10:14:27 +0100 Subject: [PATCH 1/2] add integrated log system (#14191) --- passerelle/base/migrations/0005_resourcelog.py | 31 ++++++++ passerelle/base/models.py | 90 ++++++++++++++++++++-- passerelle/base/templatetags/passerelle.py | 25 +++++- passerelle/static/css/style.css | 28 +++++++ .../passerelle/includes/resource-logs-table.html | 46 +++++++++++ .../templates/passerelle/manage/service_view.html | 11 +++ passerelle/views.py | 1 + tests/test_generic_endpoint.py | 72 ++++++++++++++++- 8 files changed, 295 insertions(+), 9 deletions(-) create mode 100644 passerelle/base/migrations/0005_resourcelog.py create mode 100644 passerelle/templates/passerelle/includes/resource-logs-table.html diff --git a/passerelle/base/migrations/0005_resourcelog.py b/passerelle/base/migrations/0005_resourcelog.py new file mode 100644 index 0000000..d945ea1 --- /dev/null +++ b/passerelle/base/migrations/0005_resourcelog.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0004_auto_20170117_0326'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), + ('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), + ('levelno', models.IntegerField(verbose_name=b'log level')), + ('sourceip', models.GenericIPAddressField(null=True, verbose_name='Source IP', blank=True)), + ('message', models.TextField(max_length=2048, verbose_name=b'message')), + ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), + ], + options={ + 'permissions': (('view_resourcelog', 'Can view resource logs'),), + }, + ), + ] diff --git a/passerelle/base/models.py b/passerelle/base/models.py index 7abd600..22693a1 100644 --- a/passerelle/base/models.py +++ b/passerelle/base/models.py @@ -8,8 +8,8 @@ from django.core.exceptions import ValidationError, ObjectDoesNotExist, Permissi from django.core.urlresolvers import reverse from django.db import models, transaction from django.db.models import Q -from django.utils.translation import ugettext_lazy as _ from django.utils.text import slugify +from django.utils.translation import ugettext_lazy as _ from django.core.files.base import ContentFile from django.contrib.contenttypes.models import ContentType @@ -19,6 +19,8 @@ from jsonfield import JSONField from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager +import jsonfield + import passerelle import requests @@ -130,10 +132,7 @@ class BaseResource(models.Model): def __init__(self, *args, **kwargs): super(BaseResource, self).__init__(*args, **kwargs) - self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( - slugify(unicode(self.__class__.__name__)), self.slug) - ) - self.logger.setLevel(getattr(logging, self.log_level)) + self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) def __unicode__(self): return self.title @@ -307,5 +306,82 @@ class AccessRight(models.Model): ) def __unicode__(self): - return '%s (on %s <%s>) (for %s)' % (self.codename, - self.resource_type, self.resource_pk, self.apiuser) + return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) + + +class ResourceLog(models.Model): + timestamp = models.DateTimeField(auto_now_add=True) + appname = models.CharField(max_length=128, verbose_name='appname', null=True) + slug = models.CharField(max_length=128, verbose_name='slug', null=True) + levelno = models.IntegerField(verbose_name='log level') + sourceip = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('Source IP')) + message = models.TextField(max_length=2048, verbose_name='message') + extra = jsonfield.JSONField(verbose_name='extras', default={}) + + class Meta: + permissions = ( + ('view_resourcelog', 'Can view resource logs'), + ) + + @property + def level(self): + return slugify(logging.getLevelName(self.levelno)) + + def __unicode__(self): + return '%s %s %s %s' % (self.timestamp, self.levelno, self.appname, self.slug) + + +class ProxyLogger(object): + + def __init__(self, level, appname=None, slug=None): + self.appname = appname + self.slug = slug + if appname: + logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) + else: + logger_name = 'passerelle.resource' + + self._logger = logging.getLogger(logger_name) + self._logger.setLevel(level) + + def _log(self, levelname, message, *args, **kwargs): + levelno = getattr(logging, levelname) + attr = {} + attr['levelno'] = levelno + attr['message'] = message[:ResourceLog._meta.get_field('message').max_length] + attr['appname'] = self.appname + attr['slug'] = self.slug + attr['extra'] = kwargs.get('extra', {}) + request = kwargs.pop('request', None) + + if getattr(request, 'META', None): + if 'HTTP_X_FORWARDED_FOR' in request.META: + sourceip = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() + else: + sourceip = request.META.get('REMOTE_ADDR') + else: + sourceip = None + attr['sourceip'] = sourceip + + if self._logger.level <= levelno: + ResourceLog.objects.create(**attr) + + getattr(self._logger, levelname.lower())(message, *args, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log('DEBUG', message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log('INFO', message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log('WARNING', message, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log('CRITICAL', message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log('ERROR', message, *args, **kwargs) + + def fatal(self, message, *args, **kwargs): + self._log('FATAL', message, *args, **kwargs) diff --git a/passerelle/base/templatetags/passerelle.py b/passerelle/base/templatetags/passerelle.py index a6e94b8..2e51b37 100644 --- a/passerelle/base/templatetags/passerelle.py +++ b/passerelle/base/templatetags/passerelle.py @@ -3,9 +3,10 @@ from __future__ import absolute_import from django import template from django.contrib.contenttypes.models import ContentType from django.contrib.auth import get_permission_codename +from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from passerelle.utils import get_trusted_services -from ..models import AccessRight +from ..models import AccessRight, ResourceLog register = template.Library() @@ -22,6 +23,28 @@ def access_rights_table(context, resource, permission): return context +@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) +def resource_logs_table(context, resource): + request = context.get('request') + page = request.GET.get('page', 1) + + connector = resource.get_connector_slug() + context['connector'] = connector + context['slug'] = resource.slug + qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') + + paginator = Paginator(qs, 10) + try: + logrecords = paginator.page(page) + except PageNotAnInteger: + logrecords = paginator.page(1) + except (EmptyPage,): + logrecords = paginator.page(paginator.num_pages) + + context['logrecords'] = logrecords + return context + + @register.filter def can_edit(obj, user): return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) diff --git a/passerelle/static/css/style.css b/passerelle/static/css/style.css index e8963cb..6b05fa5 100644 --- a/passerelle/static/css/style.css +++ b/passerelle/static/css/style.css @@ -1,5 +1,6 @@ div#queries, div#security, +div#logs, div#endpoints { margin-bottom: 2em; border: 1px solid #bcbcbc; @@ -7,6 +8,7 @@ div#endpoints { div#queries h3, div#security h3, +div#logs h3, div#endpoints h3 { background: #FCFCFC; border-bottom: 1px solid #bcbcbc; @@ -17,12 +19,14 @@ div#endpoints h3 { div#queries > div, div#security > div, +div#logs > div, div#endpoints > div { padding: 1rem; } div#queries ul, div#security ul, +div#logs ul, div#endpoints ul { padding-left: 2em; line-height: 140%; @@ -37,6 +41,30 @@ div#endpoints h4 { border-bottom: 1px solid #bcbcbc; } +div#logs table th, +div#logs table td.timestamp { + white-space: nowrap; +} + +div#logs table td.message { + text-align: left; +} + +div#logs table tr.level-debug { + color: #666; +} + +div#logs table tr.level-warning, +div#logs table tr.level-error, +div#logs table tr.level-critical { + color: #c33; +} + +div#logs table tr.level-error, +div#logs table tr.level-critical { + font-weight: bold; +} + select#id_msg_class { max-width: 30em; } li.webservices a { background-image: url(icons/icon-webservices.png); } diff --git a/passerelle/templates/passerelle/includes/resource-logs-table.html b/passerelle/templates/passerelle/includes/resource-logs-table.html new file mode 100644 index 0000000..b35bfd5 --- /dev/null +++ b/passerelle/templates/passerelle/includes/resource-logs-table.html @@ -0,0 +1,46 @@ +{% load i18n passerelle %} +{% load tz %} + +{% block content %} +{% if logrecords %} + + + + + + + + {% for record in logrecords %} + + + + + + {% endfor %} + +
{% trans 'Timestamp' %}{% trans 'Ip Source' %}{% trans 'Message' %}
{{ record.timestamp|localtime }}{{ record.sourceip|default:"-" }}{{ record.message}}
+ +{% if logrecords.has_other_pages %} +

+ {% if logrecords.has_previous %} + << + {% else %} + << + {% endif %} +   + + {{ logrecords.number }} / {{ logrecords.paginator.num_pages }} + +   + {% if logrecords.has_next %} + >> + {% else %} + >> + {% endif %} + + {% endif %} + +{% else %} +

{% trans 'No records found' %}

+{% endif %} +{% endblock %} diff --git a/passerelle/templates/passerelle/manage/service_view.html b/passerelle/templates/passerelle/manage/service_view.html index d39af39..356ce54 100644 --- a/passerelle/templates/passerelle/manage/service_view.html +++ b/passerelle/templates/passerelle/manage/service_view.html @@ -48,4 +48,15 @@ {% endif %} +{% if perms.base.view_resourcelog %} +
+

{% trans "Logs" %}

+
+ {% block logs %} + {% resource_logs_table resource=object %} + {% endblock %} +
+
+{% endif %} + {% endblock %} diff --git a/passerelle/views.py b/passerelle/views.py index 62e564d..dfd4122 100644 --- a/passerelle/views.py +++ b/passerelle/views.py @@ -253,6 +253,7 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): payload = request.body[:5000] connector.logger.debug('endpoint %s %s (%r) ' % (request.method, url, payload), + request=request, extra={ 'connector': connector_name, 'connector_endpoint': endpoint_name, diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py index b08e0a2..c340d36 100644 --- a/tests/test_generic_endpoint.py +++ b/tests/test_generic_endpoint.py @@ -25,7 +25,9 @@ import pytest import utils +from passerelle.base.models import ResourceLog, ProxyLogger from passerelle.contrib.mdel.models import MDEL +from passerelle.contrib.arcgis.models import Arcgis @pytest.fixture @@ -33,6 +35,11 @@ def mdel(db): return utils.setup_access_rights(MDEL.objects.create(slug='test')) +@pytest.fixture +def arcgis(db): + return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) + + DEMAND_STATUS = { 'closed': True, 'status': 'accepted', @@ -57,7 +64,6 @@ def test_generic_payload_logging(caplog, app, mdel): records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] for record in records: - assert record.module == 'views' assert record.levelname == 'DEBUG' assert record.connector == 'mdel' if record.connector_endpoint_method == 'POST': @@ -67,3 +73,67 @@ def test_generic_payload_logging(caplog, app, mdel): assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message assert record.connector_endpoint == 'status' assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' + + +@mock.patch('passerelle.utils.LoggedRequest.get') +def test_proxy_logger(mocked_get, caplog, app, arcgis): + payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() + mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) + + # simple logger + logger = ProxyLogger('DEBUG') + logger.debug('this is a debug test') + logger.info('this is an info test') + + assert ResourceLog.objects.count() == 2 + for log in ResourceLog.objects.all(): + if log.levelno == 10: + assert log.message == 'this is a debug test' + else: + assert log.message == 'this is an info test' + + resp = app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200) + + logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) + + # Resource Custom DB Logger + log = ResourceLog.objects.filter(appname='arcgis', slug='test').first() + assert log.appname == 'arcgis' + assert log.slug == 'test' + assert log.levelno == 10 + assert log.sourceip == '127.0.0.1' + assert log.extra['connector'] == 'arcgis' + assert log.extra['connector_endpoint'] == 'district' + assert log.extra['connector_endpoint_method'] == 'GET' + assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' + + # Resource Generic Logger + for record in caplog.records(): + if record.name != 'passerelle.resource.arcgis.test': + continue + assert record.levelno == 10 + assert record.levelname == 'DEBUG' + assert record.name == 'passerelle.resource.arcgis.test' + assert u"endpoint GET /arcgis/test/district?" in record.message + + data = resp.json['data'] + assert data['id'] == 4 + assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' + + # when changing log level + ResourceLog.objects.all().delete() + arcgis.log_level = 'INFO' + arcgis.save() + app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200) + assert ResourceLog.objects.count() == 0 + + arcgis.logger.info('testing info log message') + assert ResourceLog.objects.count() == 1 + log = ResourceLog.objects.first() + assert log.levelno == 20 + assert log.message == 'testing info log message' + + arcgis.logger.warning('first warning') + assert ResourceLog.objects.count() == 2 + assert ResourceLog.objects.last().message == 'first warning' + assert ResourceLog.objects.last().levelno == 30 -- 2.11.0