From a0cdb51af6fd015dba19c27def2a2d093efd5e00 Mon Sep 17 00:00:00 2001 From: Josue Kouka Date: Mon, 5 Dec 2016 10:14:27 +0100 Subject: [PATCH] add integrated log system (#14191) --- passerelle/base/migrations/0003_resourcelog.py | 32 ++++++++ passerelle/base/models.py | 92 ++++++++++++++++++++-- passerelle/base/templatetags/passerelle.py | 25 +++++- passerelle/settings.py | 4 +- .../passerelle/includes/resource-logs-table.html | 53 +++++++++++++ .../templates/passerelle/manage/log_detail.html | 39 +++++++++ .../templates/passerelle/manage/service_view.html | 12 ++- passerelle/urls.py | 10 ++- passerelle/views.py | 8 +- tests/test_generic_endpoint.py | 74 ++++++++++++++++- 10 files changed, 337 insertions(+), 12 deletions(-) create mode 100644 passerelle/base/migrations/0003_resourcelog.py create mode 100644 passerelle/templates/passerelle/includes/resource-logs-table.html create mode 100644 passerelle/templates/passerelle/manage/log_detail.html diff --git a/passerelle/base/migrations/0003_resourcelog.py b/passerelle/base/migrations/0003_resourcelog.py new file mode 100644 index 0000000..7591724 --- /dev/null +++ b/passerelle/base/migrations/0003_resourcelog.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0002_auto_20151009_0326'), + ] + + operations = [ + migrations.CreateModel( + name='ResourceLog', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('connector', models.CharField(max_length=128, null=True, verbose_name=b'connector')), + ('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), + ('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), + ('loglevel', models.CharField(max_length=16, verbose_name=b'log level')), + ('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Address', blank=True)), + ('message', models.TextField(max_length=2048, verbose_name=b'message')), + ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), + ], + options={ + 'permissions': (('view_resourcelog', 'Can view resource logs'),), + }, + ), + ] diff --git a/passerelle/base/models.py b/passerelle/base/models.py index fe6ac3f..7841993 100644 --- a/passerelle/base/models.py +++ b/passerelle/base/models.py @@ -6,13 +6,14 @@ from django.core.urlresolvers import reverse from django.db import models from django.db.models import Q from django.utils.translation import ugettext_lazy as _ -from django.utils.text import slugify from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes import fields from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager +import jsonfield + import passerelle KEYTYPE_CHOICES = ( @@ -97,10 +98,7 @@ class BaseResource(models.Model): def __init__(self, *args, **kwargs): super(BaseResource, self).__init__(*args, **kwargs) - self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( - slugify(unicode(self.__class__.__name__)), self.slug) - ) - self.logger.setLevel(getattr(logging, self.log_level)) + self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) def __unicode__(self): return self.title @@ -166,3 +164,87 @@ class AccessRight(models.Model): def __unicode__(self): return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) + + +class ResourceLog(models.Model): + timestamp = models.DateTimeField(auto_now_add=True) + connector = models.CharField(max_length=128, verbose_name='connector', null=True) + appname = models.CharField(max_length=128, verbose_name='appname', null=True) + slug = models.CharField(max_length=128, verbose_name='slug', null=True) + loglevel = models.CharField(max_length=16, verbose_name='log level') + ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Address')) + message = models.TextField(max_length=2048, verbose_name='message') + extra = jsonfield.JSONField(verbose_name='extras', default={}) + + class Meta: + permissions = ( + ('view_resourcelog', 'Can view resource logs'), + ) + + def __unicode__(self): + return '%s %s %s' % (self.timestamp, self.loglevel, self.connector) + + +class ProxyLogger(object): + + def __init__(self, level, appname=None, slug=None): + self.appname = appname + self.slug = slug + self.levelname = level + if appname: + logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) + else: + logger_name = 'passerelle.resource' + + self._logger = logging.getLogger(logger_name) + self._logger.setLevel(level) + + def _log(self, levelname, message, *args, **kwargs): + attr = {} + attr['loglevel'] = levelname + attr['message'] = message + attr['appname'] = self.appname + attr['slug'] = self.slug + attr['extra'] = kwargs.get('extra', {}) + request = kwargs.pop('request', None) + + if getattr(request, 'META', None): + if 'HTTP_X_FORWARDED_FOR' in request.META: + ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() + else: + ipsource = request.META.get('REMOTE_ADDR') + else: + ipsource = None + + if self.appname and self.slug: + connector = '%s-%s' % (self.appname, self.slug) + else: + connector = None + + attr['ipsource'] = ipsource + attr['connector'] = connector + + # Resource Custom DB Loggger + if self._logger.level <= getattr(logging, levelname): + ResourceLog.objects.create(**attr) + + # Default Resource Logger + getattr(self._logger, levelname.lower())(message, *args, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log('DEBUG', message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log('INFO', message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log('WARNING', message, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log('CRITICAL', message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log('ERROR', message, *args, **kwargs) + + def fatal(self, message, *args, **kwargs): + self._log('FATAL', message, *args, **kwargs) diff --git a/passerelle/base/templatetags/passerelle.py b/passerelle/base/templatetags/passerelle.py index a6e94b8..2e51b37 100644 --- a/passerelle/base/templatetags/passerelle.py +++ b/passerelle/base/templatetags/passerelle.py @@ -3,9 +3,10 @@ from __future__ import absolute_import from django import template from django.contrib.contenttypes.models import ContentType from django.contrib.auth import get_permission_codename +from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from passerelle.utils import get_trusted_services -from ..models import AccessRight +from ..models import AccessRight, ResourceLog register = template.Library() @@ -22,6 +23,28 @@ def access_rights_table(context, resource, permission): return context +@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) +def resource_logs_table(context, resource): + request = context.get('request') + page = request.GET.get('page', 1) + + connector = resource.get_connector_slug() + context['connector'] = connector + context['slug'] = resource.slug + qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') + + paginator = Paginator(qs, 10) + try: + logrecords = paginator.page(page) + except PageNotAnInteger: + logrecords = paginator.page(1) + except (EmptyPage,): + logrecords = paginator.page(paginator.num_pages) + + context['logrecords'] = logrecords + return context + + @register.filter def can_edit(obj, user): return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) diff --git a/passerelle/settings.py b/passerelle/settings.py index 2830f08..f14d69f 100644 --- a/passerelle/settings.py +++ b/passerelle/settings.py @@ -173,8 +173,8 @@ LOGGING = { 'handlers': { 'console': { 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - }, + 'class': 'logging.StreamHandler' + }, }, 'loggers': { 'django.request': { diff --git a/passerelle/templates/passerelle/includes/resource-logs-table.html b/passerelle/templates/passerelle/includes/resource-logs-table.html new file mode 100644 index 0000000..f4bed70 --- /dev/null +++ b/passerelle/templates/passerelle/includes/resource-logs-table.html @@ -0,0 +1,53 @@ +{% load i18n passerelle %} +{% load tz %} + +{% block content %} +{% if logrecords %} + + + + + + + + + + {% for record in logrecords %} + + + + + + + + {% endfor %} + +
{% trans 'Id' %}{% trans 'Timestamp' %}{% trans 'Level' %}{% trans 'Ip Source' %}{% trans 'Message preview' %}
+ {% url 'connector-log-detail' connector slug record.id as record_detail %} + {{record.id}} + {{ record.timestamp|localtime }}{{ record.loglevel }}{{ record.ipsource }}{{ record.message|truncatechars:64 }}
+ +{% if logrecords.has_other_pages %} +

+ {% if logrecords.has_previous %} + << + {% else %} + << + {% endif %} +   + + {{ logrecords.number }} / {{ logrecords.paginator.num_pages }} + +   + {% if logrecords.has_next %} + >> + {% else %} + >> + {% endif %} + + {% endif %} + +{% else %} +

{% trans 'No records found' %}

+{% endif %} +{% endblock %} diff --git a/passerelle/templates/passerelle/manage/log_detail.html b/passerelle/templates/passerelle/manage/log_detail.html new file mode 100644 index 0000000..d0c42b0 --- /dev/null +++ b/passerelle/templates/passerelle/manage/log_detail.html @@ -0,0 +1,39 @@ +{% extends "passerelle/manage.html" %} +{% load i18n passerelle %} +{% load tz %} + +{% block content %} + + + + + + + + + + + + + + + + + + + + + + + +
{% trans 'Timestamp' %}{{object.timestamp}}
{% trans 'Level' %}{{object.loglevel|upper}}
{% trans 'Ip Source' %}{{object.ipsource}}
{% trans 'Message' %}{{object.message}}
{% trans 'Extra' %} + + {% for key, value in object.extra.items%} + + + + + {% endfor %} +
{{key}}{{value}}
+
+{% endblock %} diff --git a/passerelle/templates/passerelle/manage/service_view.html b/passerelle/templates/passerelle/manage/service_view.html index d39af39..bfe7de7 100644 --- a/passerelle/templates/passerelle/manage/service_view.html +++ b/passerelle/templates/passerelle/manage/service_view.html @@ -36,7 +36,7 @@ {% block endpoints %} {% endblock %} - +
{% if perms.base.view_accessright %} @@ -48,4 +48,14 @@ {% endif %}
+
+ {% if perms.base.view_resourcelog %} +

{% trans "Logs" %}

+ {% block logs %} + {% resource_logs_table resource=object %} + {% endblock %} + {% endif %} +
{% endblock %} + + diff --git a/passerelle/urls.py b/passerelle/urls.py index 673af0c..7826184 100644 --- a/passerelle/urls.py +++ b/passerelle/urls.py @@ -9,7 +9,8 @@ from django.views.static import serve as static_serve from .views import (HomePageView, ManageView, ManageAddView, GenericCreateConnectorView, GenericDeleteConnectorView, GenericEditConnectorView, GenericEndpointView, GenericConnectorView, - LEGACY_APPS_PATTERNS, LegacyPageView, login, logout) + LEGACY_APPS_PATTERNS, LegacyPageView, login, logout, + GenericConnectorLogDetailView) from .urls_utils import decorated_includes, required, app_enabled, manager_required from .base.urls import access_urlpatterns from .plugins import register_apps_urls @@ -17,6 +18,7 @@ from .plugins import register_apps_urls import choosit.urls import pastell.urls + admin.autodiscover() urlpatterns = patterns('', @@ -84,6 +86,12 @@ urlpatterns += patterns('', GenericEditConnectorView.as_view(), name='edit-connector'), ))))) +# Intagrated logs +urlpatterns += patterns('', + url(r'^(?P[\w,-]+)/(?P[\w,-]+)/_logs/(?P[\w,-]+)$', + GenericConnectorLogDetailView.as_view(), name='connector-log-detail'), +) + urlpatterns += patterns('', url(r'^(?P[\w,-]+)/(?P[\w,-]+)/$', GenericConnectorView.as_view(), name='view-connector'), diff --git a/passerelle/views.py b/passerelle/views.py index ddbc511..6cd441c 100644 --- a/passerelle/views.py +++ b/passerelle/views.py @@ -22,7 +22,7 @@ if 'mellon' in settings.INSTALLED_APPS: else: get_idps = lambda: [] -from passerelle.base.models import BaseResource +from passerelle.base.models import BaseResource, ResourceLog from .utils import to_json, response_for_json, is_authorized from .forms import GenericConnectorForm @@ -228,6 +228,7 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): payload = request.body[:5000] connector.logger.debug('endpoint %s %s (%r) ' % (request.method, url, payload), + request=request, extra={ 'connector': connector_name, 'connector_endpoint': endpoint_name, @@ -257,6 +258,11 @@ class GenericEndpointView(GenericConnectorMixin, SingleObjectMixin, View): return self.get(request, *args, **kwargs) +class GenericConnectorLogDetailView(DetailView): + template_name = 'passerelle/manage/log_detail.html' + model = ResourceLog + + # legacy LEGACY_APPS_PATTERNS = { 'datasources': {'url': 'data', 'name': 'Data Sources'}, diff --git a/tests/test_generic_endpoint.py b/tests/test_generic_endpoint.py index 45e1460..bbdfbd6 100644 --- a/tests/test_generic_endpoint.py +++ b/tests/test_generic_endpoint.py @@ -25,7 +25,9 @@ import pytest import utils +from passerelle.base.models import ResourceLog, ProxyLogger from passerelle.contrib.mdel.models import MDEL +from passerelle.contrib.arcgis.models import Arcgis @pytest.fixture @@ -33,6 +35,11 @@ def mdel(db): return utils.setup_access_rights(MDEL.objects.create(slug='test')) +@pytest.fixture +def arcgis(db): + return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) + + DEMAND_STATUS = { 'closed': True, 'status': 'accepted', @@ -57,7 +64,6 @@ def test_generic_payload_logging(caplog, app, mdel): records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] for record in records: - assert record.module == 'views' assert record.levelname == 'DEBUG' assert record.connector == 'mdel' if record.connector_endpoint_method == 'POST': @@ -67,3 +73,69 @@ def test_generic_payload_logging(caplog, app, mdel): assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message assert record.connector_endpoint == 'status' assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' + + +@mock.patch('passerelle.utils.LoggedRequest.get') +def test_proxy_logger(mocked_get, caplog, app, arcgis): + payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() + mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) + + # simple logger + logger = ProxyLogger('DEBUG') + logger.debug('this is a debug test') + logger.info('this is an info test') + + assert ResourceLog.objects.count() == 2 + for log in ResourceLog.objects.all(): + if log.loglevel == 'DEBUG': + assert log.message == 'this is a debug test' + else: + assert log.message == 'this is an info test' + + resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + + logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) + + # Resource Custom DB Logger + log = ResourceLog.objects.filter(connector='arcgis-test')[0] + assert log.connector == 'arcgis-test' + assert log.appname == 'arcgis' + assert log.slug == 'test' + assert log.loglevel == 'DEBUG' + assert log.ipsource == '127.0.0.1' + assert log.extra['connector'] == 'arcgis' + assert log.extra['connector_endpoint'] == 'district' + assert log.extra['connector_endpoint_method'] == 'GET' + assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' + + # Resource Generic Logger + for record in caplog.records(): + if record.name != 'passerelle.resource.arcgis.test': + continue + assert record.levelno == 10 + assert record.levelname == 'DEBUG' + assert record.name == 'passerelle.resource.arcgis.test' + assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " + + data = resp.json['data'] + assert data['id'] == 4 + assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' + + # when changing log level + ResourceLog.objects.all().delete() + arcgis.log_level = 'INFO' + arcgis.save() + app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) + assert ResourceLog.objects.count() == 0 + + arcgis.logger.info('testing info log message') + assert ResourceLog.objects.count() == 1 + log = ResourceLog.objects.first() + assert log.connector == 'arcgis-test' + assert log.loglevel == 'INFO' + assert log.message == 'testing info log message' + + arcgis.logger.warning('first warning') + assert ResourceLog.objects.count() == 2 + assert ResourceLog.objects.last().message == 'first warning' + assert ResourceLog.objects.last().loglevel == 'WARNING' -- 2.11.0