0001-add-integrated-log-system-14191.patch

Josué Kouka, 18 janvier 2017 16:24

Voir les différences: en ligne côte à côte

Subject: [PATCH] add integrated log system (#14191)

 passerelle/base/migrations/0003_resourcelog.py     | 32 ++++++++
 passerelle/base/models.py                          | 92 ++++++++++++++++++++--
 passerelle/base/urls.py                            |  3 +
 passerelle/settings.py                             |  4 +-
 .../templates/passerelle/manage/log_detail.html    | 39 +++++++++
 .../templates/passerelle/manage/log_view.html      | 58 ++++++++++++++
 .../templates/passerelle/manage/service_view.html  |  3 +
 passerelle/urls.py                                 | 12 ++-
 passerelle/views.py                                | 26 +++++-
 tests/test_generic_endpoint.py                     | 75 +++++++++++++++++-
 10 files changed, 333 insertions(+), 11 deletions(-)
 create mode 100644 passerelle/base/migrations/0003_resourcelog.py
 create mode 100644 passerelle/templates/passerelle/manage/log_detail.html
 create mode 100644 passerelle/templates/passerelle/manage/log_view.html

     # -*- coding: utf-8 -*-
     from __future__ import unicode_literals
     from django.db import migrations, models
     import jsonfield.fields
     class Migration(migrations.Migration):
         dependencies = [
             ('base', '0002_auto_20151009_0326'),
+        ]
         operations = [
             migrations.CreateModel(
                 name='ResourceLog',
                 fields=[
                     ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                     ('timestamp', models.DateTimeField(auto_now_add=True)),
                     ('connector', models.CharField(max_length=128, null=True, verbose_name=b'connector')),
                     ('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')),
                     ('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')),
                     ('loglevel', models.CharField(max_length=16, verbose_name=b'log level')),
                     ('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Address', blank=True)),
                     ('message', models.TextField(max_length=2048, verbose_name=b'message')),
                     ('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')),
                 ],
                 options={
                     'permissions': (('view_resourcelog', 'Can view resource logs'),),
                 },
             ),
+        ]

     from django.db import models
     from django.db.models import Q
     from django.utils.translation import ugettext_lazy as _
     from django.utils.text import slugify
     from django.contrib.contenttypes.models import ContentType
     from django.contrib.contenttypes import fields
     from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager
     import jsonfield
     import passerelle
     KEYTYPE_CHOICES = (
-...
         def __init__(self, *args, **kwargs):
             super(BaseResource, self).__init__(*args, **kwargs)
             self.logger = logging.getLogger('passerelle.resource.%s.%s' % (
                 slugify(unicode(self.__class__.__name__)), self.slug)
+            )
             self.logger.setLevel(getattr(logging, self.log_level))
             self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug)
         def __unicode__(self):
             return self.title
-...
         def __unicode__(self):
             return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser)
     class ResourceLog(models.Model):
         timestamp = models.DateTimeField(auto_now_add=True)
         connector = models.CharField(max_length=128, verbose_name='connector', null=True)
         appname = models.CharField(max_length=128, verbose_name='appname', null=True)
         slug = models.CharField(max_length=128, verbose_name='slug', null=True)
         loglevel = models.CharField(max_length=16, verbose_name='log level')
         ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Address'))
         message = models.TextField(max_length=2048, verbose_name='message')
         extra = jsonfield.JSONField(verbose_name='extras', default={})
         class Meta:
             permissions = (
                 ('view_resourcelog', 'Can view resource logs'),
+            )
         def __unicode__(self):
             return '%s %s %s' % (self.timestamp, self.loglevel, self.connector)
     class ProxyLogger(object):
         def __init__(self, level, appname=None, slug=None):
             self.appname = appname
             self.slug = slug
             self.levelname = level
             if appname:
                 logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug)
             else:
                 logger_name = 'passerelle.resource'
             self._logger = logging.getLogger(logger_name)
             self._logger.setLevel(level)
         def _log(self, levelname, message, *args, **kwargs):
             attr = {}
             attr['loglevel'] = levelname
             attr['message'] = message
             attr['appname'] = self.appname
             attr['slug'] = self.slug
             attr['extra'] = kwargs.get('extra', {})
             request = kwargs.pop('request', None)
             if getattr(request, 'META', None):
                 if 'HTTP_X_FORWARDED_FOR' in request.META:
                     ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip()
                 else:
                     ipsource = request.META.get('REMOTE_ADDR')
             else:
                 ipsource = None
             if self.appname and self.slug:
                 connector = '%s-%s' % (self.appname, self.slug)
             else:
                 connector = None
             attr['ipsource'] = ipsource
             attr['connector'] = connector
             # Resource Custom DB Loggger
             if self._logger.level <= getattr(logging, levelname):
                 ResourceLog.objects.create(**attr)
             # Default Resource Logger
             getattr(self._logger, levelname.lower())(message, *args, **kwargs)
         def debug(self, message, *args, **kwargs):
             self._log('DEBUG', message, *args, **kwargs)
         def info(self, message, *args, **kwargs):
             self._log('INFO', message, *args, **kwargs)
         def warning(self, message, *args, **kwargs):
             self._log('WARNING', message, *args, **kwargs)
         def critical(self, message, *args, **kwargs):
             self._log('CRITICAL', message, *args, **kwargs)
         def error(self, message, *args, **kwargs):
             self._log('ERROR', message, *args, **kwargs)
         def fatal(self, message, *args, **kwargs):
             self._log('FATAL', message, *args, **kwargs)

passerelle/base/urls.py
14	14	url(r'^accessright/add/(?P<resource_type>[\w,-]+)/(?P<resource_pk>[\w,-]+)/(?P<codename>[\w,-]+)/',
15	15	AccessRightCreateView.as_view(), name='access-right-add')
16	16	)
	17
	18	# logs_urlpatterns = patterns('',
	19	# url(r'^$', ResourceLogListView.as_view(), name='resourcelog-list'),)

         'handlers': {
             'console': {
                 'level': 'DEBUG',
                 'class': 'logging.StreamHandler',
                 },
                 'class': 'logging.StreamHandler'
             },
         },
         'loggers': {
             'django.request': {

     {% extends "passerelle/manage.html" %}
     {% load i18n passerelle %}
     {% load tz %}
     {% block content %}
     <table class="main">
         <tbody>
             <tr>
                 <td>{% trans 'Timestamp' %}</td>
                 <td>{{object.timestamp}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Level' %}</td>
                 <td>{{object.loglevel|upper}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Ip Source' %}</td>
                 <td>{{object.ipsource}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Message' %}</td>
                 <td>{{object.message}}</td>
             </tr>
             <tr>
                 <td>{% trans 'Extra' %}</td>
                 <td>
                     <table class="main">
                         {% for key, value in object.extra.items%}
                         <tr>
                             <td>{{key}}</td>
                             <td>{{value}}</td>
                         </tr>
                         {% endfor %}
                     </table>
                 </td>
             </tr>
         </tbody>
     </table>
     {% endblock %}

     {% extends "passerelle/manage.html" %}
     {% load i18n passerelle %}
     {% load tz %}
     {% block appbar %}
         <h2>{% trans 'Log Records'%}</h2>
     {% endblock %}
     {% block content %}
     {% if object_list %}
     <table class="main">
         <thead>
             <th>{% trans 'Id' %}</th>
             <th>{% trans 'Timestamp' %}</th>
             <th>{% trans 'Level' %}</th>
             <th>{% trans 'Ip Source' %}</th>
             <th>{% trans 'Message preview' %}</th>
         </thead>
         <tbody>
         {% for record in object_list %}
         <tr>
             <td>
                 {% url 'connector-log-detail' connector slug record.id as record_detail %}
                 <a href="{{record_detail}}">{{record.id}}</a>
             </td>
             <td>{{ record.timestamp|localtime }}</td>
             <td>{{ record.loglevel }}</td>
             <td>{{ record.ipsource }}</td>
             <td>{{ record.message|truncatechars:64 }}</td>
         </tr>
         {% endfor %}
         </tbody>
     </table>
     {% if is_paginated %}
     <p class="paginator">
       {% if page_obj.has_previous %}
           <a href="?page={{ page_obj.previous_page_number }}">&lt;&lt;</a>
       {% else %}
       <span>&lt;&lt;</span>
       {% endif %}
         &nbsp;
         <span class="current">
             {{ page_obj.number }} / {{ page_obj.paginator.num_pages }}
         </span>
          &nbsp;
         {% if page_obj.has_next %}
             <a href="?page={{ page_obj.next_page_number }}">&gt;&gt;</a>
         {% else %}
             <span>&gt;&gt;</span>
         {% endif %}
             </div>
         {% endif %}
     {% else %}
     <p>{% trans 'No records found' %}</p>
     {% endif %}
     {% endblock %}

passerelle/templates/passerelle/manage/service_view.html
48	48	{% endif %}
49	49	</div>
50	50
	51	<div id="logs">
	52	</div>
	53
51	54	{% endblock %}

     from .views import (HomePageView, ManageView, ManageAddView,
             GenericCreateConnectorView, GenericDeleteConnectorView,
             GenericEditConnectorView, GenericEndpointView, GenericConnectorView,
             LEGACY_APPS_PATTERNS, LegacyPageView, login, logout)
             LEGACY_APPS_PATTERNS, LegacyPageView, login, logout,
             GenericConnectorLogDetailView, GenericConnectorLogView)
     from .urls_utils import decorated_includes, required, app_enabled, manager_required
     from .base.urls import access_urlpatterns
     from .plugins import register_apps_urls
-...
     import choosit.urls
     import pastell.urls
     admin.autodiscover()
     urlpatterns = patterns('',
-...
                     GenericEditConnectorView.as_view(), name='edit-connector'),
             )))))
     # Intagrated logs
     urlpatterns += patterns('',
         url(r'^(?P<connector>[\w,-]+)/(?P<slug>[\w,-]+)/_logs/$',
             GenericConnectorLogView.as_view(), name='connector-log-list'),
         url(r'^(?P<connector>[\w,-]+)/(?P<slug>[\w,-]+)/_logs/(?P<pk>[\w,-]+)$',
             GenericConnectorLogDetailView.as_view(), name='connector-log-detail'),
+    )
     urlpatterns += patterns('',
         url(r'^(?P<connector>[\w,-]+)/(?P<slug>[\w,-]+)/$',
             GenericConnectorView.as_view(), name='view-connector'),

     from django.http import HttpResponseBadRequest, HttpResponseRedirect, Http404
     from django.views.decorators.csrf import csrf_exempt
     from django.views.generic import (RedirectView, View, TemplateView, CreateView,
             DeleteView, UpdateView, DetailView)
             DeleteView, UpdateView, DetailView, ListView)
     from django.views.generic.detail import SingleObjectMixin
     from django.conf import settings
     from django.db import models
-...
     else:
         get_idps = lambda: []
     from passerelle.base.models import BaseResource
     from passerelle.base.models import BaseResource, ResourceLog
     from .utils import to_json, response_for_json, is_authorized
     from .forms import GenericConnectorForm
-...
             payload = request.body[:5000]
             connector.logger.debug('endpoint %s %s (%r) ' %
                                    (request.method, url, payload),
                                    request=request,
                                    extra={
                                        'connector': connector_name,
                                        'connector_endpoint': endpoint_name,
-...
             return self.get(request, *args, **kwargs)
     class GenericConnectorLogView(ListView):
         template_name = 'passerelle/manage/log_view.html'
         model = ResourceLog
         paginate_by = 10
         def get_context_data(self, **kwargs):
             context = super(GenericConnectorLogView, self).get_context_data(**kwargs)
             context.update(self.kwargs)
             return context
         def get_queryset(self, **kwargs):
             connector = self.kwargs.get('connector')
             slug = self.kwargs.get('slug')
             return ResourceLog.objects.filter(appname=connector, slug=slug).order_by('-timestamp')
     class GenericConnectorLogDetailView(DetailView):
         template_name = 'passerelle/manage/log_detail.html'
         model = ResourceLog
     # legacy
     LEGACY_APPS_PATTERNS = {
         'datasources': {'url': 'data', 'name': 'Data Sources'},

     import utils
     from passerelle.base.models import ResourceLog, ProxyLogger
     from passerelle.contrib.mdel.models import MDEL
     from passerelle.contrib.arcgis.models import Arcgis
     @pytest.fixture
-...
         return utils.setup_access_rights(MDEL.objects.create(slug='test'))
     @pytest.fixture
     def arcgis(db):
         return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG'))
     DEMAND_STATUS = {
         'closed': True,
         'status': 'accepted',
-...
         records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test']
         for record in records:
             assert record.module == 'views'
             # assert record.module == 'views'
             assert record.levelname == 'DEBUG'
             assert record.connector == 'mdel'
             if record.connector_endpoint_method == 'POST':
-...
                 assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message
                 assert record.connector_endpoint == 'status'
                 assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA'
     @mock.patch('passerelle.utils.LoggedRequest.get')
     def test_proxy_logger(mocked_get, caplog, app, arcgis):
         payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read()
         mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200)
         # simple logger
         logger = ProxyLogger('DEBUG')
         logger.debug('this is a debug test')
         logger.info('this is an info test')
         assert ResourceLog.objects.count() == 2
         for log in ResourceLog.objects.all():
             if log.loglevel == 'DEBUG':
                 assert log.message == 'this is a debug test'
             else:
                 assert log.message == 'this is an info test'
         resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200)
         logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45)
         # Resource Custom DB Logger
         log = ResourceLog.objects.filter(connector='arcgis-test')[0]
         assert log.connector == 'arcgis-test'
         assert log.appname == 'arcgis'
         assert log.slug == 'test'
         assert log.loglevel == 'DEBUG'
         assert log.ipsource == '127.0.0.1'
         assert log.extra['connector'] == 'arcgis'
         assert log.extra['connector_endpoint'] == 'district'
         assert log.extra['connector_endpoint_method'] == 'GET'
         assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122'
         # Resource Generic Logger
         for record in caplog.records():
             if record.name != 'passerelle.resource.arcgis.test':
                 continue
             assert record.levelno == 10
             assert record.levelname == 'DEBUG'
             assert record.name == 'passerelle.resource.arcgis.test'
             assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') "
         data = resp.json['data']
         assert data['id'] == 4
         assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT'
         # when changing log level
         ResourceLog.objects.all().delete()
         arcgis.log_level = 'INFO'
         arcgis.save()
         app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200)
         assert ResourceLog.objects.count() == 0
         arcgis.logger.info('testing info log message')
         assert ResourceLog.objects.count() == 1
         log = ResourceLog.objects.first()
         assert log.connector == 'arcgis-test'
         assert log.loglevel == 'INFO'
         assert log.message == 'testing info log message'
         arcgis.logger.warning('first warning')
         assert ResourceLog.objects.count() == 2
         assert ResourceLog.objects.last().message == 'first warning'
         assert ResourceLog.objects.last().loglevel == 'WARNING'
+    -

Projet

Général

Profil

Produits Entr'ouvert » Passerelle

0001-add-integrated-log-system-14191.patch