0001-add-integrated-log-system-14191.patch
passerelle/base/migrations/0005_resourcelog.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations, models |
|
5 |
import jsonfield.fields |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('base', '0004_auto_20170117_0326'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.CreateModel( |
|
16 |
name='ResourceLog', |
|
17 |
fields=[ |
|
18 |
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), |
|
19 |
('timestamp', models.DateTimeField(auto_now_add=True)), |
|
20 |
('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), |
|
21 |
('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), |
|
22 |
('levelno', models.IntegerField(verbose_name=b'log level')), |
|
23 |
('sourceip', models.GenericIPAddressField(null=True, verbose_name='Source IP', blank=True)), |
|
24 |
('message', models.TextField(max_length=2048, verbose_name=b'message')), |
|
25 |
('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), |
|
26 |
], |
|
27 |
options={ |
|
28 |
'permissions': (('view_resourcelog', 'Can view resource logs'),), |
|
29 |
}, |
|
30 |
), |
|
31 |
] |
passerelle/base/models.py | ||
---|---|---|
8 | 8 |
from django.core.urlresolvers import reverse |
9 | 9 |
from django.db import models, transaction |
10 | 10 |
from django.db.models import Q |
11 |
from django.utils.translation import ugettext_lazy as _ |
|
12 | 11 |
from django.utils.text import slugify |
12 |
from django.utils.translation import ugettext_lazy as _ |
|
13 | 13 |
from django.core.files.base import ContentFile |
14 | 14 | |
15 | 15 |
from django.contrib.contenttypes.models import ContentType |
... | ... | |
19 | 19 | |
20 | 20 |
from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager |
21 | 21 | |
22 |
import jsonfield |
|
23 | ||
22 | 24 |
import passerelle |
23 | 25 |
import requests |
24 | 26 | |
... | ... | |
130 | 132 | |
131 | 133 |
def __init__(self, *args, **kwargs): |
132 | 134 |
super(BaseResource, self).__init__(*args, **kwargs) |
133 |
self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( |
|
134 |
slugify(unicode(self.__class__.__name__)), self.slug) |
|
135 |
) |
|
136 |
self.logger.setLevel(getattr(logging, self.log_level)) |
|
135 |
self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) |
|
137 | 136 | |
138 | 137 |
def __unicode__(self): |
139 | 138 |
return self.title |
... | ... | |
307 | 306 |
) |
308 | 307 | |
309 | 308 |
def __unicode__(self): |
310 |
return '%s (on %s <%s>) (for %s)' % (self.codename, |
|
311 |
self.resource_type, self.resource_pk, self.apiuser) |
|
309 |
return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) |
|
310 | ||
311 | ||
312 |
class ResourceLog(models.Model): |
|
313 |
timestamp = models.DateTimeField(auto_now_add=True) |
|
314 |
appname = models.CharField(max_length=128, verbose_name='appname', null=True) |
|
315 |
slug = models.CharField(max_length=128, verbose_name='slug', null=True) |
|
316 |
levelno = models.IntegerField(verbose_name='log level') |
|
317 |
sourceip = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('Source IP')) |
|
318 |
message = models.TextField(max_length=2048, verbose_name='message') |
|
319 |
extra = jsonfield.JSONField(verbose_name='extras', default={}) |
|
320 | ||
321 |
class Meta: |
|
322 |
permissions = ( |
|
323 |
('view_resourcelog', 'Can view resource logs'), |
|
324 |
) |
|
325 | ||
326 |
@property |
|
327 |
def level(self): |
|
328 |
return slugify(logging.getLevelName(self.levelno)) |
|
329 | ||
330 |
def __unicode__(self): |
|
331 |
return '%s %s %s %s' % (self.timestamp, self.levelno, self.appname, self.slug) |
|
332 | ||
333 | ||
334 |
class ProxyLogger(object): |
|
335 | ||
336 |
def __init__(self, level, appname=None, slug=None): |
|
337 |
self.appname = appname |
|
338 |
self.slug = slug |
|
339 |
if appname: |
|
340 |
logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) |
|
341 |
else: |
|
342 |
logger_name = 'passerelle.resource' |
|
343 | ||
344 |
self._logger = logging.getLogger(logger_name) |
|
345 |
self._logger.setLevel(level) |
|
346 | ||
347 |
def _log(self, levelname, message, *args, **kwargs): |
|
348 |
levelno = getattr(logging, levelname) |
|
349 |
attr = {} |
|
350 |
attr['levelno'] = levelno |
|
351 |
attr['message'] = message[:ResourceLog._meta.get_field('message').max_length] |
|
352 |
attr['appname'] = self.appname |
|
353 |
attr['slug'] = self.slug |
|
354 |
attr['extra'] = kwargs.get('extra', {}) |
|
355 |
request = kwargs.pop('request', None) |
|
356 | ||
357 |
if getattr(request, 'META', None): |
|
358 |
if 'HTTP_X_FORWARDED_FOR' in request.META: |
|
359 |
sourceip = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() |
|
360 |
else: |
|
361 |
sourceip = request.META.get('REMOTE_ADDR') |
|
362 |
else: |
|
363 |
sourceip = None |
|
364 |
attr['sourceip'] = sourceip |
|
365 | ||
366 |
if self._logger.level <= levelno: |
|
367 |
ResourceLog.objects.create(**attr) |
|
368 | ||
369 |
getattr(self._logger, levelname.lower())(message, *args, **kwargs) |
|
370 | ||
371 |
def debug(self, message, *args, **kwargs): |
|
372 |
self._log('DEBUG', message, *args, **kwargs) |
|
373 | ||
374 |
def info(self, message, *args, **kwargs): |
|
375 |
self._log('INFO', message, *args, **kwargs) |
|
376 | ||
377 |
def warning(self, message, *args, **kwargs): |
|
378 |
self._log('WARNING', message, *args, **kwargs) |
|
379 | ||
380 |
def critical(self, message, *args, **kwargs): |
|
381 |
self._log('CRITICAL', message, *args, **kwargs) |
|
382 | ||
383 |
def error(self, message, *args, **kwargs): |
|
384 |
self._log('ERROR', message, *args, **kwargs) |
|
385 | ||
386 |
def fatal(self, message, *args, **kwargs): |
|
387 |
self._log('FATAL', message, *args, **kwargs) |
passerelle/base/templatetags/passerelle.py | ||
---|---|---|
3 | 3 |
from django import template |
4 | 4 |
from django.contrib.contenttypes.models import ContentType |
5 | 5 |
from django.contrib.auth import get_permission_codename |
6 |
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
|
6 | 7 | |
7 | 8 |
from passerelle.utils import get_trusted_services |
8 |
from ..models import AccessRight |
|
9 |
from ..models import AccessRight, ResourceLog
|
|
9 | 10 | |
10 | 11 |
register = template.Library() |
11 | 12 | |
... | ... | |
22 | 23 |
return context |
23 | 24 | |
24 | 25 | |
26 |
@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) |
|
27 |
def resource_logs_table(context, resource): |
|
28 |
request = context.get('request') |
|
29 |
page = request.GET.get('page', 1) |
|
30 | ||
31 |
connector = resource.get_connector_slug() |
|
32 |
context['connector'] = connector |
|
33 |
context['slug'] = resource.slug |
|
34 |
qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') |
|
35 | ||
36 |
paginator = Paginator(qs, 10) |
|
37 |
try: |
|
38 |
logrecords = paginator.page(page) |
|
39 |
except PageNotAnInteger: |
|
40 |
logrecords = paginator.page(1) |
|
41 |
except (EmptyPage,): |
|
42 |
logrecords = paginator.page(paginator.num_pages) |
|
43 | ||
44 |
context['logrecords'] = logrecords |
|
45 |
return context |
|
46 | ||
47 | ||
25 | 48 |
@register.filter |
26 | 49 |
def can_edit(obj, user): |
27 | 50 |
return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) |
passerelle/static/css/style.css | ||
---|---|---|
1 | 1 |
div#queries, |
2 | 2 |
div#security, |
3 |
div#logs, |
|
3 | 4 |
div#endpoints { |
4 | 5 |
margin-bottom: 2em; |
5 | 6 |
border: 1px solid #bcbcbc; |
... | ... | |
7 | 8 | |
8 | 9 |
div#queries h3, |
9 | 10 |
div#security h3, |
11 |
div#logs h3, |
|
10 | 12 |
div#endpoints h3 { |
11 | 13 |
background: #FCFCFC; |
12 | 14 |
border-bottom: 1px solid #bcbcbc; |
... | ... | |
17 | 19 | |
18 | 20 |
div#queries > div, |
19 | 21 |
div#security > div, |
22 |
div#logs > div, |
|
20 | 23 |
div#endpoints > div { |
21 | 24 |
padding: 1rem; |
22 | 25 |
} |
23 | 26 | |
24 | 27 |
div#queries ul, |
25 | 28 |
div#security ul, |
29 |
div#logs ul, |
|
26 | 30 |
div#endpoints ul { |
27 | 31 |
padding-left: 2em; |
28 | 32 |
line-height: 140%; |
... | ... | |
37 | 41 |
border-bottom: 1px solid #bcbcbc; |
38 | 42 |
} |
39 | 43 | |
44 |
div#logs table th, |
|
45 |
div#logs table td.timestamp { |
|
46 |
white-space: nowrap; |
|
47 |
} |
|
48 | ||
49 |
div#logs table td.message { |
|
50 |
text-align: left; |
|
51 |
} |
|
52 | ||
53 |
div#logs table tr.level-debug { |
|
54 |
color: #666; |
|
55 |
} |
|
56 | ||
57 |
div#logs table tr.level-warning, |
|
58 |
div#logs table tr.level-error, |
|
59 |
div#logs table tr.level-critical { |
|
60 |
color: #c33; |
|
61 |
} |
|
62 | ||
63 |
div#logs table tr.level-error, |
|
64 |
div#logs table tr.level-critical { |
|
65 |
font-weight: bold; |
|
66 |
} |
|
67 | ||
40 | 68 |
select#id_msg_class { max-width: 30em; } |
41 | 69 | |
42 | 70 |
li.webservices a { background-image: url(icons/icon-webservices.png); } |
passerelle/templates/passerelle/includes/resource-logs-table.html | ||
---|---|---|
1 |
{% load i18n passerelle %} |
|
2 |
{% load tz %} |
|
3 | ||
4 |
{% block content %} |
|
5 |
{% if logrecords %} |
|
6 |
<table class="main"> |
|
7 |
<thead> |
|
8 |
<th>{% trans 'Timestamp' %}</th> |
|
9 |
<th>{% trans 'Ip Source' %}</th> |
|
10 |
<th>{% trans 'Message' %}</th> |
|
11 |
</thead> |
|
12 |
<tbody> |
|
13 |
{% for record in logrecords %} |
|
14 |
<tr class="level-{{record.level}}"> |
|
15 |
<td class="timestamp">{{ record.timestamp|localtime }}</td> |
|
16 |
<td>{{ record.ipsource|default:"-" }}</td> |
|
17 |
<td class="message">{{ record.message}}</td> |
|
18 |
</tr> |
|
19 |
{% endfor %} |
|
20 |
</tbody> |
|
21 |
</table> |
|
22 | ||
23 |
{% if logrecords.has_other_pages %} |
|
24 |
<p class="paginator"> |
|
25 |
{% if logrecords.has_previous %} |
|
26 |
<a href="?page={{ logrecords.previous_page_number }}#logs"><<</a> |
|
27 |
{% else %} |
|
28 |
<span><<</span> |
|
29 |
{% endif %} |
|
30 |
|
|
31 |
<span class="current"> |
|
32 |
{{ logrecords.number }} / {{ logrecords.paginator.num_pages }} |
|
33 |
</span> |
|
34 |
|
|
35 |
{% if logrecords.has_next %} |
|
36 |
<a href="?page={{ logrecords.next_page_number }}#logs">>></a> |
|
37 |
{% else %} |
|
38 |
<span>>></span> |
|
39 |
{% endif %} |
|
40 |
</div> |
|
41 |
{% endif %} |
|
42 | ||
43 |
{% else %} |
|
44 |
<p>{% trans 'No records found' %}</p> |
|
45 |
{% endif %} |
|
46 |
{% endblock %} |
passerelle/templates/passerelle/manage/service_view.html | ||
---|---|---|
48 | 48 |
{% endif %} |
49 | 49 |
</div> |
50 | 50 | |
51 |
{% if perms.base.view_resourcelog %} |
|
52 |
<div id="logs"> |
|
53 |
<h3>{% trans "Logs" %}</h3> |
|
54 |
<div> |
|
55 |
{% block logs %} |
|
56 |
{% resource_logs_table resource=object %} |
|
57 |
{% endblock %} |
|
58 |
</div> |
|
59 |
</div> |
|
60 |
{% endif %} |
|
61 | ||
51 | 62 |
{% endblock %} |
passerelle/views.py | ||
---|---|---|
253 | 253 |
payload = request.body[:5000] |
254 | 254 |
connector.logger.debug('endpoint %s %s (%r) ' % |
255 | 255 |
(request.method, url, payload), |
256 |
request=request, |
|
256 | 257 |
extra={ |
257 | 258 |
'connector': connector_name, |
258 | 259 |
'connector_endpoint': endpoint_name, |
tests/test_generic_endpoint.py | ||
---|---|---|
25 | 25 | |
26 | 26 |
import utils |
27 | 27 | |
28 |
from passerelle.base.models import ResourceLog, ProxyLogger |
|
28 | 29 |
from passerelle.contrib.mdel.models import MDEL |
30 |
from passerelle.contrib.arcgis.models import Arcgis |
|
29 | 31 | |
30 | 32 | |
31 | 33 |
@pytest.fixture |
... | ... | |
33 | 35 |
return utils.setup_access_rights(MDEL.objects.create(slug='test')) |
34 | 36 | |
35 | 37 | |
38 |
@pytest.fixture |
|
39 |
def arcgis(db): |
|
40 |
return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) |
|
41 | ||
42 | ||
36 | 43 |
DEMAND_STATUS = { |
37 | 44 |
'closed': True, |
38 | 45 |
'status': 'accepted', |
... | ... | |
57 | 64 | |
58 | 65 |
records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] |
59 | 66 |
for record in records: |
60 |
assert record.module == 'views' |
|
61 | 67 |
assert record.levelname == 'DEBUG' |
62 | 68 |
assert record.connector == 'mdel' |
63 | 69 |
if record.connector_endpoint_method == 'POST': |
... | ... | |
67 | 73 |
assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message |
68 | 74 |
assert record.connector_endpoint == 'status' |
69 | 75 |
assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' |
76 | ||
77 | ||
78 |
@mock.patch('passerelle.utils.LoggedRequest.get') |
|
79 |
def test_proxy_logger(mocked_get, caplog, app, arcgis): |
|
80 |
payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() |
|
81 |
mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) |
|
82 | ||
83 |
# simple logger |
|
84 |
logger = ProxyLogger('DEBUG') |
|
85 |
logger.debug('this is a debug test') |
|
86 |
logger.info('this is an info test') |
|
87 | ||
88 |
assert ResourceLog.objects.count() == 2 |
|
89 |
for log in ResourceLog.objects.all(): |
|
90 |
if log.levelno == 10: |
|
91 |
assert log.message == 'this is a debug test' |
|
92 |
else: |
|
93 |
assert log.message == 'this is an info test' |
|
94 | ||
95 |
resp = app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
96 | ||
97 |
logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) |
|
98 | ||
99 |
# Resource Custom DB Logger |
|
100 |
log = ResourceLog.objects.filter(appname='arcgis', slug='test').first() |
|
101 |
assert log.appname == 'arcgis' |
|
102 |
assert log.slug == 'test' |
|
103 |
assert log.levelno == 10 |
|
104 |
assert log.sourceip == '127.0.0.1' |
|
105 |
assert log.extra['connector'] == 'arcgis' |
|
106 |
assert log.extra['connector_endpoint'] == 'district' |
|
107 |
assert log.extra['connector_endpoint_method'] == 'GET' |
|
108 |
assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' |
|
109 | ||
110 |
# Resource Generic Logger |
|
111 |
for record in caplog.records(): |
|
112 |
if record.name != 'passerelle.resource.arcgis.test': |
|
113 |
continue |
|
114 |
assert record.levelno == 10 |
|
115 |
assert record.levelname == 'DEBUG' |
|
116 |
assert record.name == 'passerelle.resource.arcgis.test' |
|
117 |
assert u"endpoint GET /arcgis/test/district?" in record.message |
|
118 | ||
119 |
data = resp.json['data'] |
|
120 |
assert data['id'] == 4 |
|
121 |
assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' |
|
122 | ||
123 |
# when changing log level |
|
124 |
ResourceLog.objects.all().delete() |
|
125 |
arcgis.log_level = 'INFO' |
|
126 |
arcgis.save() |
|
127 |
app.get('/arcgis/test/district', params={'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
128 |
assert ResourceLog.objects.count() == 0 |
|
129 | ||
130 |
arcgis.logger.info('testing info log message') |
|
131 |
assert ResourceLog.objects.count() == 1 |
|
132 |
log = ResourceLog.objects.first() |
|
133 |
assert log.levelno == 20 |
|
134 |
assert log.message == 'testing info log message' |
|
135 | ||
136 |
arcgis.logger.warning('first warning') |
|
137 |
assert ResourceLog.objects.count() == 2 |
|
138 |
assert ResourceLog.objects.last().message == 'first warning' |
|
139 |
assert ResourceLog.objects.last().levelno == 30 |
|
70 |
- |