0001-add-integrated-log-system-14191.patch
passerelle/base/migrations/0005_resourcelog.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations, models |
|
5 |
import jsonfield.fields |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('base', '0004_auto_20170117_0326'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.CreateModel( |
|
16 |
name='ResourceLog', |
|
17 |
fields=[ |
|
18 |
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), |
|
19 |
('timestamp', models.DateTimeField(auto_now_add=True)), |
|
20 |
('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), |
|
21 |
('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), |
|
22 |
('levelno', models.IntegerField(verbose_name=b'log level')), |
|
23 |
('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Source', blank=True)), |
|
24 |
('message', models.TextField(max_length=2048, verbose_name=b'message')), |
|
25 |
('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), |
|
26 |
], |
|
27 |
options={ |
|
28 |
'permissions': (('view_resourcelog', 'Can view resource logs'),), |
|
29 |
}, |
|
30 |
), |
|
31 |
] |
passerelle/base/models.py | ||
---|---|---|
9 | 9 |
from django.db import models, transaction |
10 | 10 |
from django.db.models import Q |
11 | 11 |
from django.utils.translation import ugettext_lazy as _ |
12 |
from django.utils.text import slugify |
|
13 | 12 |
from django.core.files.base import ContentFile |
14 | 13 | |
15 | 14 |
from django.contrib.contenttypes.models import ContentType |
... | ... | |
19 | 18 | |
20 | 19 |
from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager |
21 | 20 | |
21 |
import jsonfield |
|
22 | ||
22 | 23 |
import passerelle |
23 | 24 | |
24 | 25 |
KEYTYPE_CHOICES = ( |
... | ... | |
126 | 127 | |
127 | 128 |
def __init__(self, *args, **kwargs): |
128 | 129 |
super(BaseResource, self).__init__(*args, **kwargs) |
129 |
self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( |
|
130 |
slugify(unicode(self.__class__.__name__)), self.slug) |
|
131 |
) |
|
132 |
self.logger.setLevel(getattr(logging, self.log_level)) |
|
130 |
self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) |
|
133 | 131 | |
134 | 132 |
def __unicode__(self): |
135 | 133 |
return self.title |
... | ... | |
304 | 302 | |
305 | 303 |
def __unicode__(self): |
306 | 304 |
return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) |
305 | ||
306 | ||
307 |
class ResourceLog(models.Model): |
|
308 |
timestamp = models.DateTimeField(auto_now_add=True) |
|
309 |
appname = models.CharField(max_length=128, verbose_name='appname', null=True) |
|
310 |
slug = models.CharField(max_length=128, verbose_name='slug', null=True) |
|
311 |
levelno = models.IntegerField(verbose_name='log level') |
|
312 |
ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Source')) |
|
313 |
message = models.TextField(max_length=2048, verbose_name='message') |
|
314 |
extra = jsonfield.JSONField(verbose_name='extras', default={}) |
|
315 | ||
316 |
class Meta: |
|
317 |
permissions = ( |
|
318 |
('view_resourcelog', 'Can view resource logs'), |
|
319 |
) |
|
320 | ||
321 |
def __unicode__(self): |
|
322 |
return '%s %s %s %s' % (self.timestamp, self.levelno, self.appname, self.slug) |
|
323 | ||
324 | ||
325 |
class ProxyLogger(object): |
|
326 | ||
327 |
def __init__(self, level, appname=None, slug=None): |
|
328 |
self.appname = appname |
|
329 |
self.slug = slug |
|
330 |
if appname: |
|
331 |
logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) |
|
332 |
else: |
|
333 |
logger_name = 'passerelle.resource' |
|
334 | ||
335 |
self._logger = logging.getLogger(logger_name) |
|
336 |
self._logger.setLevel(level) |
|
337 | ||
338 |
def _log(self, levelname, message, *args, **kwargs): |
|
339 |
levelno = getattr(logging, levelname) |
|
340 |
attr = {} |
|
341 |
attr['levelno'] = levelno |
|
342 |
attr['message'] = message[:ResourceLog._meta.get_field('message').max_length] |
|
343 |
attr['appname'] = self.appname |
|
344 |
attr['slug'] = self.slug |
|
345 |
attr['extra'] = kwargs.get('extra', {}) |
|
346 |
request = kwargs.pop('request', None) |
|
347 | ||
348 |
if getattr(request, 'META', None): |
|
349 |
if 'HTTP_X_FORWARDED_FOR' in request.META: |
|
350 |
ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() |
|
351 |
else: |
|
352 |
ipsource = request.META.get('REMOTE_ADDR') |
|
353 |
else: |
|
354 |
ipsource = None |
|
355 |
attr['ipsource'] = ipsource |
|
356 | ||
357 |
if self._logger.level <= levelno: |
|
358 |
ResourceLog.objects.create(**attr) |
|
359 | ||
360 |
getattr(self._logger, levelname.lower())(message, *args, **kwargs) |
|
361 | ||
362 |
def debug(self, message, *args, **kwargs): |
|
363 |
self._log('DEBUG', message, *args, **kwargs) |
|
364 | ||
365 |
def info(self, message, *args, **kwargs): |
|
366 |
self._log('INFO', message, *args, **kwargs) |
|
367 | ||
368 |
def warning(self, message, *args, **kwargs): |
|
369 |
self._log('WARNING', message, *args, **kwargs) |
|
370 | ||
371 |
def critical(self, message, *args, **kwargs): |
|
372 |
self._log('CRITICAL', message, *args, **kwargs) |
|
373 | ||
374 |
def error(self, message, *args, **kwargs): |
|
375 |
self._log('ERROR', message, *args, **kwargs) |
|
376 | ||
377 |
def fatal(self, message, *args, **kwargs): |
|
378 |
self._log('FATAL', message, *args, **kwargs) |
passerelle/base/templatetags/passerelle.py | ||
---|---|---|
3 | 3 |
from django import template |
4 | 4 |
from django.contrib.contenttypes.models import ContentType |
5 | 5 |
from django.contrib.auth import get_permission_codename |
6 |
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
|
6 | 7 | |
7 | 8 |
from passerelle.utils import get_trusted_services |
8 |
from ..models import AccessRight |
|
9 |
from ..models import AccessRight, ResourceLog
|
|
9 | 10 | |
10 | 11 |
register = template.Library() |
11 | 12 | |
... | ... | |
22 | 23 |
return context |
23 | 24 | |
24 | 25 | |
26 |
@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) |
|
27 |
def resource_logs_table(context, resource): |
|
28 |
request = context.get('request') |
|
29 |
page = request.GET.get('page', 1) |
|
30 | ||
31 |
connector = resource.get_connector_slug() |
|
32 |
context['connector'] = connector |
|
33 |
context['slug'] = resource.slug |
|
34 |
qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') |
|
35 | ||
36 |
paginator = Paginator(qs, 10) |
|
37 |
try: |
|
38 |
logrecords = paginator.page(page) |
|
39 |
except PageNotAnInteger: |
|
40 |
logrecords = paginator.page(1) |
|
41 |
except (EmptyPage,): |
|
42 |
logrecords = paginator.page(paginator.num_pages) |
|
43 | ||
44 |
context['logrecords'] = logrecords |
|
45 |
return context |
|
46 | ||
47 | ||
25 | 48 |
@register.filter |
26 | 49 |
def can_edit(obj, user): |
27 | 50 |
return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) |
passerelle/templates/passerelle/includes/resource-logs-table.html | ||
---|---|---|
1 |
{% load i18n passerelle %} |
|
2 |
{% load tz %} |
|
3 | ||
4 |
{% block content %} |
|
5 |
{% if logrecords %} |
|
6 |
<table class="main"> |
|
7 |
<thead> |
|
8 |
<th>{% trans 'Id' %}</th> |
|
9 |
<th>{% trans 'Timestamp' %}</th> |
|
10 |
<th>{% trans 'Level' %}</th> |
|
11 |
<th>{% trans 'Ip Source' %}</th> |
|
12 |
<th>{% trans 'Message' %}</th> |
|
13 |
</thead> |
|
14 |
<tbody> |
|
15 |
{% for record in logrecords %} |
|
16 |
<tr> |
|
17 |
<td>{{ record.id }}</td> |
|
18 |
<td>{{ record.timestamp|localtime }}</td> |
|
19 |
<td>{{ record.levelno }}</td> |
|
20 |
<td>{{ record.ipsource }}</td> |
|
21 |
<td>{{ record.message}}</td> |
|
22 |
</tr> |
|
23 |
{% endfor %} |
|
24 |
</tbody> |
|
25 |
</table> |
|
26 | ||
27 |
{% if logrecords.has_other_pages %} |
|
28 |
<p class="paginator"> |
|
29 |
{% if logrecords.has_previous %} |
|
30 |
<a href="?page={{ logrecords.previous_page_number }}#logs"><<</a> |
|
31 |
{% else %} |
|
32 |
<span><<</span> |
|
33 |
{% endif %} |
|
34 |
|
|
35 |
<span class="current"> |
|
36 |
{{ logrecords.number }} / {{ logrecords.paginator.num_pages }} |
|
37 |
</span> |
|
38 |
|
|
39 |
{% if logrecords.has_next %} |
|
40 |
<a href="?page={{ logrecords.next_page_number }}#logs">>></a> |
|
41 |
{% else %} |
|
42 |
<span>>></span> |
|
43 |
{% endif %} |
|
44 |
</div> |
|
45 |
{% endif %} |
|
46 | ||
47 |
{% else %} |
|
48 |
<p>{% trans 'No records found' %}</p> |
|
49 |
{% endif %} |
|
50 |
{% endblock %} |
passerelle/templates/passerelle/manage/service_view.html | ||
---|---|---|
48 | 48 |
{% endif %} |
49 | 49 |
</div> |
50 | 50 | |
51 |
{% if perms.base.view_resourcelog %} |
|
52 |
<div id="logs"> |
|
53 |
<h3>{% trans "Logs" %}</h3> |
|
54 |
{% block logs %} |
|
55 |
{% resource_logs_table resource=object %} |
|
56 |
{% endblock %} |
|
57 |
</div> |
|
58 |
{% endif %} |
|
59 | ||
51 | 60 |
{% endblock %} |
passerelle/views.py | ||
---|---|---|
253 | 253 |
payload = request.body[:5000] |
254 | 254 |
connector.logger.debug('endpoint %s %s (%r) ' % |
255 | 255 |
(request.method, url, payload), |
256 |
request=request, |
|
256 | 257 |
extra={ |
257 | 258 |
'connector': connector_name, |
258 | 259 |
'connector_endpoint': endpoint_name, |
tests/test_generic_endpoint.py | ||
---|---|---|
25 | 25 | |
26 | 26 |
import utils |
27 | 27 | |
28 |
from passerelle.base.models import ResourceLog, ProxyLogger |
|
28 | 29 |
from passerelle.contrib.mdel.models import MDEL |
30 |
from passerelle.contrib.arcgis.models import Arcgis |
|
29 | 31 | |
30 | 32 | |
31 | 33 |
@pytest.fixture |
... | ... | |
33 | 35 |
return utils.setup_access_rights(MDEL.objects.create(slug='test')) |
34 | 36 | |
35 | 37 | |
38 |
@pytest.fixture |
|
39 |
def arcgis(db): |
|
40 |
return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) |
|
41 | ||
42 | ||
36 | 43 |
DEMAND_STATUS = { |
37 | 44 |
'closed': True, |
38 | 45 |
'status': 'accepted', |
... | ... | |
57 | 64 | |
58 | 65 |
records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] |
59 | 66 |
for record in records: |
60 |
assert record.module == 'views' |
|
61 | 67 |
assert record.levelname == 'DEBUG' |
62 | 68 |
assert record.connector == 'mdel' |
63 | 69 |
if record.connector_endpoint_method == 'POST': |
... | ... | |
67 | 73 |
assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message |
68 | 74 |
assert record.connector_endpoint == 'status' |
69 | 75 |
assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' |
76 | ||
77 | ||
78 |
@mock.patch('passerelle.utils.LoggedRequest.get') |
|
79 |
def test_proxy_logger(mocked_get, caplog, app, arcgis): |
|
80 |
payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() |
|
81 |
mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) |
|
82 | ||
83 |
# simple logger |
|
84 |
logger = ProxyLogger('DEBUG') |
|
85 |
logger.debug('this is a debug test') |
|
86 |
logger.info('this is an info test') |
|
87 | ||
88 |
assert ResourceLog.objects.count() == 2 |
|
89 |
for log in ResourceLog.objects.all(): |
|
90 |
if log.levelno == 10: |
|
91 |
assert log.message == 'this is a debug test' |
|
92 |
else: |
|
93 |
assert log.message == 'this is an info test' |
|
94 | ||
95 |
resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
96 | ||
97 |
logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) |
|
98 | ||
99 |
# Resource Custom DB Logger |
|
100 |
log = ResourceLog.objects.filter(appname='arcgis', slug='test').first() |
|
101 |
assert log.appname == 'arcgis' |
|
102 |
assert log.slug == 'test' |
|
103 |
assert log.levelno == 10 |
|
104 |
assert log.ipsource == '127.0.0.1' |
|
105 |
assert log.extra['connector'] == 'arcgis' |
|
106 |
assert log.extra['connector_endpoint'] == 'district' |
|
107 |
assert log.extra['connector_endpoint_method'] == 'GET' |
|
108 |
assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' |
|
109 | ||
110 |
# Resource Generic Logger |
|
111 |
for record in caplog.records(): |
|
112 |
if record.name != 'passerelle.resource.arcgis.test': |
|
113 |
continue |
|
114 |
assert record.levelno == 10 |
|
115 |
assert record.levelname == 'DEBUG' |
|
116 |
assert record.name == 'passerelle.resource.arcgis.test' |
|
117 |
assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " |
|
118 | ||
119 |
data = resp.json['data'] |
|
120 |
assert data['id'] == 4 |
|
121 |
assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' |
|
122 | ||
123 |
# when changing log level |
|
124 |
ResourceLog.objects.all().delete() |
|
125 |
arcgis.log_level = 'INFO' |
|
126 |
arcgis.save() |
|
127 |
app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
128 |
assert ResourceLog.objects.count() == 0 |
|
129 | ||
130 |
arcgis.logger.info('testing info log message') |
|
131 |
assert ResourceLog.objects.count() == 1 |
|
132 |
log = ResourceLog.objects.first() |
|
133 |
assert log.levelno == 20 |
|
134 |
assert log.message == 'testing info log message' |
|
135 | ||
136 |
arcgis.logger.warning('first warning') |
|
137 |
assert ResourceLog.objects.count() == 2 |
|
138 |
assert ResourceLog.objects.last().message == 'first warning' |
|
139 |
assert ResourceLog.objects.last().levelno == 30 |
|
70 |
- |