0001-add-integrated-log-system-14191.patch
passerelle/base/migrations/0003_resourcelog.py | ||
---|---|---|
1 |
# -*- coding: utf-8 -*- |
|
2 |
from __future__ import unicode_literals |
|
3 | ||
4 |
from django.db import migrations, models |
|
5 |
import jsonfield.fields |
|
6 | ||
7 | ||
8 |
class Migration(migrations.Migration): |
|
9 | ||
10 |
dependencies = [ |
|
11 |
('base', '0002_auto_20151009_0326'), |
|
12 |
] |
|
13 | ||
14 |
operations = [ |
|
15 |
migrations.CreateModel( |
|
16 |
name='ResourceLog', |
|
17 |
fields=[ |
|
18 |
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), |
|
19 |
('timestamp', models.DateTimeField(auto_now_add=True)), |
|
20 |
('connector', models.CharField(max_length=128, null=True, verbose_name=b'connector')), |
|
21 |
('appname', models.CharField(max_length=128, null=True, verbose_name=b'appname')), |
|
22 |
('slug', models.CharField(max_length=128, null=True, verbose_name=b'slug')), |
|
23 |
('loglevel', models.CharField(max_length=16, verbose_name=b'log level')), |
|
24 |
('ipsource', models.GenericIPAddressField(null=True, verbose_name='IP Address', blank=True)), |
|
25 |
('message', models.TextField(max_length=2048, verbose_name=b'message')), |
|
26 |
('extra', jsonfield.fields.JSONField(default={}, verbose_name=b'extras')), |
|
27 |
], |
|
28 |
options={ |
|
29 |
'permissions': (('view_resourcelog', 'Can view resource logs'),), |
|
30 |
}, |
|
31 |
), |
|
32 |
] |
passerelle/base/models.py | ||
---|---|---|
6 | 6 |
from django.db import models |
7 | 7 |
from django.db.models import Q |
8 | 8 |
from django.utils.translation import ugettext_lazy as _ |
9 |
from django.utils.text import slugify |
|
10 | 9 | |
11 | 10 |
from django.contrib.contenttypes.models import ContentType |
12 | 11 |
from django.contrib.contenttypes import fields |
13 | 12 | |
14 | 13 |
from model_utils.managers import InheritanceManager as ModelUtilsInheritanceManager |
15 | 14 | |
15 |
import jsonfield |
|
16 | ||
16 | 17 |
import passerelle |
17 | 18 | |
18 | 19 |
KEYTYPE_CHOICES = ( |
... | ... | |
97 | 98 | |
98 | 99 |
def __init__(self, *args, **kwargs): |
99 | 100 |
super(BaseResource, self).__init__(*args, **kwargs) |
100 |
self.logger = logging.getLogger('passerelle.resource.%s.%s' % ( |
|
101 |
slugify(unicode(self.__class__.__name__)), self.slug) |
|
102 |
) |
|
103 |
self.logger.setLevel(getattr(logging, self.log_level)) |
|
101 |
self.logger = ProxyLogger(self.log_level, self.get_connector_slug(), self.slug) |
|
104 | 102 | |
105 | 103 |
def __unicode__(self): |
106 | 104 |
return self.title |
... | ... | |
166 | 164 | |
167 | 165 |
def __unicode__(self): |
168 | 166 |
return '%s (on %s <%s>) (for %s)' % (self.codename, self.resource_type, self.resource_pk, self.apiuser) |
167 | ||
168 | ||
169 |
class ResourceLog(models.Model): |
|
170 |
timestamp = models.DateTimeField(auto_now_add=True) |
|
171 |
connector = models.CharField(max_length=128, verbose_name='connector', null=True) |
|
172 |
appname = models.CharField(max_length=128, verbose_name='appname', null=True) |
|
173 |
slug = models.CharField(max_length=128, verbose_name='slug', null=True) |
|
174 |
loglevel = models.CharField(max_length=16, verbose_name='log level') |
|
175 |
ipsource = models.GenericIPAddressField(blank=True, null=True, verbose_name=_('IP Address')) |
|
176 |
message = models.TextField(max_length=2048, verbose_name='message') |
|
177 |
extra = jsonfield.JSONField(verbose_name='extras', default={}) |
|
178 | ||
179 |
class Meta: |
|
180 |
permissions = ( |
|
181 |
('view_resourcelog', 'Can view resource logs'), |
|
182 |
) |
|
183 | ||
184 |
def __unicode__(self): |
|
185 |
return '%s %s %s' % (self.timestamp, self.loglevel, self.connector) |
|
186 | ||
187 | ||
188 |
class ProxyLogger(object): |
|
189 | ||
190 |
def __init__(self, level, appname=None, slug=None): |
|
191 |
self.appname = appname |
|
192 |
self.slug = slug |
|
193 |
self.levelname = level |
|
194 |
if appname: |
|
195 |
logger_name = 'passerelle.resource.%s.%s' % (self.appname, self.slug) |
|
196 |
else: |
|
197 |
logger_name = 'passerelle.resource' |
|
198 | ||
199 |
self._logger = logging.getLogger(logger_name) |
|
200 |
self._logger.setLevel(level) |
|
201 | ||
202 |
def _log(self, levelname, message, *args, **kwargs): |
|
203 |
attr = {} |
|
204 |
attr['loglevel'] = levelname |
|
205 |
attr['message'] = message |
|
206 |
attr['appname'] = self.appname |
|
207 |
attr['slug'] = self.slug |
|
208 |
attr['extra'] = kwargs.get('extra', {}) |
|
209 |
request = kwargs.pop('request', None) |
|
210 | ||
211 |
if getattr(request, 'META', None): |
|
212 |
if 'HTTP_X_FORWARDED_FOR' in request.META: |
|
213 |
ipsource = request.META.get('HTTP_X_FORWARDED_FOR', '').split(",")[0].strip() |
|
214 |
else: |
|
215 |
ipsource = request.META.get('REMOTE_ADDR') |
|
216 |
else: |
|
217 |
ipsource = None |
|
218 | ||
219 |
if self.appname and self.slug: |
|
220 |
connector = '%s-%s' % (self.appname, self.slug) |
|
221 |
else: |
|
222 |
connector = None |
|
223 | ||
224 |
attr['ipsource'] = ipsource |
|
225 |
attr['connector'] = connector |
|
226 | ||
227 |
# Resource Custom DB Loggger |
|
228 |
if self._logger.level <= getattr(logging, levelname): |
|
229 |
ResourceLog.objects.create(**attr) |
|
230 | ||
231 |
# Default Resource Logger |
|
232 |
getattr(self._logger, levelname.lower())(message, *args, **kwargs) |
|
233 | ||
234 |
def debug(self, message, *args, **kwargs): |
|
235 |
self._log('DEBUG', message, *args, **kwargs) |
|
236 | ||
237 |
def info(self, message, *args, **kwargs): |
|
238 |
self._log('INFO', message, *args, **kwargs) |
|
239 | ||
240 |
def warning(self, message, *args, **kwargs): |
|
241 |
self._log('WARNING', message, *args, **kwargs) |
|
242 | ||
243 |
def critical(self, message, *args, **kwargs): |
|
244 |
self._log('CRITICAL', message, *args, **kwargs) |
|
245 | ||
246 |
def error(self, message, *args, **kwargs): |
|
247 |
self._log('ERROR', message, *args, **kwargs) |
|
248 | ||
249 |
def fatal(self, message, *args, **kwargs): |
|
250 |
self._log('FATAL', message, *args, **kwargs) |
passerelle/base/templatetags/passerelle.py | ||
---|---|---|
3 | 3 |
from django import template |
4 | 4 |
from django.contrib.contenttypes.models import ContentType |
5 | 5 |
from django.contrib.auth import get_permission_codename |
6 |
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
|
6 | 7 | |
7 | 8 |
from passerelle.utils import get_trusted_services |
8 |
from ..models import AccessRight |
|
9 |
from ..models import AccessRight, ResourceLog
|
|
9 | 10 | |
10 | 11 |
register = template.Library() |
11 | 12 | |
... | ... | |
22 | 23 |
return context |
23 | 24 | |
24 | 25 | |
26 |
@register.inclusion_tag('passerelle/includes/resource-logs-table.html', takes_context=True) |
|
27 |
def resource_logs_table(context, resource): |
|
28 |
request = context.get('request') |
|
29 |
page = request.GET.get('page', 1) |
|
30 | ||
31 |
connector = resource.get_connector_slug() |
|
32 |
context['connector'] = connector |
|
33 |
context['slug'] = resource.slug |
|
34 |
qs = ResourceLog.objects.filter(appname=connector, slug=resource.slug).order_by('-timestamp') |
|
35 | ||
36 |
paginator = Paginator(qs, 10) |
|
37 |
try: |
|
38 |
logrecords = paginator.page(page) |
|
39 |
except PageNotAnInteger: |
|
40 |
logrecords = paginator.page(1) |
|
41 |
except (EmptyPage,): |
|
42 |
logrecords = paginator.page(paginator.num_pages) |
|
43 | ||
44 |
context['logrecords'] = logrecords |
|
45 |
return context |
|
46 | ||
47 | ||
25 | 48 |
@register.filter |
26 | 49 |
def can_edit(obj, user): |
27 | 50 |
return user.has_perm(get_permission_codename('change', obj._meta), obj=obj) |
passerelle/settings.py | ||
---|---|---|
173 | 173 |
'handlers': { |
174 | 174 |
'console': { |
175 | 175 |
'level': 'DEBUG', |
176 |
'class': 'logging.StreamHandler',
|
|
177 |
},
|
|
176 |
'class': 'logging.StreamHandler' |
|
177 |
}, |
|
178 | 178 |
}, |
179 | 179 |
'loggers': { |
180 | 180 |
'django.request': { |
passerelle/templates/passerelle/includes/resource-logs-table.html | ||
---|---|---|
1 |
{% load i18n passerelle %} |
|
2 |
{% load tz %} |
|
3 | ||
4 |
{% block content %} |
|
5 |
{% if logrecords %} |
|
6 |
<table class="main"> |
|
7 |
<thead> |
|
8 |
<th>{% trans 'Id' %}</th> |
|
9 |
<th>{% trans 'Timestamp' %}</th> |
|
10 |
<th>{% trans 'Level' %}</th> |
|
11 |
<th>{% trans 'Ip Source' %}</th> |
|
12 |
<th>{% trans 'Message preview' %}</th> |
|
13 |
</thead> |
|
14 |
<tbody> |
|
15 |
{% for record in logrecords %} |
|
16 |
<tr> |
|
17 |
<td> |
|
18 |
{% url 'connector-log-detail' connector slug record.id as record_detail %} |
|
19 |
<a href="{{record_detail}}">{{record.id}}</a> |
|
20 |
</td> |
|
21 |
<td>{{ record.timestamp|localtime }}</td> |
|
22 |
<td>{{ record.loglevel }}</td> |
|
23 |
<td>{{ record.ipsource }}</td> |
|
24 |
<td>{{ record.message|truncatechars:64 }}</td> |
|
25 |
</tr> |
|
26 |
{% endfor %} |
|
27 |
</tbody> |
|
28 |
</table> |
|
29 | ||
30 |
{% if logrecords.has_other_pages %} |
|
31 |
<p class="paginator"> |
|
32 |
{% if logrecords.has_previous %} |
|
33 |
<a href="?page={{ logrecords.previous_page_number }}"><<</a> |
|
34 |
{% else %} |
|
35 |
<span><<</span> |
|
36 |
{% endif %} |
|
37 |
|
|
38 |
<span class="current"> |
|
39 |
{{ logrecords.number }} / {{ logrecords.paginator.num_pages }} |
|
40 |
</span> |
|
41 |
|
|
42 |
{% if logrecords.has_next %} |
|
43 |
<a href="?page={{ logrecords.next_page_number }}">>></a> |
|
44 |
{% else %} |
|
45 |
<span>>></span> |
|
46 |
{% endif %} |
|
47 |
</div> |
|
48 |
{% endif %} |
|
49 | ||
50 |
{% else %} |
|
51 |
<p>{% trans 'No records found' %}</p> |
|
52 |
{% endif %} |
|
53 |
{% endblock %} |
passerelle/templates/passerelle/manage/log_detail.html | ||
---|---|---|
1 |
{% extends "passerelle/manage.html" %} |
|
2 |
{% load i18n passerelle %} |
|
3 |
{% load tz %} |
|
4 | ||
5 |
{% block content %} |
|
6 |
<table class="main"> |
|
7 |
<tbody> |
|
8 |
<tr> |
|
9 |
<td>{% trans 'Timestamp' %}</td> |
|
10 |
<td>{{object.timestamp}}</td> |
|
11 |
</tr> |
|
12 |
<tr> |
|
13 |
<td>{% trans 'Level' %}</td> |
|
14 |
<td>{{object.loglevel|upper}}</td> |
|
15 |
</tr> |
|
16 |
<tr> |
|
17 |
<td>{% trans 'Ip Source' %}</td> |
|
18 |
<td>{{object.ipsource}}</td> |
|
19 |
</tr> |
|
20 |
<tr> |
|
21 |
<td>{% trans 'Message' %}</td> |
|
22 |
<td>{{object.message}}</td> |
|
23 |
</tr> |
|
24 |
<tr> |
|
25 |
<td>{% trans 'Extra' %}</td> |
|
26 |
<td> |
|
27 |
<table class="main"> |
|
28 |
{% for key, value in object.extra.items%} |
|
29 |
<tr> |
|
30 |
<td>{{key}}</td> |
|
31 |
<td>{{value}}</td> |
|
32 |
</tr> |
|
33 |
{% endfor %} |
|
34 |
</table> |
|
35 |
</td> |
|
36 |
</tr> |
|
37 |
</tbody> |
|
38 |
</table> |
|
39 |
{% endblock %} |
passerelle/templates/passerelle/manage/service_view.html | ||
---|---|---|
48 | 48 |
{% endif %} |
49 | 49 |
</div> |
50 | 50 | |
51 |
<div id="logs"> |
|
52 |
{% if perms.base.view_resourcelog %} |
|
53 |
<h3>{% trans "Logs" %}</h3> |
|
54 |
{% block logs %} |
|
55 |
{% resource_logs_table resource=object %} |
|
56 |
{% endblock %} |
|
57 |
{% endif %} |
|
58 |
</div> |
|
51 | 59 |
{% endblock %} |
60 | ||
61 |
passerelle/urls.py | ||
---|---|---|
9 | 9 |
from .views import (HomePageView, ManageView, ManageAddView, |
10 | 10 |
GenericCreateConnectorView, GenericDeleteConnectorView, |
11 | 11 |
GenericEditConnectorView, GenericEndpointView, GenericConnectorView, |
12 |
LEGACY_APPS_PATTERNS, LegacyPageView, login, logout) |
|
12 |
LEGACY_APPS_PATTERNS, LegacyPageView, login, logout, |
|
13 |
GenericConnectorLogDetailView) |
|
13 | 14 |
from .urls_utils import decorated_includes, required, app_enabled, manager_required |
14 | 15 |
from .base.urls import access_urlpatterns |
15 | 16 |
from .plugins import register_apps_urls |
... | ... | |
17 | 18 |
import choosit.urls |
18 | 19 |
import pastell.urls |
19 | 20 | |
21 | ||
20 | 22 |
admin.autodiscover() |
21 | 23 | |
22 | 24 |
urlpatterns = patterns('', |
... | ... | |
84 | 86 |
GenericEditConnectorView.as_view(), name='edit-connector'), |
85 | 87 |
))))) |
86 | 88 | |
89 |
# Intagrated logs |
|
90 |
urlpatterns += patterns('', |
|
91 |
url(r'^(?P<connector>[\w,-]+)/(?P<slug>[\w,-]+)/_logs/(?P<pk>[\w,-]+)$', |
|
92 |
GenericConnectorLogDetailView.as_view(), name='connector-log-detail'), |
|
93 |
) |
|
94 | ||
87 | 95 |
urlpatterns += patterns('', |
88 | 96 |
url(r'^(?P<connector>[\w,-]+)/(?P<slug>[\w,-]+)/$', |
89 | 97 |
GenericConnectorView.as_view(), name='view-connector'), |
passerelle/views.py | ||
---|---|---|
22 | 22 |
else: |
23 | 23 |
get_idps = lambda: [] |
24 | 24 | |
25 |
from passerelle.base.models import BaseResource |
|
25 |
from passerelle.base.models import BaseResource, ResourceLog
|
|
26 | 26 | |
27 | 27 |
from .utils import to_json, response_for_json, is_authorized |
28 | 28 |
from .forms import GenericConnectorForm |
... | ... | |
228 | 228 |
payload = request.body[:5000] |
229 | 229 |
connector.logger.debug('endpoint %s %s (%r) ' % |
230 | 230 |
(request.method, url, payload), |
231 |
request=request, |
|
231 | 232 |
extra={ |
232 | 233 |
'connector': connector_name, |
233 | 234 |
'connector_endpoint': endpoint_name, |
... | ... | |
257 | 258 |
return self.get(request, *args, **kwargs) |
258 | 259 | |
259 | 260 | |
261 |
class GenericConnectorLogDetailView(DetailView): |
|
262 |
template_name = 'passerelle/manage/log_detail.html' |
|
263 |
model = ResourceLog |
|
264 | ||
265 | ||
260 | 266 |
# legacy |
261 | 267 |
LEGACY_APPS_PATTERNS = { |
262 | 268 |
'datasources': {'url': 'data', 'name': 'Data Sources'}, |
tests/test_generic_endpoint.py | ||
---|---|---|
25 | 25 | |
26 | 26 |
import utils |
27 | 27 | |
28 |
from passerelle.base.models import ResourceLog, ProxyLogger |
|
28 | 29 |
from passerelle.contrib.mdel.models import MDEL |
30 |
from passerelle.contrib.arcgis.models import Arcgis |
|
29 | 31 | |
30 | 32 | |
31 | 33 |
@pytest.fixture |
... | ... | |
33 | 35 |
return utils.setup_access_rights(MDEL.objects.create(slug='test')) |
34 | 36 | |
35 | 37 | |
38 |
@pytest.fixture |
|
39 |
def arcgis(db): |
|
40 |
return utils.setup_access_rights(Arcgis.objects.create(slug='test', log_level='DEBUG')) |
|
41 | ||
42 | ||
36 | 43 |
DEMAND_STATUS = { |
37 | 44 |
'closed': True, |
38 | 45 |
'status': 'accepted', |
... | ... | |
57 | 64 | |
58 | 65 |
records = [record for record in caplog.records() if record.name == 'passerelle.resource.mdel.test'] |
59 | 66 |
for record in records: |
60 |
assert record.module == 'views' |
|
61 | 67 |
assert record.levelname == 'DEBUG' |
62 | 68 |
assert record.connector == 'mdel' |
63 | 69 |
if record.connector_endpoint_method == 'POST': |
... | ... | |
67 | 73 |
assert 'endpoint GET /mdel/test/status?demand_id=1-14-ILE-LA' in record.message |
68 | 74 |
assert record.connector_endpoint == 'status' |
69 | 75 |
assert record.connector_endpoint_url == '/mdel/test/status?demand_id=1-14-ILE-LA' |
76 | ||
77 | ||
78 |
@mock.patch('passerelle.utils.LoggedRequest.get') |
|
79 |
def test_proxy_logger(mocked_get, caplog, app, arcgis): |
|
80 |
payload = file(os.path.join(os.path.dirname(__file__), 'data', 'nancy_arcgis', 'sigresponse.json')).read() |
|
81 |
mocked_get.return_value = utils.FakedResponse(content=payload, status_code=200) |
|
82 | ||
83 |
# simple logger |
|
84 |
logger = ProxyLogger('DEBUG') |
|
85 |
logger.debug('this is a debug test') |
|
86 |
logger.info('this is an info test') |
|
87 | ||
88 |
assert ResourceLog.objects.count() == 2 |
|
89 |
for log in ResourceLog.objects.all(): |
|
90 |
if log.loglevel == 'DEBUG': |
|
91 |
assert log.message == 'this is a debug test' |
|
92 |
else: |
|
93 |
assert log.message == 'this is an info test' |
|
94 | ||
95 |
resp = app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
96 | ||
97 |
logger.debug('new token: %s (timeout %ss)', 'hfgjsfg=', 45) |
|
98 | ||
99 |
# Resource Custom DB Logger |
|
100 |
log = ResourceLog.objects.filter(connector='arcgis-test')[0] |
|
101 |
assert log.connector == 'arcgis-test' |
|
102 |
assert log.appname == 'arcgis' |
|
103 |
assert log.slug == 'test' |
|
104 |
assert log.loglevel == 'DEBUG' |
|
105 |
assert log.ipsource == '127.0.0.1' |
|
106 |
assert log.extra['connector'] == 'arcgis' |
|
107 |
assert log.extra['connector_endpoint'] == 'district' |
|
108 |
assert log.extra['connector_endpoint_method'] == 'GET' |
|
109 |
assert log.extra['connector_endpoint_url'] == '/arcgis/test/district?lat=48.673836&lon=6.172122' |
|
110 | ||
111 |
# Resource Generic Logger |
|
112 |
for record in caplog.records(): |
|
113 |
if record.name != 'passerelle.resource.arcgis.test': |
|
114 |
continue |
|
115 |
assert record.levelno == 10 |
|
116 |
assert record.levelname == 'DEBUG' |
|
117 |
assert record.name == 'passerelle.resource.arcgis.test' |
|
118 |
assert record.message == u"endpoint GET /arcgis/test/district?lat=48.673836&lon=6.172122 ('') " |
|
119 | ||
120 |
data = resp.json['data'] |
|
121 |
assert data['id'] == 4 |
|
122 |
assert data['text'] == 'HAUSSONVILLE / BLANDAN / MON DESERT / SAURUPT' |
|
123 | ||
124 |
# when changing log level |
|
125 |
ResourceLog.objects.all().delete() |
|
126 |
arcgis.log_level = 'INFO' |
|
127 |
arcgis.save() |
|
128 |
app.get('/arcgis/test/district', {'lon': 6.172122, 'lat': 48.673836}, status=200) |
|
129 |
assert ResourceLog.objects.count() == 0 |
|
130 | ||
131 |
arcgis.logger.info('testing info log message') |
|
132 |
assert ResourceLog.objects.count() == 1 |
|
133 |
log = ResourceLog.objects.first() |
|
134 |
assert log.connector == 'arcgis-test' |
|
135 |
assert log.loglevel == 'INFO' |
|
136 |
assert log.message == 'testing info log message' |
|
137 | ||
138 |
arcgis.logger.warning('first warning') |
|
139 |
assert ResourceLog.objects.count() == 2 |
|
140 |
assert ResourceLog.objects.last().message == 'first warning' |
|
141 |
assert ResourceLog.objects.last().loglevel == 'WARNING' |
|
70 |
- |