From 9a9159bcb8a4025bc4571ce41f7e717c97097663 Mon Sep 17 00:00:00 2001 From: Benjamin Dauvergne Date: Fri, 27 Apr 2018 00:54:04 +0200 Subject: [PATCH 2/2] debian: add journald support to debian_config_common (fixes #23471) --- debian/control | 3 +- debian/debian_config_common.py | 22 +- hobo/journal.py | 224 ++++++++++++++++++ .../test_request_context_filter.py | 57 +++++ tox.ini | 1 + 5 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 hobo/journal.py diff --git a/debian/control b/debian/control index 33a0609..6ef26f8 100644 --- a/debian/control +++ b/debian/control @@ -21,7 +21,8 @@ Depends: ${misc:Depends}, Recommends: python-django (>= 1.8), python-gadjo, python-django-mellon (>= 1.2.22.26), - memcached + memcached, + python-systemd Description: Rapid Remote Deployment python module Package: hobo diff --git a/debian/debian_config_common.py b/debian/debian_config_common.py index 685898f..b9b0ee9 100644 --- a/debian/debian_config_common.py +++ b/debian/debian_config_common.py @@ -9,6 +9,8 @@ # execfile('/etc/%s/settings.py' % PROJECT_NAME) import os +import warnings + from django.conf import global_settings from django.core.exceptions import ImproperlyConfigured @@ -147,22 +149,18 @@ LOGGING = { }, } -# Graylog support -if 'GRAYLOG_URL' in os.environ: +# Journald support +if os.path.exists('/run/systemd/journal/socket'): try: - from graypy import GELFHandler + from systemd import journal except ImportError: - raise ImproperlyConfigured('cannot configure graypy, import of GELFHandler failed') + warnings.warn('journald will not be used directly, please install python-systemd') else: - host = os.environ['GRAYLOG_URL'].split(':')[0] - port = int(os.environ['GRAYLOG_URL'].split(':')[1]) - LOGGING['handlers']['gelf'] = { - 'class': 'graypy.GELFHandler', - 'fqdn': True, - 'host': host, - 'port': port, + LOGGING['handlers']['journald'] = { + 'class': 'hobo.journal.JournalHandler', } - LOGGING['loggers']['']['handlers'].append('gelf') + LOGGING['loggers']['']['handlers'].remove('syslog') + LOGGING['loggers']['']['handlers'].append('journald') # Sentry support if 'SENTRY_DSN' in os.environ: diff --git a/hobo/journal.py b/hobo/journal.py new file mode 100644 index 0000000..b6a7a94 --- /dev/null +++ b/hobo/journal.py @@ -0,0 +1,224 @@ +# -*- Mode: python; coding:utf-8; indent-tabs-mode: nil -*- */ +# +# +# Copyright 2012 David Strauss +# Copyright 2012 Zbigniew Jędrzejewski-Szmek +# Copyright 2012 Marti Raudsepp +# +# python-systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# python-systemd is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with python-systemd; If not, see . + +import sys as _sys +import traceback as _traceback +import logging as _logging +from syslog import (LOG_ALERT, LOG_CRIT, LOG_ERR, + LOG_WARNING, LOG_INFO, LOG_DEBUG) + +from systemd._journal import sendv + +_IDENT_CHARACTER = set('ABCDEFGHIJKLMNOPQRTSUVWXYZ_0123456789') + + +def _valid_field_name(s): + return not (set(s) - _IDENT_CHARACTER) + + +def _make_line(field, value): + if isinstance(value, bytes): + return field.encode('utf-8') + b'=' + value + elif isinstance(value, str): + return field + '=' + value + else: + return field + '=' + str(value) + + +def send(MESSAGE, MESSAGE_ID=None, + CODE_FILE=None, CODE_LINE=None, CODE_FUNC=None, + **kwargs): + r"""Send a message to the journal. + + >>> from systemd import journal + >>> journal.send('Hello world') + >>> journal.send('Hello, again, world', FIELD2='Greetings!') + >>> journal.send('Binary message', BINARY=b'\xde\xad\xbe\xef') + + Value of the MESSAGE argument will be used for the MESSAGE= field. MESSAGE + must be a string and will be sent as UTF-8 to the journal. + + MESSAGE_ID can be given to uniquely identify the type of message. It must be + a string or a uuid.UUID object. + + CODE_LINE, CODE_FILE, and CODE_FUNC can be specified to identify the caller. + Unless at least on of the three is given, values are extracted from the + stack frame of the caller of send(). CODE_FILE and CODE_FUNC must be + strings, CODE_LINE must be an integer. + + Additional fields for the journal entry can only be specified as keyword + arguments. The payload can be either a string or bytes. A string will be + sent as UTF-8, and bytes will be sent as-is to the journal. + + Other useful fields include PRIORITY, SYSLOG_FACILITY, SYSLOG_IDENTIFIER, + SYSLOG_PID. + """ + + args = ['MESSAGE=' + MESSAGE] + + if MESSAGE_ID is not None: + id = getattr(MESSAGE_ID, 'hex', MESSAGE_ID) + args.append('MESSAGE_ID=' + id) + + if CODE_LINE is CODE_FILE is CODE_FUNC is None: + CODE_FILE, CODE_LINE, CODE_FUNC = _traceback.extract_stack(limit=2)[0][:3] + if CODE_FILE is not None: + args.append('CODE_FILE=' + CODE_FILE) + if CODE_LINE is not None: + args.append('CODE_LINE={:d}'.format(CODE_LINE)) + if CODE_FUNC is not None: + args.append('CODE_FUNC=' + CODE_FUNC) + + args.extend(_make_line(key, val) for key, val in kwargs.items()) + return sendv(*args) + + +class JournalHandler(_logging.Handler): + """Journal handler class for the Python logging framework. + + Please see the Python logging module documentation for an overview: + http://docs.python.org/library/logging.html. + + To create a custom logger whose messages go only to journal: + + >>> import logging + >>> log = logging.getLogger('custom_logger_name') + >>> log.propagate = False + >>> log.addHandler(JournalHandler()) + >>> log.warning("Some message: %s", 'detail') + + Note that by default, message levels `INFO` and `DEBUG` are ignored by the + logging framework. To enable those log levels: + + >>> log.setLevel(logging.DEBUG) + + To redirect all logging messages to journal regardless of where they come + from, attach it to the root logger: + + >>> logging.root.addHandler(JournalHandler()) + + For more complex configurations when using `dictConfig` or `fileConfig`, + specify `systemd.journal.JournalHandler` as the handler class. Only + standard handler configuration options are supported: `level`, `formatter`, + `filters`. + + To attach journal MESSAGE_ID, an extra field is supported: + + >>> import uuid + >>> mid = uuid.UUID('0123456789ABCDEF0123456789ABCDEF') + >>> log.warning("Message with ID", extra={'MESSAGE_ID': mid}) + + Fields to be attached to all messages sent through this handler can be + specified as keyword arguments. This probably makes sense only for + SYSLOG_IDENTIFIER and similar fields which are constant for the whole + program: + + >>> JournalHandler(SYSLOG_IDENTIFIER='my-cool-app') + <...JournalHandler ...> + + The following journal fields will be sent: `MESSAGE`, `PRIORITY`, + `THREAD_NAME`, `CODE_FILE`, `CODE_LINE`, `CODE_FUNC`, `LOGGER` (name as + supplied to getLogger call), `MESSAGE_ID` (optional, see above), + `SYSLOG_IDENTIFIER` (defaults to sys.argv[0]). + + The function used to actually send messages can be overridden using + the `sender_function` parameter. + """ + + def __init__(self, level=_logging.NOTSET, sender_function=send, **kwargs): + super(JournalHandler, self).__init__(level) + + for name in kwargs: + if not _valid_field_name(name): + raise ValueError('Invalid field name: ' + name) + if 'SYSLOG_IDENTIFIER' not in kwargs: + kwargs['SYSLOG_IDENTIFIER'] = _sys.argv[0] + + self.send = sender_function + self._extra = kwargs + + def emit(self, record): + """Write `record` as a journal event. + + MESSAGE is taken from the message provided by the user, and PRIORITY, + LOGGER, THREAD_NAME, CODE_{FILE,LINE,FUNC} fields are appended + automatically. In addition, record.MESSAGE_ID will be used if present. + """ + try: + msg = self.format(record) + pri = self.map_priority(record.levelno) + # defaults + extras = self._extra.copy() + + # higher priority + if record.exc_text: + extras['EXCEPTION_TEXT'] = record.exc_text + + if record.exc_info: + extras['EXCEPTION_INFO'] = record.exc_info + + if record.args: + extras['CODE_ARGS'] = str(record.args) + + # explicit arguments — highest priority + for key, value in record.__dict__.items(): + new_key = key.upper() + if new_key in ['PRIORITY', 'LOGGER', 'THREAD_NAME', + 'PROCESS_NAME', 'CODE_FILE', 'MESSAGE', + 'CODE_LINE', 'CODE_FUNC']: + continue + if key in ['threadName', 'processName', 'pathname', 'lineno', + 'funcName']: + continue + extras[key.upper()] = value + + self.send(msg, + PRIORITY=format(pri), + LOGGER=record.name, + THREAD_NAME=record.threadName, + PROCESS_NAME=record.processName, + CODE_FILE=record.pathname, + CODE_LINE=record.lineno, + CODE_FUNC=record.funcName, + **extras) + except Exception: + self.handleError(record) + + @staticmethod + def map_priority(levelno): + """Map logging levels to journald priorities. + + Since Python log level numbers are "sparse", we have to map numbers in + between the standard levels too. + """ + if levelno <= _logging.DEBUG: + return LOG_DEBUG + elif levelno <= _logging.INFO: + return LOG_INFO + elif levelno <= _logging.WARNING: + return LOG_WARNING + elif levelno <= _logging.ERROR: + return LOG_ERR + elif levelno <= _logging.CRITICAL: + return LOG_CRIT + else: + return LOG_ALERT + + mapPriority = map_priority diff --git a/tests_multitenant/test_request_context_filter.py b/tests_multitenant/test_request_context_filter.py index 161be26..4e00fcf 100644 --- a/tests_multitenant/test_request_context_filter.py +++ b/tests_multitenant/test_request_context_filter.py @@ -1,5 +1,9 @@ +import pytest + import logging + from hobo.logger import RequestContextFilter + from tenant_schemas.utils import tenant_context from django.contrib.auth.models import User @@ -36,3 +40,56 @@ def test_request_context_filter(caplog, settings, tenants, client): assert record.user_display_name == 'John Doe' assert record.user_uuid == 'ab' * 16 assert record.application == 'fake-agent' + + +@pytest.fixture +def journald_handler(): + from hobo.journal import JournalHandler + + root_logger = logging.getLogger() + journald_handler = JournalHandler() + root_logger.handlers.append(journald_handler) + try: + yield journald_handler + finally: + root_logger.handlers.remove(journald_handler) + + +def test_systemd(settings, tenants, client, journald_handler): + root_logger = logging.getLogger() + assert len(root_logger.handlers) == 2 + journald_handler.addFilter(RequestContextFilter()) + + for tenant in tenants: + with tenant_context(tenant): + user = User.objects.create(first_name='John', last_name='Doe', username='john.doe', + email='jodn.doe@example.com') + user.set_password('john.doe') + user.save() + user.saml_identifiers.create(name_id='ab' * 16, issuer='https://idp.example.com') + + for tenant in tenants: + settings.ALLOWED_HOSTS.append(tenant.domain_url) + with tenant_context(tenant): + client.login(username='john.doe', password='john.doe') + client.get('/', SERVER_NAME=tenant.domain_url, + HTTP_X_FORWARDED_FOR='99.99.99.99, 127.0.0.1') + + from systemd.journal import Reader + import time + + reader = Reader() + reader.seek_realtime(time.time() - 10) + records = [l for l in reader if l['MESSAGE'] == 'wat!'] + assert len(records) == 2 + for tenant, record in zip(tenants, records): + assert record['IP'] == '99.99.99.99' + assert record['TENANT'] == tenant.domain_url + assert record['PATH'] == '/' + assert record['REQUEST_ID'].startswith('r:') + assert record['USER'] == user.username + assert record['USER_EMAIL'] == user.email + assert record['USER_NAME'] == user.username + assert record['USER_DISPLAY_NAME'] == 'John Doe' + assert record['USER_UUID'] == 'ab' * 16 + assert record['APPLICATION'] == 'fake-agent' diff --git a/tox.ini b/tox.ini index 594fbdd..c33eaa6 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ deps: passerelle: http://git.entrouvert.org/passerelle.git/snapshot/passerelle-master.tar.gz passerelle: suds passerelle: python-memcached + multitenant: systemd-python http://git.entrouvert.org/debian/django-tenant-schemas.git/snapshot/django-tenant-schemas-master.tar.gz httmock requests -- 2.18.0