--- /tmp/journal.230.py 2018-12-04 20:32:51.165635634 +0100 +++ /tmp/journal.234.py 2018-12-04 20:32:58.561589785 +0100 @@ -5,18 +5,18 @@ # Copyright 2012 Zbigniew Jędrzejewski-Szmek # Copyright 2012 Marti Raudsepp # -# systemd is free software; you can redistribute it and/or modify it +# python-systemd is free software; you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 2.1 of the License, or # (at your option) any later version. # -# systemd is distributed in the hope that it will be useful, but +# python-systemd is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License -# along with systemd; If not, see . +# along with python-systemd; If not, see . from __future__ import division @@ -26,14 +26,16 @@ import traceback as _traceback import os as _os import logging as _logging -if _sys.version_info >= (3,3): - from collections import ChainMap as _ChainMap from syslog import (LOG_EMERG, LOG_ALERT, LOG_CRIT, LOG_ERR, LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_DEBUG) +if _sys.version_info >= (3,3): + from collections import ChainMap as _ChainMap + from ._journal import __version__, sendv, stream_fd from ._reader import (_Reader, NOP, APPEND, INVALIDATE, LOCAL_ONLY, RUNTIME_ONLY, SYSTEM, SYSTEM_ONLY, CURRENT_USER, + OS_ROOT, _get_catalog) from . import id128 as _id128 @@ -42,19 +44,24 @@ else: Monotonic = tuple + def _convert_monotonic(m): return Monotonic((_datetime.timedelta(microseconds=m[0]), _uuid.UUID(bytes=m[1]))) + def _convert_source_monotonic(s): return _datetime.timedelta(microseconds=int(s)) + def _convert_realtime(t): return _datetime.datetime.fromtimestamp(t / 1000000) + def _convert_timestamp(s): return _datetime.datetime.fromtimestamp(int(s) / 1000000) + def _convert_trivial(x): return x @@ -100,57 +107,72 @@ 'COREDUMP_TIMESTAMP': _convert_timestamp, } -_IDENT_LETTER = set('ABCDEFGHIJKLMNOPQRTSUVWXYZ_') +_IDENT_CHARACTER = set('ABCDEFGHIJKLMNOPQRTSUVWXYZ_0123456789') + def _valid_field_name(s): - return not (set(s) - _IDENT_LETTER) + return not (set(s) - _IDENT_CHARACTER) + class Reader(_Reader): - """Reader allows the access and filtering of systemd journal - entries. Note that in order to access the system journal, a - non-root user must be in the `systemd-journal` group. + """Access systemd journal entries. - Example usage to print out all informational or higher level - messages for systemd-udevd for this boot: + Entries are subject to filtering and limits, see `add_match`, `this_boot`, + `this_machine` functions and the `data_treshold` attribute. + Note that in order to access the system journal, a non-root user must have + the necessary privileges, see journalctl(1) for details. Unprivileged users + can access only their own journal. + + Example usage to print out all informational or higher level messages for + systemd-udevd for this boot: + + >>> from systemd import journal >>> j = journal.Reader() >>> j.this_boot() >>> j.log_level(journal.LOG_INFO) >>> j.add_match(_SYSTEMD_UNIT="systemd-udevd.service") - >>> for entry in j: + >>> for entry in j: # doctest: +SKIP ... print(entry['MESSAGE']) + starting version ... + + See systemd.journal-fields(7) for more info on typical fields found in the + journal. - See systemd.journal-fields(7) for more info on typical fields - found in the journal. """ - def __init__(self, flags=0, path=None, files=None, converters=None): - """Create an instance of Reader, which allows filtering and - return of journal entries. - - Argument `flags` sets open flags of the journal, which can be one - of, or ORed combination of constants: LOCAL_ONLY (default) opens - journal on local machine only; RUNTIME_ONLY opens only - volatile journal files; and SYSTEM_ONLY opens only - journal files of system services and the kernel. + def __init__(self, flags=None, path=None, files=None, converters=None): + """Create a new Reader. - Argument `path` is the directory of journal files. Note that - `flags` and `path` are exclusive. + Argument `flags` defines the open flags of the journal, which can be one + of, or ORed combination of constants: LOCAL_ONLY (default) opens journal + on local machine only; RUNTIME_ONLY opens only volatile journal files; + and SYSTEM_ONLY opens only journal files of system services and the kernel. + + Argument `path` is the directory of journal files, either a file system + path or a file descriptor. Note that `flags`, `path`, and `files` are + exclusive. Argument `converters` is a dictionary which updates the - DEFAULT_CONVERTERS to convert journal field values. Field - names are used as keys into this dictionary. The values must - be single argument functions, which take a `bytes` object and - return a converted value. When there's no entry for a field - name, then the default UTF-8 decoding will be attempted. If - the conversion fails with a ValueError, unconverted bytes - object will be returned. (Note that ValueEror is a superclass - of UnicodeDecodeError). + DEFAULT_CONVERTERS to convert journal field values. Field names are used + as keys into this dictionary. The values must be single argument + functions, which take a `bytes` object and return a converted + value. When there's no entry for a field name, then the default UTF-8 + decoding will be attempted. If the conversion fails with a ValueError, + unconverted bytes object will be returned. (Note that ValueEror is a + superclass of UnicodeDecodeError). + + Reader implements the context manager protocol: the journal will be + closed when exiting the block. + """ + if flags is None: + if path is None and files is None: + # This mimics journalctl behaviour of default to local journal only + flags = LOCAL_ONLY + else: + flags = 0 - Reader implements the context manager protocol: the journal - will be closed when exiting the block. - """ super(Reader, self).__init__(flags, path, files) - if _sys.version_info >= (3,3): + if _sys.version_info >= (3, 3): self.converters = _ChainMap() if converters is not None: self.converters.maps.append(converters) @@ -161,12 +183,12 @@ self.converters.update(converters) def _convert_field(self, key, value): - """Convert value using self.converters[key] + """Convert value using self.converters[key]. - If `key` is not present in self.converters, a standard unicode - decoding will be attempted. If the conversion (either - key-specific or the default one) fails with a ValueError, the - original bytes object will be returned. + If `key` is not present in self.converters, a standard unicode decoding + will be attempted. If the conversion (either key-specific or the + default one) fails with a ValueError, the original bytes object will be + returned. """ convert = self.converters.get(key, bytes.decode) try: @@ -176,7 +198,7 @@ return value def _convert_entry(self, entry): - """Convert entire journal entry utilising _covert_field""" + """Convert entire journal entry utilising _convert_field.""" result = {} for key, value in entry.items(): if isinstance(value, list): @@ -186,14 +208,18 @@ return result def __iter__(self): - """Part of iterator protocol. - Returns self. + """Return self. + + Part of the iterator protocol. """ return self def __next__(self): - """Part of iterator protocol. + """Return the next entry in the journal. + Returns self.get_next() or raises StopIteration. + + Part of the iterator protocol. """ ans = self.get_next() if ans: @@ -206,11 +232,11 @@ def add_match(self, *args, **kwargs): """Add one or more matches to the filter journal log entries. - All matches of different field are combined in a logical AND, - and matches of the same field are automatically combined in a - logical OR. - Matches can be passed as strings of form "FIELD=value", or - keyword arguments FIELD="value". + + All matches of different field are combined with logical AND, and + matches of the same field are automatically combined with logical OR. + Matches can be passed as strings of form "FIELD=value", or keyword + arguments FIELD="value". """ args = list(args) args.extend(_make_line(key, val) for key, val in kwargs.items()) @@ -218,13 +244,16 @@ super(Reader, self).add_match(arg) def get_next(self, skip=1): - """Return the next log entry as a mapping type, currently - a standard dictionary of fields. + r"""Return the next log entry as a dictionary. - Optional skip value will return the `skip`\-th log entry. + Entries will be processed with converters specified during Reader + creation. - Entries will be processed with converters specified during - Reader creation. + Optional `skip` value will return the `skip`-th log entry. + + Currently a standard dictionary of fields is returned, but in the + future this might be changed to a different mapping type, so the + calling code should not make assumptions about a specific type. """ if super(Reader, self)._next(skip): entry = super(Reader, self)._get_all() @@ -236,20 +265,24 @@ return dict() def get_previous(self, skip=1): - """Return the previous log entry as a mapping type, - currently a standard dictionary of fields. + r"""Return the previous log entry. - Optional skip value will return the -`skip`\-th log entry. + Equivalent to get_next(-skip). - Entries will be processed with converters specified during - Reader creation. + Optional `skip` value will return the -`skip`-th log entry. - Equivalent to get_next(-skip). + Entries will be processed with converters specified during Reader + creation. + + Currently a standard dictionary of fields is returned, but in the + future this might be changed to a different mapping type, so the + calling code should not make assumptions about a specific type. """ return self.get_next(-skip) def query_unique(self, field): - """Return unique values appearing in the journal for given `field`. + """Return a list of unique values appearing in the journal for the given + `field`. Note this does not respect any journal matches. @@ -260,36 +293,49 @@ for value in super(Reader, self).query_unique(field)) def wait(self, timeout=None): - """Wait for a change in the journal. `timeout` is the maximum - time in seconds to wait, or None, to wait forever. + """Wait for a change in the journal. + + `timeout` is the maximum time in seconds to wait, or None which + means to wait forever. - Returns one of NOP (no change), APPEND (new entries have been - added to the end of the journal), or INVALIDATE (journal files - have been added or removed). + Returns one of NOP (no change), APPEND (new entries have been added to + the end of the journal), or INVALIDATE (journal files have been added or + removed). """ us = -1 if timeout is None else int(timeout * 1000000) return super(Reader, self).wait(us) def seek_realtime(self, realtime): - """Seek to a matching journal entry nearest to `realtime` time. + """Seek to a matching journal entry nearest to `timestamp` time. + + Argument `realtime` must be either an integer UNIX timestamp (in + microseconds since the beginning of the UNIX epoch), or an float UNIX + timestamp (in seconds since the beginning of the UNIX epoch), or a + datetime.datetime instance. The integer form is deprecated. + + >>> import time + >>> from systemd import journal - Argument `realtime` must be either an integer unix timestamp - or datetime.datetime instance. + >>> yesterday = time.time() - 24 * 60**2 + >>> j = journal.Reader() + >>> j.seek_realtime(yesterday) """ if isinstance(realtime, _datetime.datetime): - realtime = float(realtime.strftime("%s.%f")) * 1000000 - return super(Reader, self).seek_realtime(int(realtime)) + realtime = int(float(realtime.strftime("%s.%f")) * 1000000) + elif not isinstance(realtime, int): + realtime = int(realtime * 1000000) + return super(Reader, self).seek_realtime(realtime) def seek_monotonic(self, monotonic, bootid=None): """Seek to a matching journal entry nearest to `monotonic` time. - Argument `monotonic` is a timestamp from boot in either - seconds or a datetime.timedelta instance. Argument `bootid` - is a string or UUID representing which boot the monotonic time - is reference to. Defaults to current bootid. + Argument `monotonic` is a timestamp from boot in either seconds or a + datetime.timedelta instance. Argument `bootid` is a string or UUID + representing which boot the monotonic time is reference to. Defaults to + current bootid. """ if isinstance(monotonic, _datetime.timedelta): - monotonic = monotonic.totalseconds() + monotonic = monotonic.total_seconds() monotonic = int(monotonic * 1000000) if isinstance(bootid, _uuid.UUID): bootid = bootid.hex @@ -317,7 +363,7 @@ self.add_match(MESSAGE_ID=messageid) def this_boot(self, bootid=None): - """Add match for _BOOT_ID equal to current boot ID or the specified boot ID. + """Add match for _BOOT_ID for current boot or the specified boot ID. If specified, bootid should be either a UUID or a 32 digit hex number. @@ -332,7 +378,8 @@ def this_machine(self, machineid=None): """Add match for _MACHINE_ID equal to the ID of this machine. - If specified, machineid should be either a UUID or a 32 digit hex number. + If specified, machineid should be either a UUID or a 32 digit hex + number. Equivalent to add_match(_MACHINE_ID='machineid'). """ @@ -344,47 +391,51 @@ def get_catalog(mid): + """Return catalog entry for the specified ID. + + `mid` should be either a UUID or a 32 digit hex number. + """ if isinstance(mid, _uuid.UUID): mid = mid.hex return _get_catalog(mid) + def _make_line(field, value): if isinstance(value, bytes): return field.encode('utf-8') + b'=' + value - elif isinstance(value, int): - return field + '=' + str(value) - else: + elif isinstance(value, str): return field + '=' + value + else: + return field + '=' + str(value) + def send(MESSAGE, MESSAGE_ID=None, CODE_FILE=None, CODE_LINE=None, CODE_FUNC=None, **kwargs): r"""Send a message to the journal. + >>> from systemd import journal >>> journal.send('Hello world') >>> journal.send('Hello, again, world', FIELD2='Greetings!') >>> journal.send('Binary message', BINARY=b'\xde\xad\xbe\xef') - Value of the MESSAGE argument will be used for the MESSAGE= - field. MESSAGE must be a string and will be sent as UTF-8 to - the journal. - - MESSAGE_ID can be given to uniquely identify the type of - message. It must be a string or a uuid.UUID object. - - CODE_LINE, CODE_FILE, and CODE_FUNC can be specified to - identify the caller. Unless at least on of the three is given, - values are extracted from the stack frame of the caller of - send(). CODE_FILE and CODE_FUNC must be strings, CODE_LINE - must be an integer. - - Additional fields for the journal entry can only be specified - as keyword arguments. The payload can be either a string or - bytes. A string will be sent as UTF-8, and bytes will be sent - as-is to the journal. + Value of the MESSAGE argument will be used for the MESSAGE= field. MESSAGE + must be a string and will be sent as UTF-8 to the journal. + + MESSAGE_ID can be given to uniquely identify the type of message. It must be + a string or a uuid.UUID object. + + CODE_LINE, CODE_FILE, and CODE_FUNC can be specified to identify the caller. + Unless at least on of the three is given, values are extracted from the + stack frame of the caller of send(). CODE_FILE and CODE_FUNC must be + strings, CODE_LINE must be an integer. + + Additional fields for the journal entry can only be specified as keyword + arguments. The payload can be either a string or bytes. A string will be + sent as UTF-8, and bytes will be sent as-is to the journal. - Other useful fields include PRIORITY, SYSLOG_FACILITY, - SYSLOG_IDENTIFIER, SYSLOG_PID. + Other useful fields include PRIORITY, SYSLOG_FACILITY, SYSLOG_IDENTIFIER, + SYSLOG_PID. """ args = ['MESSAGE=' + MESSAGE] @@ -393,9 +444,8 @@ id = getattr(MESSAGE_ID, 'hex', MESSAGE_ID) args.append('MESSAGE_ID=' + id) - if CODE_LINE == CODE_FILE == CODE_FUNC == None: - CODE_FILE, CODE_LINE, CODE_FUNC = \ - _traceback.extract_stack(limit=2)[0][:3] + if CODE_LINE is CODE_FILE is CODE_FUNC is None: + CODE_FILE, CODE_LINE, CODE_FUNC = _traceback.extract_stack(limit=2)[0][:3] if CODE_FILE is not None: args.append('CODE_FILE=' + CODE_FILE) if CODE_LINE is not None: @@ -406,19 +456,19 @@ args.extend(_make_line(key, val) for key, val in kwargs.items()) return sendv(*args) -def stream(identifier, priority=LOG_DEBUG, level_prefix=False): + +def stream(identifier=None, priority=LOG_INFO, level_prefix=False): r"""Return a file object wrapping a stream to journal. - Log messages written to this file as simple newline sepearted - text strings are written to the journal. + Log messages written to this file as simple newline sepearted text strings + are written to the journal. - The file will be line buffered, so messages are actually sent - after a newline character is written. + The file will be line buffered, so messages are actually sent after a + newline character is written. - >>> stream = journal.stream('myapp') - >>> stream - ', mode 'w' at 0x...> - >>> stream.write('message...\n') + >>> from systemd import journal + >>> stream = journal.stream('myapp') # doctest: +SKIP + >>> res = stream.write('message...\n') # doctest: +SKIP will produce the following message in the journal:: @@ -426,72 +476,84 @@ SYSLOG_IDENTIFIER=myapp MESSAGE=message... - Using the interface with print might be more convinient: + If identifier is None, a suitable default based on sys.argv[0] will be used. + + This interface can be used conveniently with the print function: >>> from __future__ import print_function - >>> print('message...', file=stream) + >>> stream = journal.stream() # doctest: +SKIP + >>> print('message...', file=stream) # doctest: +SKIP - priority is the syslog priority, one of `LOG_EMERG`, - `LOG_ALERT`, `LOG_CRIT`, `LOG_ERR`, `LOG_WARNING`, - `LOG_NOTICE`, `LOG_INFO`, `LOG_DEBUG`. - - level_prefix is a boolean. If true, kernel-style log priority - level prefixes (such as '<1>') are interpreted. See - sd-daemon(3) for more information. + priority is the syslog priority, one of `LOG_EMERG`, `LOG_ALERT`, + `LOG_CRIT`, `LOG_ERR`, `LOG_WARNING`, `LOG_NOTICE`, `LOG_INFO`, `LOG_DEBUG`. + + level_prefix is a boolean. If true, kernel-style log priority level prefixes + (such as '<1>') are interpreted. See sd-daemon(3) for more information. """ + if identifier is None: + if not _sys.argv or not _sys.argv[0] or _sys.argv[0] == '-c': + identifier = 'python' + else: + identifier = _sys.argv[0] + fd = stream_fd(identifier, priority, level_prefix) return _os.fdopen(fd, 'w', 1) + class JournalHandler(_logging.Handler): """Journal handler class for the Python logging framework. - Please see the Python logging module documentation for an - overview: http://docs.python.org/library/logging.html. + Please see the Python logging module documentation for an overview: + http://docs.python.org/library/logging.html. To create a custom logger whose messages go only to journal: + >>> import logging >>> log = logging.getLogger('custom_logger_name') >>> log.propagate = False - >>> log.addHandler(journal.JournalHandler()) - >>> log.warn("Some message: %s", detail) + >>> log.addHandler(JournalHandler()) + >>> log.warning("Some message: %s", 'detail') - Note that by default, message levels `INFO` and `DEBUG` are - ignored by the logging framework. To enable those log levels: + Note that by default, message levels `INFO` and `DEBUG` are ignored by the + logging framework. To enable those log levels: >>> log.setLevel(logging.DEBUG) - To redirect all logging messages to journal regardless of where - they come from, attach it to the root logger: + To redirect all logging messages to journal regardless of where they come + from, attach it to the root logger: - >>> logging.root.addHandler(journal.JournalHandler()) + >>> logging.root.addHandler(JournalHandler()) - For more complex configurations when using `dictConfig` or - `fileConfig`, specify `systemd.journal.JournalHandler` as the - handler class. Only standard handler configuration options - are supported: `level`, `formatter`, `filters`. + For more complex configurations when using `dictConfig` or `fileConfig`, + specify `systemd.journal.JournalHandler` as the handler class. Only + standard handler configuration options are supported: `level`, `formatter`, + `filters`. To attach journal MESSAGE_ID, an extra field is supported: >>> import uuid >>> mid = uuid.UUID('0123456789ABCDEF0123456789ABCDEF') - >>> log.warn("Message with ID", extra={'MESSAGE_ID': mid}) + >>> log.warning("Message with ID", extra={'MESSAGE_ID': mid}) + + Fields to be attached to all messages sent through this handler can be + specified as keyword arguments. This probably makes sense only for + SYSLOG_IDENTIFIER and similar fields which are constant for the whole + program: - Fields to be attached to all messages sent through this - handler can be specified as keyword arguments. This probably - makes sense only for SYSLOG_IDENTIFIER and similar fields - which are constant for the whole program: + >>> JournalHandler(SYSLOG_IDENTIFIER='my-cool-app') + <...JournalHandler ...> - >>> journal.JournalHandler(SYSLOG_IDENTIFIER='my-cool-app') + The following journal fields will be sent: `MESSAGE`, `PRIORITY`, + `THREAD_NAME`, `CODE_FILE`, `CODE_LINE`, `CODE_FUNC`, `LOGGER` (name as + supplied to getLogger call), `MESSAGE_ID` (optional, see above), + `SYSLOG_IDENTIFIER` (defaults to sys.argv[0]). - The following journal fields will be sent: - `MESSAGE`, `PRIORITY`, `THREAD_NAME`, `CODE_FILE`, `CODE_LINE`, - `CODE_FUNC`, `LOGGER` (name as supplied to getLogger call), - `MESSAGE_ID` (optional, see above), `SYSLOG_IDENTIFIER` (defaults - to sys.argv[0]). + The function used to actually send messages can be overridden using + the `sender_function` parameter. """ - def __init__(self, level=_logging.NOTSET, **kwargs): + def __init__(self, level=_logging.NOTSET, sender_function=send, **kwargs): super(JournalHandler, self).__init__(level) for name in kwargs: @@ -499,39 +561,54 @@ raise ValueError('Invalid field name: ' + name) if 'SYSLOG_IDENTIFIER' not in kwargs: kwargs['SYSLOG_IDENTIFIER'] = _sys.argv[0] + + self.send = sender_function self._extra = kwargs def emit(self, record): - """Write record as journal event. + """Write `record` as a journal event. - MESSAGE is taken from the message provided by the - user, and PRIORITY, LOGGER, THREAD_NAME, - CODE_{FILE,LINE,FUNC} fields are appended - automatically. In addition, record.MESSAGE_ID will be - used if present. + MESSAGE is taken from the message provided by the user, and PRIORITY, + LOGGER, THREAD_NAME, CODE_{FILE,LINE,FUNC} fields are appended + automatically. In addition, record.MESSAGE_ID will be used if present. """ try: msg = self.format(record) - pri = self.mapPriority(record.levelno) - mid = getattr(record, 'MESSAGE_ID', None) - send(msg, - MESSAGE_ID=mid, + pri = self.map_priority(record.levelno) + # defaults + extras = self._extra.copy() + + # higher priority + if record.exc_text: + extras['EXCEPTION_TEXT'] = record.exc_text + + if record.exc_info: + extras['EXCEPTION_INFO'] = record.exc_info + + if record.args: + extras['CODE_ARGS'] = str(record.args) + + # explicit arguments — highest priority + extras.update(record.__dict__) + + self.send(msg, PRIORITY=format(pri), LOGGER=record.name, THREAD_NAME=record.threadName, + PROCESS_NAME=record.processName, CODE_FILE=record.pathname, CODE_LINE=record.lineno, CODE_FUNC=record.funcName, - **self._extra) + **extras) except Exception: self.handleError(record) @staticmethod - def mapPriority(levelno): + def map_priority(levelno): """Map logging levels to journald priorities. - Since Python log level numbers are "sparse", we have - to map numbers in between the standard levels too. + Since Python log level numbers are "sparse", we have to map numbers in + between the standard levels too. """ if levelno <= _logging.DEBUG: return LOG_DEBUG @@ -545,3 +622,5 @@ return LOG_CRIT else: return LOG_ALERT + + mapPriority = map_priority