I am trying to use the python logging module to create a custom log file that records other information like host name and adds it to my DB. Below are the classes I created to do this, and the Handler part was working just fine, but now that I added a custom LogRecord class, it throws this error:
/src/lib/__init__.py", line 31, in __init__
logging.LogRecord.__init__(self, *args, **kwargs)
exceptions.TypeError: __init__() takes at most 9 arguments (10 given)
And here is how I execute it
logging.setLoggerClass(MyLogger)
log = logging.getLogger('testing')
log.addHandler(MyLogHandler())
d = {'host': '192.168.0.1'}
log.warn('Hi', d)
And here are the classes. It obviously has to do with the *args, **kwargs, but when I look at it, the *args is empty, and **kwargs only contains the d variable specified above. I don't understand the problem.
class MyLogRecord(logging.LogRecord):
def __init__(self, *args, **kwargs):
logging.LogRecord.__init__(self, *args, **kwargs) //THIS IS THE LINE IT DIES ON
self.host = 'localhost'
class MyLogFormatter(logging.Formatter):
def __init__(self, fmt, datefmt=None, host=None):
logging.Formatter.__init__(self, fmt, datefmt)
self.host = host
def format(self, record):
return logging.Formatter.format(record)
class MyLogger(logging.getLoggerClass()):
def makeRecord(self, *args, **kwargs):
return MyLogRecord(*args, **kwargs)
class MyLogHandler(logging.Handler): # Inherit from logging.Handler
def __init__(self):
# run the regular Handler __init__
logging.Handler.__init__(self)
# Our custom argument
self.mongo = MongoLogger()
def setupCustomLogger(self, name, this_host):
formatter = MyLogFormatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s - %(host)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
return logger
def emit(self, record):
# record.message is the log message
self.mongo.log(record)
class MongoLogger(object):
'''Logs messages to a MongoDB fh_admin log collection.'''
def log(self, message):
##todo write log to DB
print message
The error is telling you exactly what's wrong; you are calling the constructor with too many arguments. To see what I mean, take a look at how log-records are ordinarily constructed in the default implementation of makeRecord:
def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None):
"""
A factory method which can be overridden in subclasses to create
specialized LogRecords.
"""
rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func)
if extra is not None:
for key in extra:
if (key in ["message", "asctime"]) or (key in rv.__dict__):
raise KeyError("Attempt to overwrite %r in LogRecord" % key)
rv.__dict__[key] = extra[key]
return rv
Notice how makeRecord takes an extra param that it doesn't pass directly to LogRecord? You, on the other hand, are passing that directly to LogRecord.__init__, which is causing the error.
From here, you've got two options; you could provide a more complete implementation of makeRecord, or you could try using the LoggerAdapter class which should help you achieve the same goal with less code.
Here's an example:
# Common log info to be added to all logs reported with `log_adapter`
context = {'host': 'localhost'}
log = logging.getLogger('testing')
log.addHandler(logging.StreamHandler())
d = {'host': '192.168.0.1'}
log_adapter = logging.LoggerAdapter(log, context)
log_adapter.warning('Hi', d)
If you need to calculate the value of 'host' (for example) each time something is logged, you could make context an instance of a class that looks like a dictionary. Like so:
class LogContext(object):
def __getitem__(self, key):
if key == 'host':
return 'localhost'
raise KeyError(key)
def __iter__(self):
return iter(['host'])
log_adapter = logging.LoggerAdapter(log, LogContext())
log_adapter.warning('Hi', d)
One thing to note about LoggingAdapter, it apparently doesn't define all of the handy shortcut functions as the ordinary Logger class. That's why I've called the warning method instead of warn as you did above.
More info on LoggingAdapter and adding context to your logs can be found in the python docs.
NOTE - I didn't include MyLogHandler, MyLogFormatter, or MongoLogger in my examples as they were not relevant to the issue/error.
Related
The LogRecord in python's logging module has a LogRecord defined as:
class LogRecord(object):
"""
A LogRecord instance represents an event being logged.
LogRecord instances are created every time something is logged. They
contain all the information pertinent to the event being logged. The
main information passed in is in msg and args, which are combined
using str(msg) % args to create the message field of the record. The
record also includes information such as when the record was created,
the source line where the logging call was made, and any exception
information to be logged.
"""
def __init__(self, name, level, pathname, lineno,
msg, args, exc_info, func=None, sinfo=None, **kwargs):
"""
Initialize a logging record with interesting information.
"""
ct = time.time()
self.name = name
self.msg = msg
# more stuff below
This LogRecord gets created, for example whenever someone does a logging actions, such as:
def my_func():
logging.info('Hello %s', 'Person')
return 1
In the above logger call, the variables can be inferred as:
name = 'root' # using the root logger since calling `logging.`
level = 'INFO' # or, 20
msg = 'Hello %s'
args = ('Person',)
How does the introspection work to gather the other items from my logging call? For example:
pathname
lineno
exc_info
func
sinfo # what is this?
kwargs # is this just user-added kwargs, or something else?
For example, in your answer could you show an example of doing the introspection on a function call to gather the above information?
Nearly all of it happens in Logger.findCaller. It inspects the current frame for this information.
The kwargs argument of the default LogRecord constructor is not actually used. User supplied data is passed in through extra.
Recently I came across logging in python.
I have the following code in test.py file
import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())
logger.debug("test Message")
Now, is there any way I can print the resulting Logrecord object generated by logger.debug("test Message") because it's stated in the documentation that
LogRecord instances are created automatically by the Logger every time something is logged
https://docs.python.org/3/library/logging.html#logrecord-objects
I checked saving debug into a variable and print it
test = logger.debug("test Message")
print(test)
the output is NONE
My goal is to check/view the final Logrecord object generated by logging.debug(test.py) in the same test.py by using print() This is for my own understanding.
print(LogrecordObject.__dict__)
So how to get hold of the Logrecord object generated by logger.debug("test Message")
There is no return in debug()
# Here is the snippet for the source code
def debug(self, msg, *args, **kwargs):
if self.isEnabledFor(DEBUG):
self._log(DEBUG, msg, args, **kwargs)
If you wanna get LogRecord return, you need to redefine a debug(), you can overwrite like this:
import logging
DEBUG_LEVELV_NUM = 9
logging.addLevelName(DEBUG_LEVELV_NUM, "MY_DEBUG")
def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
sinfo = None
fn, lno, func = "(unknown file)", 0, "(unknown function)"
if exc_info:
if isinstance(exc_info, BaseException):
exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
elif not isinstance(exc_info, tuple):
exc_info = sys.exc_info()
record = self.makeRecord(self.name, level, fn, lno, msg, args,
exc_info, func, extra, sinfo)
self.handle(record)
return record
def my_debug(self, message, *args, **kws):
if self.isEnabledFor(DEBUG_LEVELV_NUM):
# Yes, logger takes its '*args' as 'args'.
record = self._log(DEBUG_LEVELV_NUM, message, args, **kws)
return record
logger = logging.getLogger(__name__)
logging.Logger.my_debug = my_debug
logging.Logger._log = _log
logger.setLevel(DEBUG_LEVELV_NUM)
logger.addHandler(logging.StreamHandler())
test = logger.my_debug('test custom debug')
print(test)
Reference:
How to add a custom loglevel to Python's logging facility
You can create a handler that instead of formatting the LogRecord instance to a string, just save it in a list to be viewed and inspected later:
import logging
import sys
# A new handler to store "raw" LogRecords instances
class RecordsListHandler(logging.Handler):
"""
A handler class which stores LogRecord entries in a list
"""
def __init__(self, records_list):
"""
Initiate the handler
:param records_list: a list to store the LogRecords entries
"""
self.records_list = records_list
super().__init__()
def emit(self, record):
self.records_list.append(record)
# A list to store the "raw" LogRecord instances
logs_list = []
# Your logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Add the regular stream handler to print logs to the console, if you like
logger.addHandler(logging.StreamHandler(sys.stdout))
# Add the RecordsListHandler to store the log records objects
logger.addHandler(RecordsListHandler(logs_list))
if __name__ == '__main__':
logger.debug("test Message")
print(logs_list)
Output:
test Message
[<LogRecord: __main__, 10, C:/Automation/Exercises/222.py, 36, "test Message">]
I created a custom log level as below:
class MyLoggerAdapter(logging.LoggerAdapter):
def __init__(self, logger, extra):
logging.LoggerAdapter.__init__(self, logger, extra)
self.logger = logger
logging.addLevelName(25, "NOTICE")
def notice(self, msg, *args, **kwargs):
self.logger.log(25, msg, *args, **kwargs)
The issue I have is that my logger format uses %(pathname)s and %(lineno)d, and whenever I call logger.notice() those values refer to the line within the definition of the notice method, rather than the location that the method was called from. How do I add the notice log level but print the correct location of the message to the logs?
I use logging facility for python 2.7.3. Documentation for this Python version say:
the logging package pre-dates newer formatting options such as str.format() and string.Template. These newer formatting options are supported...
I like 'new' format with curly braces. So i'm trying to do something like:
log = logging.getLogger("some.logger")
log.debug("format this message {0}", 1)
And get error:
TypeError: not all arguments converted during string formatting
What I miss here?
P.S. I don't want to use
log.debug("format this message {0}".format(1))
because in this case the message is always being formatted regardless of logger level.
EDIT: take a look at the StyleAdapter approach in #Dunes' answer unlike this answer; it allows to use alternative formatting styles without the boilerplate while calling logger's methods (debug(), info(), error(), etc).
From the docs — Use of alternative formatting styles:
Logging calls (logger.debug(), logger.info() etc.) only take
positional parameters for the actual logging message itself, with
keyword parameters used only for determining options for how to handle
the actual logging call (e.g. the exc_info keyword parameter to
indicate that traceback information should be logged, or the extra
keyword parameter to indicate additional contextual information to be
added to the log). So you cannot directly make logging calls using
str.format() or string.Template syntax, because internally the logging
package uses %-formatting to merge the format string and the variable
arguments. There would no changing this while preserving backward
compatibility, since all logging calls which are out there in existing
code will be using %-format strings.
And:
There is, however, a way that you can use {}- and $- formatting to
construct your individual log messages. Recall that for a message you
can use an arbitrary object as a message format string, and that the
logging package will call str() on that object to get the actual
format string.
Copy-paste this to wherever module:
class BraceMessage(object):
def __init__(self, fmt, *args, **kwargs):
self.fmt = fmt
self.args = args
self.kwargs = kwargs
def __str__(self):
return self.fmt.format(*self.args, **self.kwargs)
Then:
from wherever import BraceMessage as __
log.debug(__('Message with {0} {name}', 2, name='placeholders'))
Note: actual formatting is delayed until it is necessary e.g., if DEBUG messages are not logged then the formatting is not performed at all.
Here is another option that does not have the keyword problems mentioned in Dunes' answer. It can only handle positional ({0}) arguments and not keyword ({foo}) arguments. It also does not require two calls to format (using the underscore). It does have the ick-factor of subclassing str:
class BraceString(str):
def __mod__(self, other):
return self.format(*other)
def __str__(self):
return self
class StyleAdapter(logging.LoggerAdapter):
def __init__(self, logger, extra=None):
super(StyleAdapter, self).__init__(logger, extra)
def process(self, msg, kwargs):
if kwargs.pop('style', "%") == "{": # optional
msg = BraceString(msg)
return msg, kwargs
You use it like this:
logger = StyleAdapter(logging.getLogger(__name__))
logger.info("knights:{0}", "ni", style="{")
logger.info("knights:{}", "shrubbery", style="{")
Of course, you can remove the check noted with # optional to force all messages through the adapter to use new-style formatting.
Note for anyone reading this answer years later: Starting with Python 3.2, you can use the style parameter with Formatter objects:
Logging (as of 3.2) provides improved support for these two additional formatting styles. The
Formatter class been enhanced to take an additional, optional keyword parameter named style. This
defaults to '%', but other possible values are '{' and '$', which correspond to the other two
formatting styles. Backwards compatibility is maintained by default (as you would expect), but by
explicitly specifying a style parameter, you get the ability to specify format strings which work
with str.format() or
string.Template.
The docs provide the example
logging.Formatter('{asctime} {name} {levelname:8s} {message}', style='{')
Note that in this case you still can't call the logger with the new format. I.e., the following still won't work:
logger.info("knights:{say}", say="ni") # Doesn't work!
logger.info("knights:{0}", "ni") # Doesn't work either
This was my solution to the problem when I found logging only uses printf style formatting. It allows logging calls to remain the same -- no special syntax such as log.info(__("val is {}", "x")). The change required to code is to wrap the logger in a StyleAdapter.
from inspect import getargspec
class BraceMessage(object):
def __init__(self, fmt, args, kwargs):
self.fmt = fmt
self.args = args
self.kwargs = kwargs
def __str__(self):
return str(self.fmt).format(*self.args, **self.kwargs)
class StyleAdapter(logging.LoggerAdapter):
def __init__(self, logger):
self.logger = logger
def log(self, level, msg, *args, **kwargs):
if self.isEnabledFor(level):
msg, log_kwargs = self.process(msg, kwargs)
self.logger._log(level, BraceMessage(msg, args, kwargs), (),
**log_kwargs)
def process(self, msg, kwargs):
return msg, {key: kwargs[key]
for key in getargspec(self.logger._log).args[1:] if key in kwargs}
Usage is:
log = StyleAdapter(logging.getLogger(__name__))
log.info("a log message using {type} substitution", type="brace")
It's worth noting that this implementation has problems if key words used for brace substitution include level, msg, args, exc_info, extra or stack_info. These are argument names used by the log method of Logger. If you need to one of these names then modify process to exclude these names or just remove log_kwargs from the _log call. On a further note, this implementation also silently ignores misspelled keywords meant for the Logger (eg. ectra).
The easier solution would be to use the excellent logbook module
import logbook
import sys
logbook.StreamHandler(sys.stdout).push_application()
logbook.debug('Format this message {k}', k=1)
Or the more complete:
>>> import logbook
>>> import sys
>>> logbook.StreamHandler(sys.stdout).push_application()
>>> log = logbook.Logger('MyLog')
>>> log.debug('Format this message {k}', k=1)
[2017-05-06 21:46:52.578329] DEBUG: MyLog: Format this message 1
UPDATE: There is now a package on PyPI called bracelogger that implements the solution detailed below.
To enable brace-style formatting for log messages, we can monkey-patch a bit of the logger code.
The following patches the logging module to create a get_logger function that will return a logger that uses the new-style formatting for every log record that it handles.
import functools
import logging
import types
def _get_message(record):
"""Replacement for logging.LogRecord.getMessage
that uses the new-style string formatting for
its messages"""
msg = str(record.msg)
args = record.args
if args:
if not isinstance(args, tuple):
args = (args,)
msg = msg.format(*args)
return msg
def _handle_wrap(fcn):
"""Wrap the handle function to replace the passed in
record's getMessage function before calling handle"""
#functools.wraps(fcn)
def handle(record):
record.getMessage = types.MethodType(_get_message, record)
return fcn(record)
return handle
def get_logger(name=None):
"""Get a logger instance that uses new-style string formatting"""
log = logging.getLogger(name)
if not hasattr(log, "_newstyle"):
log.handle = _handle_wrap(log.handle)
log._newstyle = True
return log
Usage:
>>> log = get_logger()
>>> log.warning("{!r}", log)
<logging.RootLogger object at 0x4985a4d3987b>
Notes:
Fully compatible with normal logging methods (just replace logging.getLogger with get_logger)
Will only affect specific loggers created by the get_logger function (doesn't break 3rd party packages).
The formatting of the message is delayed until it is output (or not at all if the log message is filtered).
Args are stored on logging.LogRecord objects as usual (useful in some cases with custom log handlers).
Works in all versions of Python from 2.7 to 3.10.
Try logging.setLogRecordFactory in Python 3.2+:
import collections
import logging
class _LogRecord(logging.LogRecord):
def getMessage(self):
msg = str(self.msg)
if self.args:
if isinstance(self.args, collections.Mapping):
msg = msg.format(**self.args)
else:
msg = msg.format(*self.args)
return msg
logging.setLogRecordFactory(_LogRecord)
I created a custom Formatter, called ColorFormatter that handles the problem like this:
class ColorFormatter(logging.Formatter):
def format(self, record):
# previous stuff, copy from logging.py…
try: # Allow {} style
message = record.getMessage() # printf
except TypeError:
message = record.msg.format(*record.args)
# later stuff…
This keeps it compatible with various libraries.
The drawback is that it is probably not performant due to potentially attempting format of the string twice.
Similar solution to pR0Ps' , wrapping getMessage in LogRecord by wrapping makeRecord (instead of handle in their answer) in instances of Logger that should be new-formatting-enabled:
def getLogger(name):
log = logging.getLogger(name)
def Logger_makeRecordWrapper(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None):
self = log
record = logging.Logger.makeRecord(self, name, level, fn, lno, msg, args, exc_info, func, sinfo)
def LogRecord_getMessageNewStyleFormatting():
self = record
msg = str(self.msg)
if self.args:
msg = msg.format(*self.args)
return msg
record.getMessage = LogRecord_getMessageNewStyleFormatting
return record
log.makeRecord = Logger_makeRecordWrapper
return log
I tested this with Python 3.5.3.
Combined string.Formatter to add pprint.pformat type conversion and from logging: setLogRecordFactory, setLoggerClass. There's one neat trick- i create extra nested tuple for argument args for Logger._log method and then unpack it in LogRecord init to omit overriding in Logger.makeRecord. Using log.f wraps every attribute (log methods on purpose) with use_format so you don't have to write it explicitly. This solution is backward compatible.
from collections import namedtuple
from collections.abc import Mapping
from functools import partial
from pprint import pformat
from string import Formatter
import logging
Logger = logging.getLoggerClass()
LogRecord = logging.getLogRecordFactory()
class CustomFormatter(Formatter):
def format_field(self, value, format_spec):
if format_spec.endswith('p'):
value = pformat(value)
format_spec = format_spec[:-1]
return super().format_field(value, format_spec)
custom_formatter = CustomFormatter()
class LogWithFormat:
def __init__(self, obj):
self.obj = obj
def __getattr__(self, name):
return partial(getattr(self.obj, name), use_format=True)
ArgsSmuggler = namedtuple('ArgsSmuggler', ('args', 'smuggled'))
class CustomLogger(Logger):
def __init__(self, *ar, **kw):
super().__init__(*ar, **kw)
self.f = LogWithFormat(self)
def _log(self, level, msg, args, *ar, use_format=False, **kw):
super()._log(level, msg, ArgsSmuggler(args, use_format), *ar, **kw)
class CustomLogRecord(LogRecord):
def __init__(self, *ar, **kw):
args = ar[5]
# RootLogger use CustomLogRecord but not CustomLogger
# then just unpack only ArgsSmuggler instance
args, use_format = args if isinstance(args, ArgsSmuggler) else (args, False)
super().__init__(*ar[:5], args, *ar[6:], **kw)
self.use_format = use_format
def getMessage(self):
return self.getMessageWithFormat() if self.use_format else super().getMessage()
def getMessageWithFormat(self):
msg = str(self.msg)
args = self.args
if args:
fmt = custom_formatter.format
msg = fmt(msg, **args) if isinstance(args, Mapping) else fmt(msg, *args)
return msg
logging.setLogRecordFactory(CustomLogRecord)
logging.setLoggerClass(CustomLogger)
log = logging.getLogger(__name__)
log.info('%s %s', dict(a=1, b=2), 5)
log.f.info('{:p} {:d}', dict(a=1, b=2), 5)
Here's something real simple that works:
debug_logger: logging.Logger = logging.getLogger("app.debug")
def mydebuglog(msg: str, *args, **kwargs):
if debug_logger.isEnabledFor(logging.DEBUG):
debug_logger.debug(msg.format(*args, **kwargs))
Then:
mydebuglog("hello {} {val}", "Python", val="World")
I use logging facility for python 2.7.3. Documentation for this Python version say:
the logging package pre-dates newer formatting options such as str.format() and string.Template. These newer formatting options are supported...
I like 'new' format with curly braces. So i'm trying to do something like:
log = logging.getLogger("some.logger")
log.debug("format this message {0}", 1)
And get error:
TypeError: not all arguments converted during string formatting
What I miss here?
P.S. I don't want to use
log.debug("format this message {0}".format(1))
because in this case the message is always being formatted regardless of logger level.
EDIT: take a look at the StyleAdapter approach in #Dunes' answer unlike this answer; it allows to use alternative formatting styles without the boilerplate while calling logger's methods (debug(), info(), error(), etc).
From the docs — Use of alternative formatting styles:
Logging calls (logger.debug(), logger.info() etc.) only take
positional parameters for the actual logging message itself, with
keyword parameters used only for determining options for how to handle
the actual logging call (e.g. the exc_info keyword parameter to
indicate that traceback information should be logged, or the extra
keyword parameter to indicate additional contextual information to be
added to the log). So you cannot directly make logging calls using
str.format() or string.Template syntax, because internally the logging
package uses %-formatting to merge the format string and the variable
arguments. There would no changing this while preserving backward
compatibility, since all logging calls which are out there in existing
code will be using %-format strings.
And:
There is, however, a way that you can use {}- and $- formatting to
construct your individual log messages. Recall that for a message you
can use an arbitrary object as a message format string, and that the
logging package will call str() on that object to get the actual
format string.
Copy-paste this to wherever module:
class BraceMessage(object):
def __init__(self, fmt, *args, **kwargs):
self.fmt = fmt
self.args = args
self.kwargs = kwargs
def __str__(self):
return self.fmt.format(*self.args, **self.kwargs)
Then:
from wherever import BraceMessage as __
log.debug(__('Message with {0} {name}', 2, name='placeholders'))
Note: actual formatting is delayed until it is necessary e.g., if DEBUG messages are not logged then the formatting is not performed at all.
Here is another option that does not have the keyword problems mentioned in Dunes' answer. It can only handle positional ({0}) arguments and not keyword ({foo}) arguments. It also does not require two calls to format (using the underscore). It does have the ick-factor of subclassing str:
class BraceString(str):
def __mod__(self, other):
return self.format(*other)
def __str__(self):
return self
class StyleAdapter(logging.LoggerAdapter):
def __init__(self, logger, extra=None):
super(StyleAdapter, self).__init__(logger, extra)
def process(self, msg, kwargs):
if kwargs.pop('style', "%") == "{": # optional
msg = BraceString(msg)
return msg, kwargs
You use it like this:
logger = StyleAdapter(logging.getLogger(__name__))
logger.info("knights:{0}", "ni", style="{")
logger.info("knights:{}", "shrubbery", style="{")
Of course, you can remove the check noted with # optional to force all messages through the adapter to use new-style formatting.
Note for anyone reading this answer years later: Starting with Python 3.2, you can use the style parameter with Formatter objects:
Logging (as of 3.2) provides improved support for these two additional formatting styles. The
Formatter class been enhanced to take an additional, optional keyword parameter named style. This
defaults to '%', but other possible values are '{' and '$', which correspond to the other two
formatting styles. Backwards compatibility is maintained by default (as you would expect), but by
explicitly specifying a style parameter, you get the ability to specify format strings which work
with str.format() or
string.Template.
The docs provide the example
logging.Formatter('{asctime} {name} {levelname:8s} {message}', style='{')
Note that in this case you still can't call the logger with the new format. I.e., the following still won't work:
logger.info("knights:{say}", say="ni") # Doesn't work!
logger.info("knights:{0}", "ni") # Doesn't work either
This was my solution to the problem when I found logging only uses printf style formatting. It allows logging calls to remain the same -- no special syntax such as log.info(__("val is {}", "x")). The change required to code is to wrap the logger in a StyleAdapter.
from inspect import getargspec
class BraceMessage(object):
def __init__(self, fmt, args, kwargs):
self.fmt = fmt
self.args = args
self.kwargs = kwargs
def __str__(self):
return str(self.fmt).format(*self.args, **self.kwargs)
class StyleAdapter(logging.LoggerAdapter):
def __init__(self, logger):
self.logger = logger
def log(self, level, msg, *args, **kwargs):
if self.isEnabledFor(level):
msg, log_kwargs = self.process(msg, kwargs)
self.logger._log(level, BraceMessage(msg, args, kwargs), (),
**log_kwargs)
def process(self, msg, kwargs):
return msg, {key: kwargs[key]
for key in getargspec(self.logger._log).args[1:] if key in kwargs}
Usage is:
log = StyleAdapter(logging.getLogger(__name__))
log.info("a log message using {type} substitution", type="brace")
It's worth noting that this implementation has problems if key words used for brace substitution include level, msg, args, exc_info, extra or stack_info. These are argument names used by the log method of Logger. If you need to one of these names then modify process to exclude these names or just remove log_kwargs from the _log call. On a further note, this implementation also silently ignores misspelled keywords meant for the Logger (eg. ectra).
The easier solution would be to use the excellent logbook module
import logbook
import sys
logbook.StreamHandler(sys.stdout).push_application()
logbook.debug('Format this message {k}', k=1)
Or the more complete:
>>> import logbook
>>> import sys
>>> logbook.StreamHandler(sys.stdout).push_application()
>>> log = logbook.Logger('MyLog')
>>> log.debug('Format this message {k}', k=1)
[2017-05-06 21:46:52.578329] DEBUG: MyLog: Format this message 1
UPDATE: There is now a package on PyPI called bracelogger that implements the solution detailed below.
To enable brace-style formatting for log messages, we can monkey-patch a bit of the logger code.
The following patches the logging module to create a get_logger function that will return a logger that uses the new-style formatting for every log record that it handles.
import functools
import logging
import types
def _get_message(record):
"""Replacement for logging.LogRecord.getMessage
that uses the new-style string formatting for
its messages"""
msg = str(record.msg)
args = record.args
if args:
if not isinstance(args, tuple):
args = (args,)
msg = msg.format(*args)
return msg
def _handle_wrap(fcn):
"""Wrap the handle function to replace the passed in
record's getMessage function before calling handle"""
#functools.wraps(fcn)
def handle(record):
record.getMessage = types.MethodType(_get_message, record)
return fcn(record)
return handle
def get_logger(name=None):
"""Get a logger instance that uses new-style string formatting"""
log = logging.getLogger(name)
if not hasattr(log, "_newstyle"):
log.handle = _handle_wrap(log.handle)
log._newstyle = True
return log
Usage:
>>> log = get_logger()
>>> log.warning("{!r}", log)
<logging.RootLogger object at 0x4985a4d3987b>
Notes:
Fully compatible with normal logging methods (just replace logging.getLogger with get_logger)
Will only affect specific loggers created by the get_logger function (doesn't break 3rd party packages).
The formatting of the message is delayed until it is output (or not at all if the log message is filtered).
Args are stored on logging.LogRecord objects as usual (useful in some cases with custom log handlers).
Works in all versions of Python from 2.7 to 3.10.
Try logging.setLogRecordFactory in Python 3.2+:
import collections
import logging
class _LogRecord(logging.LogRecord):
def getMessage(self):
msg = str(self.msg)
if self.args:
if isinstance(self.args, collections.Mapping):
msg = msg.format(**self.args)
else:
msg = msg.format(*self.args)
return msg
logging.setLogRecordFactory(_LogRecord)
I created a custom Formatter, called ColorFormatter that handles the problem like this:
class ColorFormatter(logging.Formatter):
def format(self, record):
# previous stuff, copy from logging.py…
try: # Allow {} style
message = record.getMessage() # printf
except TypeError:
message = record.msg.format(*record.args)
# later stuff…
This keeps it compatible with various libraries.
The drawback is that it is probably not performant due to potentially attempting format of the string twice.
Similar solution to pR0Ps' , wrapping getMessage in LogRecord by wrapping makeRecord (instead of handle in their answer) in instances of Logger that should be new-formatting-enabled:
def getLogger(name):
log = logging.getLogger(name)
def Logger_makeRecordWrapper(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None):
self = log
record = logging.Logger.makeRecord(self, name, level, fn, lno, msg, args, exc_info, func, sinfo)
def LogRecord_getMessageNewStyleFormatting():
self = record
msg = str(self.msg)
if self.args:
msg = msg.format(*self.args)
return msg
record.getMessage = LogRecord_getMessageNewStyleFormatting
return record
log.makeRecord = Logger_makeRecordWrapper
return log
I tested this with Python 3.5.3.
Combined string.Formatter to add pprint.pformat type conversion and from logging: setLogRecordFactory, setLoggerClass. There's one neat trick- i create extra nested tuple for argument args for Logger._log method and then unpack it in LogRecord init to omit overriding in Logger.makeRecord. Using log.f wraps every attribute (log methods on purpose) with use_format so you don't have to write it explicitly. This solution is backward compatible.
from collections import namedtuple
from collections.abc import Mapping
from functools import partial
from pprint import pformat
from string import Formatter
import logging
Logger = logging.getLoggerClass()
LogRecord = logging.getLogRecordFactory()
class CustomFormatter(Formatter):
def format_field(self, value, format_spec):
if format_spec.endswith('p'):
value = pformat(value)
format_spec = format_spec[:-1]
return super().format_field(value, format_spec)
custom_formatter = CustomFormatter()
class LogWithFormat:
def __init__(self, obj):
self.obj = obj
def __getattr__(self, name):
return partial(getattr(self.obj, name), use_format=True)
ArgsSmuggler = namedtuple('ArgsSmuggler', ('args', 'smuggled'))
class CustomLogger(Logger):
def __init__(self, *ar, **kw):
super().__init__(*ar, **kw)
self.f = LogWithFormat(self)
def _log(self, level, msg, args, *ar, use_format=False, **kw):
super()._log(level, msg, ArgsSmuggler(args, use_format), *ar, **kw)
class CustomLogRecord(LogRecord):
def __init__(self, *ar, **kw):
args = ar[5]
# RootLogger use CustomLogRecord but not CustomLogger
# then just unpack only ArgsSmuggler instance
args, use_format = args if isinstance(args, ArgsSmuggler) else (args, False)
super().__init__(*ar[:5], args, *ar[6:], **kw)
self.use_format = use_format
def getMessage(self):
return self.getMessageWithFormat() if self.use_format else super().getMessage()
def getMessageWithFormat(self):
msg = str(self.msg)
args = self.args
if args:
fmt = custom_formatter.format
msg = fmt(msg, **args) if isinstance(args, Mapping) else fmt(msg, *args)
return msg
logging.setLogRecordFactory(CustomLogRecord)
logging.setLoggerClass(CustomLogger)
log = logging.getLogger(__name__)
log.info('%s %s', dict(a=1, b=2), 5)
log.f.info('{:p} {:d}', dict(a=1, b=2), 5)
Here's something real simple that works:
debug_logger: logging.Logger = logging.getLogger("app.debug")
def mydebuglog(msg: str, *args, **kwargs):
if debug_logger.isEnabledFor(logging.DEBUG):
debug_logger.debug(msg.format(*args, **kwargs))
Then:
mydebuglog("hello {} {val}", "Python", val="World")