Develop Reference

python after logging.debug() how to view its logrecord

Recently I came across logging in python.
I have the following code in test.py file
import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())
logger.debug("test Message")
Now, is there any way I can print the resulting Logrecord object generated by logger.debug("test Message") because it's stated in the documentation that
LogRecord instances are created automatically by the Logger every time something is logged
https://docs.python.org/3/library/logging.html#logrecord-objects
I checked saving debug into a variable and print it
test = logger.debug("test Message")
print(test)
the output is NONE
My goal is to check/view the final Logrecord object generated by logging.debug(test.py) in the same test.py by using print() This is for my own understanding.
print(LogrecordObject.__dict__)
So how to get hold of the Logrecord object generated by logger.debug("test Message")

There is no return in debug()
# Here is the snippet for the source code
def debug(self, msg, *args, **kwargs):
if self.isEnabledFor(DEBUG):
self._log(DEBUG, msg, args, **kwargs)
If you wanna get LogRecord return, you need to redefine a debug(), you can overwrite like this:
import logging
DEBUG_LEVELV_NUM = 9
logging.addLevelName(DEBUG_LEVELV_NUM, "MY_DEBUG")
def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
sinfo = None
fn, lno, func = "(unknown file)", 0, "(unknown function)"
if exc_info:
if isinstance(exc_info, BaseException):
exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
elif not isinstance(exc_info, tuple):
exc_info = sys.exc_info()
record = self.makeRecord(self.name, level, fn, lno, msg, args,
exc_info, func, extra, sinfo)
self.handle(record)
return record
def my_debug(self, message, *args, **kws):
if self.isEnabledFor(DEBUG_LEVELV_NUM):
# Yes, logger takes its '*args' as 'args'.
record = self._log(DEBUG_LEVELV_NUM, message, args, **kws)
return record
logger = logging.getLogger(__name__)
logging.Logger.my_debug = my_debug
logging.Logger._log = _log
logger.setLevel(DEBUG_LEVELV_NUM)
logger.addHandler(logging.StreamHandler())
test = logger.my_debug('test custom debug')
print(test)
Reference:
How to add a custom loglevel to Python's logging facility

You can create a handler that instead of formatting the LogRecord instance to a string, just save it in a list to be viewed and inspected later:
import logging
import sys
# A new handler to store "raw" LogRecords instances
class RecordsListHandler(logging.Handler):
"""
A handler class which stores LogRecord entries in a list
"""
def __init__(self, records_list):
"""
Initiate the handler
:param records_list: a list to store the LogRecords entries
"""
self.records_list = records_list
super().__init__()
def emit(self, record):
self.records_list.append(record)
# A list to store the "raw" LogRecord instances
logs_list = []
# Your logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Add the regular stream handler to print logs to the console, if you like
logger.addHandler(logging.StreamHandler(sys.stdout))
# Add the RecordsListHandler to store the log records objects
logger.addHandler(RecordsListHandler(logs_list))
if __name__ == '__main__':
logger.debug("test Message")
print(logs_list)
Output:
test Message
[<LogRecord: __main__, 10, C:/Automation/Exercises/222.py, 36, "test Message">]

How to buffer logs from multithreaded function call so that they can be logged in order the functions finish?

the problem
I'm trying to use the concurrent.futures library to run a function on a list of "things". The code looks something like this.
import concurrent.futures
import logging
logger = logging.getLogger(__name__)
def process_thing(thing, count):
logger.info(f'starting processing for thing {count}')
# Do some io related stuff
logger.info(f'finished processing for thing {count}')
def process_things_concurrently(things)
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
for count, thing in enumerate(things):
futures.append(executor.submit(process_thing, thing, count))
for future in concurrent.futures.as_completed(futures):
future.result()
As the code is now, the logging can happen in any order.
For example:
starting processing for thing 2
starting processing for thing 1
finished processing for thing 2
finished processing for thing 1
I want to change the code so that the records for a particular call of process_thing() are buffered until the future finishes.
In other words, all of the records for a particular call stick together. These 'groups' of records are ordered by when the call finished.
So from the example above the log output above would instead look like
starting processing for thing 2
finished processing for thing 2
starting processing for thing 1
finished processing for thing 1
what I've tried
I tried making a logger for each call that would have its own custom handler, possibly subclassing BufferingHandler. But eventually there will be lots of "things" and I read that making a lot of loggers is bad.
I'm open to anything that works! Thanks.

Here's a little recipe for a DelaydLogger class that puts all calls to logger's methods into a list instead of actually performing the call, until you finally do a flush where they are all fired up.
from functools import partial
class DelayedLogger:
def __init__(self, logger):
self.logger = logger
self._call_stack = [] # list of (method, *args, **kwargs) tuples
self._delayed_methods = {
name : partial(self._delayed_method_proxy, getattr(logger, name))
for name in ["info", "debug", "warning", "error", "critical"]
}
def __getattr__(self, name):
""" Proxy getattr to self.logger, except for self._delayed_methods. """
return self._delayed_methods.get(name, getattr(self.logger, name))
def _delayed_method_proxy(self, method, *args, **kwargs):
self._call_stack.append((method, args, kwargs))
def flush(self):
""" Flush self._call_stack to the real logger. """
for method, args, kwargs in self._call_stack:
method(*args, **kwargs)
self._call_stack = []
In your example, you could use it like so:
import logging
logger = logging.getLogger(__name__)
def process_thing(thing, count):
dlogger = DelayedLogger(logger)
dlogger.info(f'starting processing for thing {count}')
# Do some io related stuff
dlogger.info(f'finished processing for thing {count}')
dlogger.flush()
process_thing(None, 10)
There may be ways to beautfiy this or make it more compact, but it should get the job done if that's what you really want.

First I modified #Jeronimo's answer to come up with this
class DelayedLogger:
class ThreadLogger:
"""to be logged from a single thread"""
def __init__(self, logger):
self._call_stack = [] # list of (method, *args, **kwargs) tuples
self.logger = logger
self._delayed_methods = {
name: partial(self._delayed_method_proxy, getattr(logger, name))
for name in ["info", "debug", "warning", "error", "critical"]
}
def __getattr__(self, name):
""" Proxy getattr to self.logger, except for self._delayed_methods. """
return self._delayed_methods.get(name, getattr(self.logger, name))
def _delayed_method_proxy(self, method, *args, **kwargs):
self._call_stack.append((method, args, kwargs))
def flush(self):
""" Flush self._call_stack to the real logger. """
for method, args, kwargs in self._call_stack:
method(*args, **kwargs)
self._call_stack = []
def __init__(self, logger):
self.logger = logger
self._thread_loggers: typing.Dict[self.ThreadLogger] = {}
def new_thread(self, count):
"""Make a new sub-logger class that writes to the call stack in its slot"""
new_logger = self.ThreadLogger(self.logger)
self._thread_loggers[count] = new_logger
return new_logger
def get_thread(self, count):
return self._thread_loggers[count]
delayed_logger = DelayedLogger(logger)
Which can be used like this
delayed_logger = DelayedLogger(logger)
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
for count, thing in enumerate(things):
futures.append(executor.submit(process_thing,
count,
thing,
logger=delayed_logger.new_thread(count)))
for future in concurrent.futures.as_completed(futures):
count = future.result()
delayed_logger.get_thread(count).flush()
The problem here is that process_thing() now needs to take the logger as an argument and the logger is limited in scope. If process_thing() calls subroutines then the their logging won't be delayed.
Probably the solution is just to not try to do this at all. Instead threads can make a log filter or some other way to distinguish their messages.

Change log-level via mocking

I want to change the log-level temporarily.
My current strategy is to use mocking.
with mock.patch(...):
my_method_which_does_log()
All logging.info() calls inside the method should get ignored and not logged to the console.
How to implement the ... to make logs of level INFO get ignored?
The code is single-process and single-thread and executed during testing only.

I want to change the log-level temporarily.
A way to do this without mocking is logging.disable
class TestSomething(unittest.TestCase):
def setUp(self):
logging.disable(logging.WARNING)
def tearDown(self):
logging.disable(logging.NOTSET)
This example would only show messages of level WARNING and above for each test in the TestSomething class. (You call disable at the start and end of each test as needed. This seems a bit cleaner.)
To unset this temporary throttling, call logging.disable(logging.NOTSET):
If logging.disable(logging.NOTSET) is called, it effectively removes this overriding level, so that logging output again depends on the effective levels of individual loggers.

I don't think mocking is going to do what you want. The loggers are presumably already instantiated in this scenario, and level is an instance variable for each of the loggers (and also any of the handlers that each logger has).
You can create a custom context manager. That would look something like this:
Context Manager
import logging
class override_logging_level():
"A context manager for temporarily setting the logging level"
def __init__(self, level, process_handlers=True):
self.saved_level = {}
self.level = level
self.process_handlers = process_handlers
def __enter__(self):
# Save the root logger
self.save_logger('', logging.getLogger())
# Iterate over the other loggers
for name, logger in logging.Logger.manager.loggerDict.items():
self.save_logger(name, logger)
def __exit__(self, exception_type, exception_value, traceback):
# Restore the root logger
self.restore_logger('', logging.getLogger())
# Iterate over the loggers
for name, logger in logging.Logger.manager.loggerDict.items():
self.restore_logger(name, logger)
def save_logger(self, name, logger):
# Save off the level
self.saved_level[name] = logger.level
# Override the level
logger.setLevel(self.level)
if not self.process_handlers:
return
# Iterate over the handlers for this logger
for handler in logger.handlers:
# No reliable name. Just use the id of the object
self.saved_level[id(handler)] = handler.level
def restore_logger(self, name, logger):
# It's possible that some intervening code added one or more loggers...
if name not in self.saved_level:
return
# Restore the level for the logger
logger.setLevel(self.saved_level[name])
if not self.process_handlers:
return
# Iterate over the handlers for this logger
for handler in logger.handlers:
# Reconstruct the key for this handler
key = id(handler)
# Again, we could have possibly added more handlers
if key not in self.saved_level:
continue
# Restore the level for the handler
handler.setLevel(self.saved_level[key])
Test Code
# Setup for basic logging
logging.basicConfig(level=logging.ERROR)
# Create some loggers - the root logger and a couple others
lr = logging.getLogger()
l1 = logging.getLogger('L1')
l2 = logging.getLogger('L2')
# Won't see this message due to the level
lr.info("lr - msg 1")
l1.info("l1 - msg 1")
l2.info("l2 - msg 1")
# Temporarily override the level
with override_logging_level(logging.INFO):
# Will see
lr.info("lr - msg 2")
l1.info("l1 - msg 2")
l2.info("l2 - msg 2")
# Won't see, again...
lr.info("lr - msg 3")
l1.info("l1 - msg 3")
l2.info("l2 - msg 3")
Results
$ python ./main.py
INFO:root:lr - msg 2
INFO:L1:l1 - msg 2
INFO:L2:l2 - msg 2
Notes
The code would need to be enhanced to support multithreading; for example, logging.Logger.manager.loggerDict is a shared variable that's guarded by locks in the logging code.

Using #cryptoplex's approach of using Context Managers, here's the official version from the logging cookbook:
import logging
import sys
class LoggingContext(object):
def __init__(self, logger, level=None, handler=None, close=True):
self.logger = logger
self.level = level
self.handler = handler
self.close = close
def __enter__(self):
if self.level is not None:
self.old_level = self.logger.level
self.logger.setLevel(self.level)
if self.handler:
self.logger.addHandler(self.handler)
def __exit__(self, et, ev, tb):
if self.level is not None:
self.logger.setLevel(self.old_level)
if self.handler:
self.logger.removeHandler(self.handler)
if self.handler and self.close:
self.handler.close()
# implicit return of None => don't swallow exceptions

You could use dependency injection to pass the logger instance to the method you are testing. It is a bit more invasive though since you are changing your method a little, however it gives you more flexibility.
Add the logger parameter to your method signature, something along the lines of:
def my_method( your_other_params, logger):
pass
In your unit test file:
if __name__ == "__main__":
# define the logger you want to use:
logging.basicConfig( stream=sys.stderr )
logging.getLogger( "MyTests.test_my_method" ).setLevel( logging.DEBUG )
...
def test_my_method(self):
test_logger = logging.getLogger( "MyTests.test_my_method" )
# pass your logger to your method
my_method(your_normal_parameters, test_logger)
python logger docs: https://docs.python.org/3/library/logging.html

I use this pattern to write all logs to a list. It ignores logs of level INFO and smaller.
logs=[]
import logging
def my_log(logger_self, level, *args, **kwargs):
if level>logging.INFO:
logs.append((args, kwargs))
with mock.patch('logging.Logger._log', my_log):
my_method_which_does_log()

python logging: Custom Python LogRecord Throwing an error

I am trying to use the python logging module to create a custom log file that records other information like host name and adds it to my DB. Below are the classes I created to do this, and the Handler part was working just fine, but now that I added a custom LogRecord class, it throws this error:
/src/lib/__init__.py", line 31, in __init__
logging.LogRecord.__init__(self, *args, **kwargs)
exceptions.TypeError: __init__() takes at most 9 arguments (10 given)
And here is how I execute it
logging.setLoggerClass(MyLogger)
log = logging.getLogger('testing')
log.addHandler(MyLogHandler())
d = {'host': '192.168.0.1'}
log.warn('Hi', d)
And here are the classes. It obviously has to do with the *args, **kwargs, but when I look at it, the *args is empty, and **kwargs only contains the d variable specified above. I don't understand the problem.
class MyLogRecord(logging.LogRecord):
def __init__(self, *args, **kwargs):
logging.LogRecord.__init__(self, *args, **kwargs) //THIS IS THE LINE IT DIES ON
self.host = 'localhost'
class MyLogFormatter(logging.Formatter):
def __init__(self, fmt, datefmt=None, host=None):
logging.Formatter.__init__(self, fmt, datefmt)
self.host = host
def format(self, record):
return logging.Formatter.format(record)
class MyLogger(logging.getLoggerClass()):
def makeRecord(self, *args, **kwargs):
return MyLogRecord(*args, **kwargs)
class MyLogHandler(logging.Handler): # Inherit from logging.Handler
def __init__(self):
# run the regular Handler __init__
logging.Handler.__init__(self)
# Our custom argument
self.mongo = MongoLogger()
def setupCustomLogger(self, name, this_host):
formatter = MyLogFormatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s - %(host)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
return logger
def emit(self, record):
# record.message is the log message
self.mongo.log(record)
class MongoLogger(object):
'''Logs messages to a MongoDB fh_admin log collection.'''
def log(self, message):
##todo write log to DB
print message

The error is telling you exactly what's wrong; you are calling the constructor with too many arguments. To see what I mean, take a look at how log-records are ordinarily constructed in the default implementation of makeRecord:
def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None):
"""
A factory method which can be overridden in subclasses to create
specialized LogRecords.
"""
rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func)
if extra is not None:
for key in extra:
if (key in ["message", "asctime"]) or (key in rv.__dict__):
raise KeyError("Attempt to overwrite %r in LogRecord" % key)
rv.__dict__[key] = extra[key]
return rv
Notice how makeRecord takes an extra param that it doesn't pass directly to LogRecord? You, on the other hand, are passing that directly to LogRecord.__init__, which is causing the error.
From here, you've got two options; you could provide a more complete implementation of makeRecord, or you could try using the LoggerAdapter class which should help you achieve the same goal with less code.
Here's an example:
# Common log info to be added to all logs reported with `log_adapter`
context = {'host': 'localhost'}
log = logging.getLogger('testing')
log.addHandler(logging.StreamHandler())
d = {'host': '192.168.0.1'}
log_adapter = logging.LoggerAdapter(log, context)
log_adapter.warning('Hi', d)
If you need to calculate the value of 'host' (for example) each time something is logged, you could make context an instance of a class that looks like a dictionary. Like so:
class LogContext(object):
def __getitem__(self, key):
if key == 'host':
return 'localhost'
raise KeyError(key)
def __iter__(self):
return iter(['host'])
log_adapter = logging.LoggerAdapter(log, LogContext())
log_adapter.warning('Hi', d)
One thing to note about LoggingAdapter, it apparently doesn't define all of the handy shortcut functions as the ordinary Logger class. That's why I've called the warning method instead of warn as you did above.
More info on LoggingAdapter and adding context to your logs can be found in the python docs.
NOTE - I didn't include MyLogHandler, MyLogFormatter, or MongoLogger in my examples as they were not relevant to the issue/error.

How many times was logging.error() called?

Maybe it's just doesn't exist, as I cannot find it. But using python's logging package, is there a way to query a Logger to find out how many times a particular function was called? For example, how many errors/warnings were reported?

The logging module doesn't appear to support this. In the long run you'd probably be better off creating a new module, and adding this feature via sub-classing the items in the existing logging module to add the features you need, but you could also achieve this behavior pretty easily with a decorator:
class CallCounted:
"""Decorator to determine number of calls for a method"""
def __init__(self,method):
self.method=method
self.counter=0
def __call__(self,*args,**kwargs):
self.counter+=1
return self.method(*args,**kwargs)
import logging
logging.error = CallCounted(logging.error)
logging.error('one')
logging.error('two')
print(logging.error.counter)
Output:
ERROR:root:one
ERROR:root:two
2

You can also add a new Handler to the logger which counts all calls:
class MsgCounterHandler(logging.Handler):
level2count = None
def __init__(self, *args, **kwargs):
super(MsgCounterHandler, self).__init__(*args, **kwargs)
self.level2count = {}
def emit(self, record):
l = record.levelname
if (l not in self.level2count):
self.level2count[l] = 0
self.level2count[l] += 1
You can then use the dict afterwards to output the number of calls.

There'a a warnings module that -- to an extent -- does some of that.
You might want to add this counting feature to a customized Handler. The problem is that there are a million handlers and you might want to add it to more than one kind.
You might want to add it to a Filter, since that's independent of the Handlers in use.

Based on Rolf's answer and how to write a dictionary to a file, here another solution which stores the counts in a json file. In case the json file exists and continue_counts=True, it restores the counts on initialisation.
import json
import logging
import logging.handlers
import os
class MsgCounterHandler(logging.Handler):
"""
A handler class which counts the logging records by level and periodically writes the counts to a json file.
"""
level2count_dict = None
def __init__(self, filename, continue_counts=True, *args, **kwargs):
"""
Initialize the handler.
PARAMETER
---------
continue_counts: bool, optional
defines if the counts should be loaded and restored if the json file exists already.
"""
super(MsgCounterHandler, self).__init__(*args, **kwargs)
filename = os.fspath(filename)
self.baseFilename = os.path.abspath(filename)
self.continue_counts = continue_counts
# if another instance of this class is created, get the actual counts
if self.level2count_dict is None:
self.level2count_dict = self.load_counts_from_file()
def emit(self, record):
"""
Counts a record.
In case, create add the level to the dict.
If the time has come, update the json file.
"""
level = record.levelname
if level not in self.level2count_dict:
self.level2count_dict[level] = 0
self.level2count_dict[level] += 1
self.flush()
def flush(self):
"""
Flushes the dictionary.
"""
self.acquire()
try:
json.dump(self.level2count_dict, open(self.baseFilename, 'w'))
finally:
self.release()
def load_counts_from_file(self):
"""
Load the dictionary from a json file or create an empty dictionary
"""
if os.path.exists(self.baseFilename) and self.continue_counts:
try:
level2count_dict = json.load(open(self.baseFilename))
except Exception as a:
logging.warning(f'Failed to load counts with: {a}')
level2count_dict = {}
else:
level2count_dict = {}
return level2count_dict