Develop Reference

Python logging: access custom logger from several files

How can I declare a custom logger with custom logging method in the main.py and then use the same object (!) in all the files that main.py uses:
eg
#main.py
import logging
from additional import bar
class CustomLogger(logging.Logger):
def __init__(self)
self.foo = None
def log(self, message):
apply_external_method(self.foo, message)
print(message)
logger = logging.getLogger('the_only_logger') # Should be CustomLogger
logger.foo = "foo"
And
# additional.py
logger = logging.getLogger('the_only_logger') # Should be CustomLogger and foo should be set

I have two solutions for you,
first, if you want to use a specific object you can add it to every constructor to get a logger object.
the second thing is you can make the logger write to a file and every file will create a new object but will write to the same file.
hope it's help(:

Thanks to #Klaus D, I figured out I need to set CustomHandler, and I need to do only once, after that logger can be shared across files with logger = logging.getLogger('same-logger-name')
# main.py
import logging
from additional import foo
class CustomHandler(logging.Handler):
def __init__(self, queue, *args, **kwargs):
self.queue = queue
print("Init", id(queue))
super(CustomHandler, self).__init__(*args, **kwargs)
def emit(self, record):
print("Log")
print(self.format(record))
logger = logging.getLogger('2')
logger.setLevel(logging.DEBUG)
handler = CustomHandler("custom object")
logger.addHandler(handler)
logger.debug('information')
foo()
and
# additional.py
import logging
logger = logging.getLogger('2')
def foo():
# will be printed with "Log" before it
logger.debug('information from 2nd file')

python logging thread safety

I'm trying to understand how exactly python logging is thread safe in the folowing cases:
1. I create different handlers for the same file in the main thread. and then ask a thread to log to it.
2. I create multiple handlers pointing to the same file, from within different threads.
I see in the source code of FileHandler that every Handler created has its own lock, but how does this work in the cases I have mentioned?
if I understand it correctly, the lock's scopes is the FileHandler object, meaning that if I create 2 different handlers they don't share the lock, and they might run into a race condition.
So my question is: how is this threadsafe?
here is the relevant code. please note that the FileHandler Class inherits StreamHandler which in turn inherits Handler.
class StreamHandler(Handler):
"""
A handler class which writes logging records, appropriately formatted,
to a stream. Note that this class does not close the stream, as
sys.stdout or sys.stderr may be used.
"""
terminator = '\n'
def __init__(self, stream=None):
"""
Initialize the handler.
If stream is not specified, sys.stderr is used.
"""
Handler.__init__(self)
if stream is None:
stream = sys.stderr
self.stream = stream
def flush(self):
"""
Flushes the stream.
"""
self.acquire()
try:
if self.stream and hasattr(self.stream, "flush"):
self.stream.flush()
finally:
self.release()
def emit(self, record):
"""
Emit a record.
If a formatter is specified, it is used to format the record.
The record is then written to the stream with a trailing newline. If
exception information is present, it is formatted using
traceback.print_exception and appended to the stream. If the stream
has an 'encoding' attribute, it is used to determine how to do the
output to the stream.
"""
try:
msg = self.format(record)
stream = self.stream
stream.write(msg)
stream.write(self.terminator)
self.flush()
except Exception:
self.handleError(record)
def __repr__(self):
level = getLevelName(self.level)
name = getattr(self.stream, 'name', '')
if name:
name += ' '
return '<%s %s(%s)>' % (self.__class__.__name__, name, level)
class FileHandler(StreamHandler):
"""
A handler class which writes formatted logging records to disk files.
"""
def __init__(self, filename, mode='a', encoding=None, delay=False):
"""
Open the specified file and use it as the stream for logging.
"""
# Issue #27493: add support for Path objects to be passed in
filename = os.fspath(filename)
#keep the absolute path, otherwise derived classes which use this
#may come a cropper when the current directory changes
self.baseFilename = os.path.abspath(filename)
self.mode = mode
self.encoding = encoding
self.delay = delay
if delay:
#We don't open the stream, but we still need to call the
#Handler constructor to set level, formatter, lock etc.
Handler.__init__(self)
self.stream = None
else:
StreamHandler.__init__(self, self._open())
def close(self):
"""
Closes the stream.
"""
self.acquire()
try:
try:
if self.stream:
try:
self.flush()
finally:
stream = self.stream
self.stream = None
if hasattr(stream, "close"):
stream.close()
finally:
# Issue #19523: call unconditionally to
# prevent a handler leak when delay is set
StreamHandler.close(self)
finally:
self.release()
def _open(self):
"""
Open the current base file with the (original) mode and encoding.
Return the resulting stream.
"""
return open(self.baseFilename, self.mode, encoding=self.encoding)
def emit(self, record):
"""
Emit a record.
If the stream was not opened because 'delay' was specified in the
constructor, open it before calling the superclass's emit.
"""
if self.stream is None:
self.stream = self._open()
StreamHandler.emit(self, record)
def __repr__(self):
level = getLevelName(self.level)
return '<%s %s (%s)>' % (self.__class__.__name__, self.baseFilename, level)

That is a good question that required a bit of reverse engineering.
The simple answer: FileHandler is not thread-safe in this case itself, but one does not create it using constructor. A factory methods are used instead, and they make sure things are thread-safe:
# see here: https://github.com/python/cpython/blob/586be6f3ff68ab4034e555f1434a4427e129ad0b/Lib/logging/__init__.py#L1985
if handlers is None:
filename = kwargs.pop("filename", None)
mode = kwargs.pop("filemode", 'a')
if filename:
if 'b'in mode:
errors = None
h = FileHandler(filename, mode,
encoding=encoding, errors=errors)
else:
stream = kwargs.pop("stream", None)
h = StreamHandler(stream)
handlers = [h]
and:
# https://github.com/python/cpython/blob/586be6f3ff68ab4034e555f1434a4427e129ad0b/Lib/logging/__init__.py#L1272
def getLogger(self, name):
"""
Get a logger with the specified name (channel name), creating it
if it doesn't yet exist. This name is a dot-separated hierarchical
name, such as "a", "a.b", "a.b.c" or similar.
If a PlaceHolder existed for the specified name [i.e. the logger
didn't exist but a child of it did], replace it with the created
logger and fix up the parent/child references which pointed to the
placeholder to now point to the logger.
"""
rv = None
if not isinstance(name, str):
raise TypeError('A logger name must be a string')
_acquireLock()
try:
if name in self.loggerDict:
rv = self.loggerDict[name]
if isinstance(rv, PlaceHolder):
ph = rv
rv = (self.loggerClass or _loggerClass)(name)
rv.manager = self
self.loggerDict[name] = rv
self._fixupChildren(ph, rv)
self._fixupParents(rv)
else:
rv = (self.loggerClass or _loggerClass)(name)
rv.manager = self
self.loggerDict[name] = rv
self._fixupParents(rv)
finally:
_releaseLock()
return rv
Please pay attention to two things happening here:
1) lock is acquired before new Handler is created
2) If logger for a name is already created - it is returned. So, for the same file, one should get only one FileHandler instance.

I have already implemented logging which is thread safe. The goal was to collect tweets with different tags in parallel using threading.
Here is my implementation of logging:
import sys
import logging
import threading
class Singleton:
__lock = threading.Lock()
__instance = None
def __init__(self):
if self.__class__.__instance:
raise Exception('Tried to allocate a second instance of a singleton.\nUse getInstance() instead.')
sys.exit(-1)
#classmethod
def get_instance(cls):
if cls.__instance is None:
with cls.__lock:
if cls.__instance is None:
cls.__instance = cls()
return cls.__instance
class Logger(Singleton):
__FORMAT = '%(asctime)s - %(levelname)s - [%(threadName)s] - %(message)s'
def __init__(self, name: str = __name__):
super().__init__()
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
log_formatter = logging.Formatter(self.__class__.__FORMAT)
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(log_formatter)
console_handler.setLevel(logging.DEBUG)
file_handler = logging.FileHandler('log.log')
file_handler.setFormatter(log_formatter)
file_handler.setLevel(logging.INFO)
if logger.handlers:
logger.handlers = []
logger.addHandler(file_handler)
logger.addHandler(console_handler)
self.__logger = logger
self.__class__.__instance = self
#property
def logger(self):
return self.__logger
def debug(self, message: str):
self.__logger.debug(message)
def info(self, message: str):
self.__logger.info(message)
def warn(self, message: str):
self.__logger.warning(message)
def error(self, message: str):
self.__logger.error(message)
def critical(self, message: str):
self.__logger.critical(message)
def exception(self, message: str):
self.__logger.exception(message)
Then in your threads, you just have to call get_instance method of Logger class.
Logger.get_instance().info(f'Your log here')
I hope this will help you!

So in other words, in both cases you get handlers which can end up opening the file at the same time, and writing to it at the same time.
This means it is not "safe" in the sense that messages can be "mixed up" in the file (e.g. starting to write msg1, then starting to write msg2, then continue to write msg1).
Other than that, I don't see any other harm. Separate FileHandler instances don't interfere with each other.
So to summarize, the content of the log file might not be perfect, but nothing else really breaks.

Arguments passed to create a LogRecord in python

The LogRecord in python's logging module has a LogRecord defined as:
class LogRecord(object):
"""
A LogRecord instance represents an event being logged.
LogRecord instances are created every time something is logged. They
contain all the information pertinent to the event being logged. The
main information passed in is in msg and args, which are combined
using str(msg) % args to create the message field of the record. The
record also includes information such as when the record was created,
the source line where the logging call was made, and any exception
information to be logged.
"""
def __init__(self, name, level, pathname, lineno,
msg, args, exc_info, func=None, sinfo=None, **kwargs):
"""
Initialize a logging record with interesting information.
"""
ct = time.time()
self.name = name
self.msg = msg
# more stuff below
This LogRecord gets created, for example whenever someone does a logging actions, such as:
def my_func():
logging.info('Hello %s', 'Person')
return 1
In the above logger call, the variables can be inferred as:
name = 'root' # using the root logger since calling `logging.`
level = 'INFO' # or, 20
msg = 'Hello %s'
args = ('Person',)
How does the introspection work to gather the other items from my logging call? For example:
pathname
lineno
exc_info
func
sinfo # what is this?
kwargs # is this just user-added kwargs, or something else?
For example, in your answer could you show an example of doing the introspection on a function call to gather the above information?

Nearly all of it happens in Logger.findCaller. It inspects the current frame for this information.
The kwargs argument of the default LogRecord constructor is not actually used. User supplied data is passed in through extra.

python logging: print accurrate info from custom log level

I created a custom log level as below:
class MyLoggerAdapter(logging.LoggerAdapter):
def __init__(self, logger, extra):
logging.LoggerAdapter.__init__(self, logger, extra)
self.logger = logger
logging.addLevelName(25, "NOTICE")
def notice(self, msg, *args, **kwargs):
self.logger.log(25, msg, *args, **kwargs)
The issue I have is that my logger format uses %(pathname)s and %(lineno)d, and whenever I call logger.notice() those values refer to the line within the definition of the notice method, rather than the location that the method was called from. How do I add the notice log level but print the correct location of the message to the logs?

python logging: Custom Python LogRecord Throwing an error

I am trying to use the python logging module to create a custom log file that records other information like host name and adds it to my DB. Below are the classes I created to do this, and the Handler part was working just fine, but now that I added a custom LogRecord class, it throws this error:
/src/lib/__init__.py", line 31, in __init__
logging.LogRecord.__init__(self, *args, **kwargs)
exceptions.TypeError: __init__() takes at most 9 arguments (10 given)
And here is how I execute it
logging.setLoggerClass(MyLogger)
log = logging.getLogger('testing')
log.addHandler(MyLogHandler())
d = {'host': '192.168.0.1'}
log.warn('Hi', d)
And here are the classes. It obviously has to do with the *args, **kwargs, but when I look at it, the *args is empty, and **kwargs only contains the d variable specified above. I don't understand the problem.
class MyLogRecord(logging.LogRecord):
def __init__(self, *args, **kwargs):
logging.LogRecord.__init__(self, *args, **kwargs) //THIS IS THE LINE IT DIES ON
self.host = 'localhost'
class MyLogFormatter(logging.Formatter):
def __init__(self, fmt, datefmt=None, host=None):
logging.Formatter.__init__(self, fmt, datefmt)
self.host = host
def format(self, record):
return logging.Formatter.format(record)
class MyLogger(logging.getLoggerClass()):
def makeRecord(self, *args, **kwargs):
return MyLogRecord(*args, **kwargs)
class MyLogHandler(logging.Handler): # Inherit from logging.Handler
def __init__(self):
# run the regular Handler __init__
logging.Handler.__init__(self)
# Our custom argument
self.mongo = MongoLogger()
def setupCustomLogger(self, name, this_host):
formatter = MyLogFormatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s - %(host)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
return logger
def emit(self, record):
# record.message is the log message
self.mongo.log(record)
class MongoLogger(object):
'''Logs messages to a MongoDB fh_admin log collection.'''
def log(self, message):
##todo write log to DB
print message

The error is telling you exactly what's wrong; you are calling the constructor with too many arguments. To see what I mean, take a look at how log-records are ordinarily constructed in the default implementation of makeRecord:
def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None):
"""
A factory method which can be overridden in subclasses to create
specialized LogRecords.
"""
rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func)
if extra is not None:
for key in extra:
if (key in ["message", "asctime"]) or (key in rv.__dict__):
raise KeyError("Attempt to overwrite %r in LogRecord" % key)
rv.__dict__[key] = extra[key]
return rv
Notice how makeRecord takes an extra param that it doesn't pass directly to LogRecord? You, on the other hand, are passing that directly to LogRecord.__init__, which is causing the error.
From here, you've got two options; you could provide a more complete implementation of makeRecord, or you could try using the LoggerAdapter class which should help you achieve the same goal with less code.
Here's an example:
# Common log info to be added to all logs reported with `log_adapter`
context = {'host': 'localhost'}
log = logging.getLogger('testing')
log.addHandler(logging.StreamHandler())
d = {'host': '192.168.0.1'}
log_adapter = logging.LoggerAdapter(log, context)
log_adapter.warning('Hi', d)
If you need to calculate the value of 'host' (for example) each time something is logged, you could make context an instance of a class that looks like a dictionary. Like so:
class LogContext(object):
def __getitem__(self, key):
if key == 'host':
return 'localhost'
raise KeyError(key)
def __iter__(self):
return iter(['host'])
log_adapter = logging.LoggerAdapter(log, LogContext())
log_adapter.warning('Hi', d)
One thing to note about LoggingAdapter, it apparently doesn't define all of the handy shortcut functions as the ordinary Logger class. That's why I've called the warning method instead of warn as you did above.
More info on LoggingAdapter and adding context to your logs can be found in the python docs.
NOTE - I didn't include MyLogHandler, MyLogFormatter, or MongoLogger in my examples as they were not relevant to the issue/error.