Develop Reference

Handling custom logger across different modules

Creating a custom logger for my purpose using python which can be used across different modules just by importing and calling a custom_log method.
This is MyLogger.py script.
import datetime
import logging
import logging.handlers
import os
import colorlog
from pathlib import Path
class MyLogger(logging.Logger):
def __init__(self, verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
if (verbose == 1):
max_bytes = 1024 ** 2
backup_count = 100
t = datetime.datetime.now()
file_name = file_name_format.format(year=t.year, month=t.month, day=t.day, hour=t.hour, minute=t.minute,
second=t.second)
file_name = os.path.join(log_dir_path, file_name)
file_handler = logging.handlers.RotatingFileHandler(filename=file_name, maxBytes=max_bytes, backupCount=backup_count)
file_handler.setLevel(logging.DEBUG)
file_formatter = logging.Formatter(file_msg_format)
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
if (verbose == 1):
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
def custom_log(self, level, msg):
logging.log(getattr(logging,level),msg)
I have 2 other scripts within the same directory as below, just need to initialize MyLogger() in Test1.py at beginning of the test and expecting to use custom_log in all other scripts. Missing something here to make this work,either in the way of initializing or the way of importing. Any support how to get this done.
Test1.py
from MyLogger import MyLogger
class StartTest():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
if __name__ == '__main__':
MyLogger()
StartTest()
Test2.py
from MyLogger import MyLogger
class Test():
def __init__(self):
MyLogger.custom_log('DEBUG','Debug messages are only sent to the logfile.')
def TestMethod(self):
MyLogger.custom_log('INFO','Debug messages are only sent to the logfile.')

You can achieve it by creating a logger with a specific name instead of using root logger. logging.log() uses always the root logger. So, you can define a logger with a specific name instead of creating a new Logging channel.
An example which is inline with your need
class MyLogger:
logger: logging.Logger = None
#staticmethod
def configure(verbose=1):
log_dir_path = Path("../logs")
file_name_format = '{year:04d}{month:02d}{day:02d}-{hour:02d}{minute:02d}{second:02d}.log'
file_msg_format = '%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s'
console_msg_format = '%(asctime)s %(levelname)-8s: %(message)s'
logger = logging.getLogger("mylogger")
logger.setLevel(logging.DEBUG)
cformat = '%(log_color)s' + console_msg_format
colors = {'DEBUG': 'green', 'INFO': 'cyan', 'WARNING': 'bold_yellow', 'ERROR': 'bold_red',
'CRITICAL': 'bold_purple'}
date_format = '%Y-%m-%d %H:%M:%S'
formatter = colorlog.ColoredFormatter(cformat, date_format, log_colors=colors)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
MyLogger.logger = logger
if __name__ == '__main__':
MyLogger.configure()
MyLogger.logger.error("debug message")
Once the MyLogger.configure is called, then you can use the MyLogger.logger.* any where in your app / scripts.
This can be done without a helper class as well. Create a logger configuration file. Configure your custom logger with a different name rather than root, and from your code always call logging.getLogger("name-of-logger") to create a logging instance.

Logging with multiple parameters in python

In python logging module Log is formatted using below :
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
**simple_example.py**
# 'application' code
logger.debug('debug message')
logger.info('info message')
logger.warning('warn message')
Which gives output as below :
OUTPUT:
2005-03-19 15:10:26,618 - simple_example - DEBUG - debug message
2005-03-19 15:10:26,620 - simple_example - INFO - info message
2005-03-19 15:10:26,695 - simple_example - WARNING - warn message
I am just wondering if there is any way to add multiple messages not at the end but in between i.e something like
My custom message 1 - simple_example - DEBUG - my custom message 2
Is there any way I could format it like:
formatter = logging.Formatter('%(message1)s - %(name)s - %(levelname)s - %(message2)s')
Any help would be appreciated

You could write your own Formatter class and pass your extra message as kwargs:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = record.args.get("message2")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
formatter = MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
ch.setLevel(logging.ERROR)
logger.addHandler(ch)
logger.error("debug message", {"message2": "Blub"})
Output:
2019-02-08 14:33:50,487 - Blub - test - ERROR - debug message
Edit: I do not know, why this does not work out-of-the-box with INFO level, but you could do the following, which will work:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = record.args.get("message2")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
ch.setFormatter(MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s'))
logging.basicConfig( level=logging.INFO, handlers=[ch] )
logger.info("debug message", {"message2": "Blub"})
Output:
2019-02-11 12:53:17,014 - Blub - test - INFO - debug message
Edit 2: For this to work w/o providing a dict with message2, you can change the code as follows:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = ""
if(record.args):
record.message2 = record.args.get("message2", "Fallback Value")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
ch.setFormatter(MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s'))
logging.basicConfig( level=logging.INFO, handlers=[ch] )
logger.info("debug message", {"message2": "Blub"})
logger.info("This is my sample log")
logger.info("This is my sample log", {"hello": "World"})
Output:
2019-02-11 13:20:53,419 - Blub - test - INFO - debug message
2019-02-11 13:20:53,419 - - test - INFO - This is my sample log
2019-02-11 13:20:53,419 - Fallback Value - test - INFO - This is my sample log

You can create your logger class and decorate your methods with this:
#staticmethod
def combine(func):
def wrapped(self, *args, **kwargs):
for i,v in enumerate(args):
if v:
func(self,f"\n {chr(i+65)} : {v}",**kwargs)
return wrapped
example:
import logging
class my_log():
def __init__(self, name, debug=True):
self.logger = logging.getLogger(name)
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s",
"%Y-%m-%d %H:%M:%S")
path = 'log.txt'
try:
fh = logging.FileHandler(path, 'a')
except FileNotFoundError:
fh = logging.FileHandler(path, 'w')
fh.setFormatter(formatter)
fh.setLevel(logging.DEBUG)
self.logger.addHandler(fh)
if debug:
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
self.logger.addHandler(ch)
self.logger.removeHandler(fh)
#staticmethod
def combine(func):
def wrapped(self, *args, **kwargs):
for i, v in enumerate(args):
if v:
func(self, f"\n {chr(i+65)} : {v}", **kwargs)
return wrapped
def get_logger(self) -> logging.Logger:
return self.logger
#combine
def debug(self, string: str):
self.logger.debug(string)
l = my_log("test")
l.debug(1, 2, 3, 4)
output:
2022-01-24 21:23:59;DEBUG;
A : 1
2022-01-24 21:23:59;DEBUG;
B : 2
2022-01-24 21:23:59;DEBUG;
C : 3
2022-01-24 21:23:59;DEBUG;
D : 4

Hi you can use your custom message directly in Formatter and use %(message)s to place your logging message
See below example
formatter = logging.Formatter('My custome message 1 - %(name)s - %(levelname)s - %(message)s my custom message 2')

Refining the logging method in python tornado

I am trying to log the actions in a function and I have written the following function to log responses to different files based on the type of response i.e. info,error,debug and warning.
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter()
fileHandler = logging.FileHandler(log_file)
fileHandler.setFormatter(formatter)
handler = TimedRotatingFileHandler(logger_name,
when="d",
interval=1,
backupCount=100)
l.setLevel(level)
l.addHandler(fileHandler)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
class Info(APIHandler):
#gen.coroutine
def post(self):
req = json.loads(self.request.body)
resp, content = client(item_id=req['item_id'])
debug.debug(content)
info.info(hello world)
warning.warn('warning message')
error.error('error message')
The problem that I am facing is that a response is printed twice each time I call a function.
for example:
info.log
hello world
hello world
Can anyone tell me why is it happening like. This is the case with all the log files.
Thanks

try:
import logging
import logging.handlers
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.handlers.TimedRotatingFileHandler(str(log_file)+'.log', when="d", interval=1, backupCount=100)
handler.setFormatter(formatter)
l.setLevel(level)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
if __name__ == "__main__":
info.info('hello world')
error.info('hello world')
after run this script, file info.log has one 'hello world' and error.log also has only one 'hello world'.

How to rebuild the log file in Python?

Program A by Python:
LOG_PATH = fdoc_log + "/store_plus.log"
FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(filename=LOG_PATH, filemode = 'w', level=logging.DEBUG, format=FORMAT)
Program B by bash:
mv store_plus.log store_plus.log.bk
The Program A will run in the background and don't stop. When
the Program B delete the file of store_plus.log, the Program A can't write log as well.
If I want the Program A rebuild the store_plus.log, How to solve it ?
Thank you
PS: the way :
f = open(LOG_PATH, "a")
f.close()
It can't work.

An example taken from pymotw-logging and all credit to Doug Hellmann.
import glob
import logging
import logging.handlers
LOG_FILENAME = '/tmp/logging_rotatingfile_example.out'
# Set up a specific logger with our desired output level
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=20, backupCount=5)
my_logger.addHandler(handler)
# Log some messages
for i in range(20):
my_logger.debug('i = %d' % i)
# See what files are created
logfiles = glob.glob('%s*' % LOG_FILENAME)
for filename in logfiles:
print filename

This way is OK by WatchedFileHandler :
logger = logging.getLogger('simple_example')
logger.setLevel(logging.DEBUG)
ch = logging.handlers.WatchedFileHandler('a_log')
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

Python: logging into a file and also do sys.stdout

I need to write a function that logs into a file (using logging module) and also prints the same content on the console at the same time.
What I have is :
def printScreenAndLog(msg):
log = logging.getLogger()
log.info(msg)
now = str(datetime.datetime.now())
print now,"%s" % msg
def main():
options, args = usage()
log = logging.getLogger("CMDR")
log.setLevel(logging.DEBUG)
fh = logging.FileHandler('cmdr.log')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
log.addHandler(fh)
printScreenAndLog("Testing")
if __name__ == "__main__":
main()

This function should do what you require:
def configure_logging():
log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# log to stdout
logging.basicConfig(level=logging.DEBUG, format=log_format)
# also log to file
formatter = logging.Formatter(log_format)
handler = logging.FileHandler("cmdr.log")
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logging.getLogger('').addHandler(handler)

did you try to set the logging level to info instead of debug?
or use log.debug(msg) in your printScreenAndLog function?