I am trying to log the actions in a function and I have written the following function to log responses to different files based on the type of response i.e. info,error,debug and warning.
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter()
fileHandler = logging.FileHandler(log_file)
fileHandler.setFormatter(formatter)
handler = TimedRotatingFileHandler(logger_name,
when="d",
interval=1,
backupCount=100)
l.setLevel(level)
l.addHandler(fileHandler)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
class Info(APIHandler):
#gen.coroutine
def post(self):
req = json.loads(self.request.body)
resp, content = client(item_id=req['item_id'])
debug.debug(content)
info.info(hello world)
warning.warn('warning message')
error.error('error message')
The problem that I am facing is that a response is printed twice each time I call a function.
for example:
info.log
hello world
hello world
Can anyone tell me why is it happening like. This is the case with all the log files.
Thanks
try:
import logging
import logging.handlers
logging.basicConfig(filename='indivi_service.log',
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' )
def setup_logger(logger_name, log_file, level=logging.DEBUG):
l = logging.getLogger(logger_name)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler = logging.handlers.TimedRotatingFileHandler(str(log_file)+'.log', when="d", interval=1, backupCount=100)
handler.setFormatter(formatter)
l.setLevel(level)
l.addHandler(handler)
setup_logger('debug', r'debug')
setup_logger('error', r'error')
setup_logger('warning', r'warning')
setup_logger('info', r'info')
debug = logging.getLogger('debug')
error = logging.getLogger('error')
warning = logging.getLogger('warning')
info = logging.getLogger('info')
if __name__ == "__main__":
info.info('hello world')
error.info('hello world')
after run this script, file info.log has one 'hello world' and error.log also has only one 'hello world'.
Related
I have one "basic" logging configuration that logs to stdout in my project.
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
SERVER_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(filename)s:%(lineno)d] %(message)s"
DATETIME_FORMAT = "%d/%b/%Y %H:%M:%S"
logging.basicConfig(
level=logging.INFO,
format=SERVER_FORMAT,
datefmt=DATETIME_FORMAT,
handlers=[stream_handler])
I have also another logger user_logger that should not print anything. Instead, it should store log entries in a variable.
user_logger = logging.getLogger('user_logger')
user_logger.setLevel(logging.INFO)
log_capture_string = io.StringIO()
log_capture_string_handler = logging.StreamHandler(log_capture_string)
log_capture_string_handler.setLevel(logging.INFO)
log_capture_string_handler.setFormatter(logging.Formatter(USER_FORMAT))
user_logger.handlers = [log_capture_string_handler]
The problem is that when I call:
user_logger.info('This should only be in "log_capture_string"')
it prints it to the console.
Do you know how to avoid that?
You should set user_logger.propagate = False. logging.propagate docs
If this evaluates to false, logging messages are not passed to the handlers of ancestor loggers.
So, your root logger will not send any data to stderr.
This example outputs nothing
import io
import logging
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
SERVER_FORMAT = "[%(asctime)s] %(levelname)s [%(name)s:%(filename)s:%(lineno)d] %(message)s"
DATETIME_FORMAT = "%d/%b/%Y %H:%M:%S"
logging.basicConfig(
level=logging.INFO,
format=SERVER_FORMAT,
datefmt=DATETIME_FORMAT,
handlers=[stream_handler])
user_logger = logging.getLogger('user_logger')
user_logger.propagate = False
user_logger.setLevel(logging.INFO)
log_capture_string = io.StringIO()
log_capture_string_handler = logging.StreamHandler(log_capture_string)
log_capture_string_handler.setLevel(logging.INFO)
log_capture_string_handler.setFormatter(logging.Formatter(SERVER_FORMAT))
user_logger.handlers = [log_capture_string_handler]
user_logger.info('This should only be in "log_capture_string"')
In python logging module Log is formatted using below :
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
**simple_example.py**
# 'application' code
logger.debug('debug message')
logger.info('info message')
logger.warning('warn message')
Which gives output as below :
OUTPUT:
2005-03-19 15:10:26,618 - simple_example - DEBUG - debug message
2005-03-19 15:10:26,620 - simple_example - INFO - info message
2005-03-19 15:10:26,695 - simple_example - WARNING - warn message
I am just wondering if there is any way to add multiple messages not at the end but in between i.e something like
My custom message 1 - simple_example - DEBUG - my custom message 2
Is there any way I could format it like:
formatter = logging.Formatter('%(message1)s - %(name)s - %(levelname)s - %(message2)s')
Any help would be appreciated
You could write your own Formatter class and pass your extra message as kwargs:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = record.args.get("message2")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
formatter = MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
ch.setLevel(logging.ERROR)
logger.addHandler(ch)
logger.error("debug message", {"message2": "Blub"})
Output:
2019-02-08 14:33:50,487 - Blub - test - ERROR - debug message
Edit: I do not know, why this does not work out-of-the-box with INFO level, but you could do the following, which will work:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = record.args.get("message2")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
ch.setFormatter(MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s'))
logging.basicConfig( level=logging.INFO, handlers=[ch] )
logger.info("debug message", {"message2": "Blub"})
Output:
2019-02-11 12:53:17,014 - Blub - test - INFO - debug message
Edit 2: For this to work w/o providing a dict with message2, you can change the code as follows:
import logging
class MyFormatter(logging.Formatter):
def format(self, record):
record.message2 = ""
if(record.args):
record.message2 = record.args.get("message2", "Fallback Value")
return super().format(record)
logger = logging.getLogger('test')
ch = logging.StreamHandler()
ch.setFormatter(MyFormatter('%(asctime)s - %(message2)s - %(name)s - %(levelname)s - %(message)s'))
logging.basicConfig( level=logging.INFO, handlers=[ch] )
logger.info("debug message", {"message2": "Blub"})
logger.info("This is my sample log")
logger.info("This is my sample log", {"hello": "World"})
Output:
2019-02-11 13:20:53,419 - Blub - test - INFO - debug message
2019-02-11 13:20:53,419 - - test - INFO - This is my sample log
2019-02-11 13:20:53,419 - Fallback Value - test - INFO - This is my sample log
You can create your logger class and decorate your methods with this:
#staticmethod
def combine(func):
def wrapped(self, *args, **kwargs):
for i,v in enumerate(args):
if v:
func(self,f"\n {chr(i+65)} : {v}",**kwargs)
return wrapped
example:
import logging
class my_log():
def __init__(self, name, debug=True):
self.logger = logging.getLogger(name)
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s;%(levelname)s;%(message)s",
"%Y-%m-%d %H:%M:%S")
path = 'log.txt'
try:
fh = logging.FileHandler(path, 'a')
except FileNotFoundError:
fh = logging.FileHandler(path, 'w')
fh.setFormatter(formatter)
fh.setLevel(logging.DEBUG)
self.logger.addHandler(fh)
if debug:
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
self.logger.addHandler(ch)
self.logger.removeHandler(fh)
#staticmethod
def combine(func):
def wrapped(self, *args, **kwargs):
for i, v in enumerate(args):
if v:
func(self, f"\n {chr(i+65)} : {v}", **kwargs)
return wrapped
def get_logger(self) -> logging.Logger:
return self.logger
#combine
def debug(self, string: str):
self.logger.debug(string)
l = my_log("test")
l.debug(1, 2, 3, 4)
output:
2022-01-24 21:23:59;DEBUG;
A : 1
2022-01-24 21:23:59;DEBUG;
B : 2
2022-01-24 21:23:59;DEBUG;
C : 3
2022-01-24 21:23:59;DEBUG;
D : 4
Hi you can use your custom message directly in Formatter and use %(message)s to place your logging message
See below example
formatter = logging.Formatter('My custome message 1 - %(name)s - %(levelname)s - %(message)s my custom message 2')
Not able to get this one to work. Probably lack of understanding of python logging module.
Use case - print one variable on all log messages. i.e. "jobID". When multiple instances of this utility will run in parallel in same server - syslog or ../log/messages can be parsed live based on this jobID. Here is attempt with LoggerAdapter method (error lines commented) -
def startlog(self, log_folder, testname=None):
if not os.path.exists(log_folder):
os.makedirs(log_folder)
ltime = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
if testname:
logfile = "%s/%s_log_%s.log" % (log_folder, testname, ltime)
else:
logfile = "%s/log_%s.log" % (log_folder, ltime)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formated = logging.Formatter('%(asctime)s - %(levelname)s - %(module)10s - %(funcName)10s - %(message)s')
#What is desired is -
#formated = logging.Formatter('%(asctime)s - %(jobid)s - %(levelname)s - %(module)10s - %(funcName)10s - %(message)s')
if not testname:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formated)
logger.addHandler(fh)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(formated)
logger.addHandler(ch)
# Next line errors -
# logger = logging.LoggerAdapter(logger, {"jobid": self.jobdata.jobid})
return logger
else:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formated)
self.log.addHandler(fh)
# Next line errors -
# logger = logging.LoggerAdapter(self.log, {"jobid": self.jobdata.jobid})
return fh
2nd Try with Filters:
def startlog(self, log_folder, t_name=None):
if not os.path.exists(log_folder):
os.makedirs(log_folder)
ltime = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
if t_name:
logfile = "%s/%s_log_%s.log" % (log_folder, t_name, ltime)
else:
logfile = "%s/log_%s.log" % (log_folder, ltime)
root = logging.getLogger()
root.setLevel(logging.INFO)
root.addFilter(ContextFilter(self.jobdata.jobid))
formated = logging.Formatter('%(asctime)s - %(jobid)s - %(levelname)s - %(module)10s - %(funcName)10s - %(message)s')
if not t_name:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formated)
root.addHandler(fh)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(formated)
root.addHandler(ch)
return root
else:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formated)
self.log.addHandler(fh)
return fh
class ContextFilter(logging.Filter):
"""
This is a filter which injects contextual information into the log.
"""
def __init__(self, jobid):
self.jobid = jobid
def filter(self, record):
record.jobid = self.jobid
return True
Issue faced with filters is 'keyerror' from other modules(paramiko - transport.py). Similar to How to properly add custom logging filters in Python modules
Traceback (most recent call last):
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 859, in emit
msg = self.format(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 732, in format
return fmt.format(record)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/logging/__init__.py", line 474, in format
s = self._fmt % record.__dict__
KeyError: 'jobid'
Logged from file transport.py, line 1567
Dumb me, i need to add filter to the log handler. This is working so far -
def startlog(self, log_folder, t_name=None):
if not os.path.exists(log_folder):
os.makedirs(log_folder)
ltime = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
if t_name:
logfile = "%s/%s_log_%s.log" % (log_folder, t_name, ltime)
else:
logfile = "%s/log_%s.log" % (log_folder, ltime)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formated = logging.Formatter('%(asctime)s - %(jobid)s - %(levelname)s - %(module)10s - %(funcName)10s - %(message)s')
formatted_file = logging.Formatter('%(asctime)s - %(levelname)s - %(module)10s - %(funcName)10s - %(message)s')
if not t_name:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formatted_file)
fh.addFilter(ContextFilter(self.jobdata.jobid))
logger.addHandler(fh)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(formated)
ch.addFilter(ContextFilter(self.jobdata.jobid))
logger.addHandler(ch)
return logger
else:
fh = logging.FileHandler(filename=logfile)
fh.setFormatter(formatted_file)
fh.addFilter(ContextFilter(self.jobdata.jobid))
self.log.addHandler(fh)
return fh
Program A by Python:
LOG_PATH = fdoc_log + "/store_plus.log"
FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(filename=LOG_PATH, filemode = 'w', level=logging.DEBUG, format=FORMAT)
Program B by bash:
mv store_plus.log store_plus.log.bk
The Program A will run in the background and don't stop. When
the Program B delete the file of store_plus.log, the Program A can't write log as well.
If I want the Program A rebuild the store_plus.log, How to solve it ?
Thank you
PS: the way :
f = open(LOG_PATH, "a")
f.close()
It can't work.
An example taken from pymotw-logging and all credit to Doug Hellmann.
import glob
import logging
import logging.handlers
LOG_FILENAME = '/tmp/logging_rotatingfile_example.out'
# Set up a specific logger with our desired output level
my_logger = logging.getLogger('MyLogger')
my_logger.setLevel(logging.DEBUG)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler(LOG_FILENAME, maxBytes=20, backupCount=5)
my_logger.addHandler(handler)
# Log some messages
for i in range(20):
my_logger.debug('i = %d' % i)
# See what files are created
logfiles = glob.glob('%s*' % LOG_FILENAME)
for filename in logfiles:
print filename
This way is OK by WatchedFileHandler :
logger = logging.getLogger('simple_example')
logger.setLevel(logging.DEBUG)
ch = logging.handlers.WatchedFileHandler('a_log')
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
I need to write a function that logs into a file (using logging module) and also prints the same content on the console at the same time.
What I have is :
def printScreenAndLog(msg):
log = logging.getLogger()
log.info(msg)
now = str(datetime.datetime.now())
print now,"%s" % msg
def main():
options, args = usage()
log = logging.getLogger("CMDR")
log.setLevel(logging.DEBUG)
fh = logging.FileHandler('cmdr.log')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
log.addHandler(fh)
printScreenAndLog("Testing")
if __name__ == "__main__":
main()
This function should do what you require:
def configure_logging():
log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# log to stdout
logging.basicConfig(level=logging.DEBUG, format=log_format)
# also log to file
formatter = logging.Formatter(log_format)
handler = logging.FileHandler("cmdr.log")
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logging.getLogger('').addHandler(handler)
did you try to set the logging level to info instead of debug?
or use log.debug(msg) in your printScreenAndLog function?