Develop Reference

Rotating file handler for JSON logs in Python

I am working on saving the json logs using python. Below is the code:
log_file = 'app_log.json'
log_json = dict()
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
log_json['Created'] = datetime.datetime.utcnow().isoformat()
with open(log_file, "a") as f:
json.dump(log_json, f)#, ensure_ascii=False)
f.write("\n")
Now above code is generating the log file. But I have noticed that the file size is increasing a lot and in future, I might face disk space issue. I was wondering if there is any pre built rotating file handler available for json in which we can mention fixed size lets say 100mb and upon reaching this size it will delete and recreate the new file.
I have previously used from logging.handlers import RotatingFileHandler to do this in case of .log files but also want to do this for .json files. Please help. Thanks

Python does not care about the log file name.
You can use the rotating handler which you used for .log file for .json file also.
See sample example below
# logging_example.py
import logging
import logging.handlers
import os
import time
logfile = os.path.join("/tmp", "demo_logging.json")
logger = logging.getLogger(__name__)
fh = logging.handlers.RotatingFileHandler(logfile, mode='a', maxBytes=1000, backupCount=5) # noqa:E501
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)
while 1:
time.sleep(1)
logger.info("Long string to increase the file size")
You can also look at logrotate if you are working in Unix environment. It is a great and simple tool with good documentation to do just exactly what you need.

You can implement structured logging with RotatingFileHandler
import json
import logging
import logging.handlers
from datetime import datetime
class StructuredMessage:
def __init__(self, message, /, **kwargs):
self.message = message
self.kwargs = kwargs
def __str__(self):
return '%s >>> %s' % (self.message, json.dumps(self.kwargs))
_ = StructuredMessage # optional, to improve readability
log_json = {}
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
log_json['Created'] = datetime.utcnow().isoformat()
LOG_FILENAME = 'logging_rotatingfile_example.out'
# Set up a specific logger with our desired output level
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=20, backupCount=5)
bf = logging.Formatter('%(message)s')
handler.setFormatter(bf)
logger.addHandler(handler)
logger.info(_('INFO', **log_json))
Note: check here for more info about structured-logging-python
you can use also use json-logging-python with RotatingFileHandler
import logging
import json
import traceback
from datetime import datetime
import copy
import json_logging
import sys
json_logging.ENABLE_JSON_LOGGING = True
def extra(**kw):
'''Add the required nested props layer'''
return {'extra': {'props': kw}}
class CustomJSONLog(logging.Formatter):
"""
Customized logger
"""
def get_exc_fields(self, record):
if record.exc_info:
exc_info = self.format_exception(record.exc_info)
else:
exc_info = record.exc_text
return {'python.exc_info': exc_info}
#classmethod
def format_exception(cls, exc_info):
return ''.join(traceback.format_exception(*exc_info)) if exc_info else ''
def format(self, record):
json_log_object = {"#timestamp": datetime.utcnow().isoformat(),
"level": record.levelname,
"message": record.getMessage(),
"caller": record.filename + '::' + record.funcName
}
json_log_object['data'] = {
"python.logger_name": record.name,
"python.module": record.module,
"python.funcName": record.funcName,
"python.filename": record.filename,
"python.lineno": record.lineno,
"python.thread": record.threadName,
"python.pid": record.process
}
if hasattr(record, 'props'):
json_log_object['data'].update(record.props)
if record.exc_info or record.exc_text:
json_log_object['data'].update(self.get_exc_fields(record))
return json.dumps(json_log_object)
json_logging.init_non_web(custom_formatter=CustomJSONLog, enable_json=True)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
LOG_FILENAME = 'logging_rotating_json_example.out'
handler = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=20, backupCount=5)
logger.addHandler(handler)
log_json = {}
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
logger.info('Starting')
logger.debug('Working', extra={"props":log_json})
Note: check here for more info about json-logging-python

You can try this just before you write / append to the file. This should check to see if the file has reached the max lines size, then it will remove one line of code from the beginning of the file before you append as usual to the end of the file.
filename = 'file.txt'
maxLines = 100
count = len(open(filename).readlines())
if(count > maxLines) {
with open(filename, 'r') as fin:
data = fin.read().splitlines(True)
with open(filename, 'w') as fout:
fout.writelines(data[1:])
}

Python logger - Redirecting STDOUT to logfile as well as any debug messages

I'm trying to use the logging module in Python so that, when I run my program, I end up with a log file, debug.log, containing:
Every log message (logging.DEBUG, logging.WARNING etc.)
Every time my code prints something to STDOUT
When I run the program, I only want the debug messages to appear in the log file, not to be printed on the terminal.
Based on this answer, here's my example code, test.py:
import logging
import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)
fh = logging.FileHandler('debug.log')
fh.setLevel(logging.DEBUG)
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
sh.setFormatter(formatter)
fh.setFormatter(formatter)
root.addHandler(sh)
root.addHandler(fh)
x = 4
y = 5
logging.debug("X: %s", x)
logging.debug("Y: %s", y)
print("x is", x)
print("y is", y)
print("x * y =", x*y)
print("x^y =", x**y)
And here's what I would want to be the contents of debug.log:
2021-02-01 12:10:48,263 - root - DEBUG - X: 4
2021-02-01 12:10:48,264 - root - DEBUG - Y: 5
x is 4
y is 5
x * y = 20
x^y = 1024
Instead, the contents of debug.log is just the first two lines:
2021-02-01 12:10:48,263 - root - DEBUG - X: 4
2021-02-01 12:10:48,264 - root - DEBUG - Y: 5
When I run test.py, I get this output:
2021-02-01 12:17:04,201 - root - DEBUG - X: 4
2021-02-01 12:17:04,201 - root - DEBUG - Y: 5
x is 4
y is 5
x * y = 20
x^y = 1024
So I've actually got the opposite results to what I want: the log file excludes STDOUT prints where I want them included, and the program output includes the debug messages where I want them excluded.
How can I fix this, so that running test.py only outputs the lines in print statements, and the resulting debug.log file contains both the DEBUG logs and the print lines?

Well, I can make it work, but I don't yet know whether there are any repercussions due to it. Perhaps others will be able to point out any potential pitfalls, such as multithreading.
You can set sys.stdout to any file-like object that you like. This would include the file to which your logging.FileHandler() is writing. Try this:
fh = logging.FileHandler('debug.log')
fh.setLevel(logging.DEBUG)
old_stdout = sys.stdout # in case you want to restore later
sys.stdout = fh.stream # the file to which fh writes
And you can remove the code that deals with sh that is hooked up to stdout.

If you really want all output to stdout to end up in the log file, see e.g. mhawke's answer, as well as the linked questions and answers.
However, if you are really just interested in the output from your own print() calls, then I would replace all of those by Logger.log() calls, using a custom logging level. That gives you fine-grained control over what happens.
Below, I define a custom log level with a value higher than logging.CRITICAL, so our console output is always printed, even if the logger's level is CRITICAL. See docs.
Here's a minimal implementation, based on the OP's example:
import sys
import logging
# define a custom log level with a value higher than CRITICAL
CUSTOM_LEVEL = 100
# dedicated formatter that just prints the unformatted message
# (actually this is the default behavior, but we make it explicit here)
# See docs: https://docs.python.org/3/library/logging.html#logging.Formatter
console_formatter = logging.Formatter('%(message)s')
# your basic console stream handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(console_formatter)
# only use this handler for our custom level messages (highest level)
console_handler.setLevel(CUSTOM_LEVEL)
# your basic file formatter and file handler
file_formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler = logging.FileHandler('debug.log')
file_handler.setFormatter(file_formatter)
file_handler.setLevel(logging.DEBUG)
# use a module logger instead of the root logger
logger = logging.getLogger(__name__)
# add the handlers
logger.addHandler(console_handler)
logger.addHandler(file_handler)
# include messages with level DEBUG and higher
logger.setLevel(logging.DEBUG)
# NOW, instead of using print(), we use logger.log() with our CUSTOM_LEVEL
x = 4
y = 5
logger.debug(f'X: {x}')
logger.debug(f'Y: {y}')
logger.log(CUSTOM_LEVEL, f'x is {x}')
logger.log(CUSTOM_LEVEL, f'y is {y}')
logger.log(CUSTOM_LEVEL, f'x * y = {x*y}')
logger.log(CUSTOM_LEVEL, f'x^y = {x**y}')

I don't think you want ALL stdout output going to log file.
What you could do is set the logging level of the console handler to logging.INFO and the logging level of the file handler to logging.DEBUG. Then replace your print statements with calls to logging.info. This way only info messages and above will be output to the console.
Something like this:
import logging
import sys
logger = logging.getLogger(__name__)
console_handler = logging.StreamHandler(sys.stdout)
file_handler = logging.FileHandler("debug.log")
console_handler.setLevel(logging.INFO)
file_handler.setLevel(logging.DEBUG)
console_formatter = logging.Formatter('%(message)s')
file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
console_handler.setFormatter(console_formatter)
file_handler.setFormatter(file_formatter)
logger.addHandler(console_handler)
logger.addHandler(file_handler)
logger.setLevel(logging.DEBUG) #set root logging level to DEBUG
if __name__ == "__main__":
x = 4
y = 5
logger.debug("X: %s", x)
logger.debug("Y: %s", y)
logger.info("x is {}".format(x))
logger.info("y is {}".format(y))
logger.info("x * y = {}".format(x * y))
logger.info("x^y = {}".format(x ** y))
Demo

How to log time spent from when script started instead of actual time when using python logging module?

I am looking for a way to change python logging module to display time spent form when the script started instead of current time.

Use %(relativeCreated)s in your format string, as indicated in the documentation.
Update: You can use normal Python format specifiers to control precision, e.g. %(relativeCreated).0f to show floating point values with zero decimal places.

You can subclass logging.Formatter and reimplement formatTime. Something like that:
start_time = datetime.now()
class MyFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None):
delta = (datetime.now() - start_time).total_seconds()
return "{}".format(delta)
And then:
handler = logging.StreamHandler()
fmt = MyFormatter('%(filename)s %(levelname)-8s [%(asctime)s] %(message)s')
handler.setFormatter(fmt)
log = logging.getLogger('main')
log.addHandler(handler)
log.debug("=)")

You could try something like this.
Define a custom formatter at the beginning of your script:
import time
import logging
import datetime as dt
class MyFormatter(logging.Formatter):
def __init__(self,fmt=None,datefmt=None):
super(MyFormatter,self).__init__(fmt,datefmt)
self.reftime = dt.datetime.fromtimestamp(time.mktime(time.localtime()))
def formatTime(self, record, datefmt=None):
ctime = dt.datetime.fromtimestamp(time.mktime(self.converter(record.created)))
ct = (ctime - self.reftime).timetuple()
if datefmt:
s = time.strftime(datefmt, ct)
else:
t = time.strftime("%Y-%m-%d %H:%M:%S", ct)
s = "%s,%03d" % (t, record.msecs)
return s
Then setup your logging system:
handler = logging.StreamHandler(MyFormatter())
logger = logging.getLogger()
logger.addHandler(handler)
etc...

How can I log current line, and stack info with Python?

I have logging function as follows.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def printinfo(string):
if DEBUG:
logging.info(string)
def printerror(string):
if DEBUG:
logging.error(string)
print string
I need to login the line number, stack information. For example:
1: def hello():
2: goodbye()
3:
4: def goodbye():
5: printinfo()
---> Line 5: goodbye()/hello()
How can I do this with Python?
SOLVED
def printinfo(string):
if DEBUG:
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
logging.debug(stack_trace[:-1])
if LOG:
logging.info(string)
gives me this info which is exactly what I need.
DEBUG 2011-02-23 10:09:13,500 [
' File "/abc.py", line 553, in <module>\n runUnitTest(COVERAGE, PROFILE)\n',
' File "/abc.py", line 411, in runUnitTest\n printinfo(string)\n']

Current function name, module and line number you can do simply by changing your format string to include them.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(module)s:%(lineno)s %(funcName)s %(message)s",
level = logging.DEBUG
)
Most people only want the stack when logging an exception, and the logging module does that automatically if you call logging.exception(). If you really want stack information at other times then you will need to use the traceback module for extract the additional information you need.

import inspect
import traceback
def method():
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
print ''.join(stack_trace)
Use stack_trace[:-1] to avoid including method/printinfo in the stack trace.

As of Python 3.2, this can be simplified to passing the stack_info=True flag to the logging calls. However, you'll need to use one of the above answers for any earlier version.

Late answer, but oh well.
Another solution is that you can create your own formatter with a filter as specified in the docs here. This is a really great feature as you now no longer have to use a helper function (and have to put the helper function everywhere you want the stack trace). Instead, a custom formatted implements it directly into the logs themselves.
import logging
class ContextFilter(logging.Filter):
def __init__(self, trim_amount)
self.trim_amount = trim_amount
def filter(self, record):
import traceback
record.stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim_amount]
)
return True
# Now you can create the logger and apply the filter.
logger = logging.getLogger(__name__)
logger.addFilter(ContextFilter(5))
# And then you can directly implement a stack trace in the formatter.
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s \n %(stack)s')
Note: In the above code I trim the last 5 stack frames. This is just for convenience and so that we don't show stack frames from the python logging package itself.(It also might have to be adjusted for different versions of the logging package)

Use the traceback module.
logging.error(traceback.format_exc())

Here is an example that i hope it can help you:
import inspect
import logging
logging.basicConfig(
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def test():
caller_list = []
frame = inspect.currentframe()
this_frame = frame # Save current frame.
while frame.f_back:
caller_list.append('{0}()'.format(frame.f_code.co_name))
frame = frame.f_back
caller_line = this_frame.f_back.f_lineno
callers = '/'.join(reversed(caller_list))
logging.info('Line {0} : {1}'.format(caller_line, callers))
def foo():
test()
def bar():
foo()
bar()
Result:
INFO 2011-02-23 17:03:26,426 Line 28 : bar()/foo()/test()

Look at traceback module
>>> import traceback
>>> def test():
>>> print "/".join( str(x[2]) for x in traceback.extract_stack() )
>>> def main():
>>> test()
>>> main()
<module>/launch_new_instance/mainloop/mainloop/interact/push/runsource/runcode/<module>/main/test

This is based on #mouad's answer but made more useful (IMO) by including at each level the filename (but not its full path) and line number of the call stack, and by leaving the stack in most-recently-called-from (i.e. NOT reversed) order because that's the way I want to read it :-)
Each entry has file:line:func() which is the same sequence as the normal stacktrace, but all on the same line so much more compact.
import inspect
def callers(self):
caller_list = []
frame = inspect.currentframe()
while frame.f_back:
caller_list.append('{2}:{1}:{0}()'.format(frame.f_code.co_name,frame.f_lineno,frame.f_code.co_filename.split("\\")[-1]))
frame = frame.f_back
callers = ' <= '.join(caller_list)
return callers
You may need to add an extra f_back if you have any intervening calls to produce the log text.
frame = inspect.currentframe().f_back
Produces output like this:
file2.py:620:func1() <= file3.py:211:func2() <= file3.py:201:func3() <= main.py:795:func4() <= file4.py:295:run() <= main.py:881:main()
I only need this stacktrace in two key functions, so I add the output of callers into the text in the logger.debug() call, like htis:
logger.debug("\nWIRE: justdoit request -----\n"+callers()+"\n\n")

logger chain in python

I'm writing python package/module and would like the logging messages mention what module/class/function they come from. I.e. if I run this code:
import mymodule.utils.worker as worker
w = worker.Worker()
w.run()
I'd like to logging messages looks like this:
2010-06-07 15:15:29 INFO mymodule.utils.worker.Worker.run <pid/threadid>: Hello from worker
How can I accomplish this?
Thanks.

I tend to use the logging module in my packages/modules like so:
import logging
log = logging.getLogger(__name__)
log.info("Whatever your info message.")
This sets the name of your logger to the name of the module for inclusion in the log message. You can control where the name is by where %(name)s is found in the format string. Similarly you can place the pid with %(process)d and the thread id with %(thread)d. See the docs for all the options.
Formatting example:
import logging
logging.basicConfig(format="%(asctime)s %(levelname)s %(name)s %(process)d/%(threadName)s: %(message)s")
logging.getLogger('this.is.the.module').warning('Testing for SO')
Gives me:
2010-06-07 08:43:10,494 WARNING this.is.the.module 14980/MainThread: Testing for SO

Here is my solution that came out of this discussion. Thanks to everyone for suggestions.
Usage:
>>> import logging
>>> logging.basicConfig(level=logging.DEBUG)
>>> from hierlogger import hierlogger as logger
>>> def main():
... logger().debug("test")
...
>>> main()
DEBUG:main:test
By default it will name logger as ... You can also control the depth by providing parameter:
3 - module.class.method default
2 - module.class
1 - module only
Logger instances are also cached to prevent calculating logger name on each call.
I hope someone will enjoy it.
The code:
import logging
import inspect
class NullHandler(logging.Handler):
def emit(self, record): pass
def hierlogger(level=3):
callerFrame = inspect.stack()[1]
caller = callerFrame[0]
lname = '__heirlogger'+str(level)+'__'
if lname not in caller.f_locals:
loggerName = str()
if level >= 1:
try:
loggerName += inspect.getmodule(inspect.stack()[1][0]).__name__
except: pass
if 'self' in caller.f_locals and (level >= 2):
loggerName += ('.' if len(loggerName) > 0 else '') +
caller.f_locals['self'].__class__.__name__
if callerFrame[3] != '' and level >= 3:
loggerName += ('.' if len(loggerName) > 0 else '') + callerFrame[3]
caller.f_locals[lname] = logging.getLogger(loggerName)
caller.f_locals[lname].addHandler(NullHandler())
return caller.f_locals[lname]