I am working on saving the json logs using python. Below is the code:
log_file = 'app_log.json'
log_json = dict()
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
log_json['Created'] = datetime.datetime.utcnow().isoformat()
with open(log_file, "a") as f:
json.dump(log_json, f)#, ensure_ascii=False)
f.write("\n")
Now above code is generating the log file. But I have noticed that the file size is increasing a lot and in future, I might face disk space issue. I was wondering if there is any pre built rotating file handler available for json in which we can mention fixed size lets say 100mb and upon reaching this size it will delete and recreate the new file.
I have previously used from logging.handlers import RotatingFileHandler to do this in case of .log files but also want to do this for .json files. Please help. Thanks
Python does not care about the log file name.
You can use the rotating handler which you used for .log file for .json file also.
See sample example below
# logging_example.py
import logging
import logging.handlers
import os
import time
logfile = os.path.join("/tmp", "demo_logging.json")
logger = logging.getLogger(__name__)
fh = logging.handlers.RotatingFileHandler(logfile, mode='a', maxBytes=1000, backupCount=5) # noqa:E501
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.setLevel(logging.DEBUG)
while 1:
time.sleep(1)
logger.info("Long string to increase the file size")
You can also look at logrotate if you are working in Unix environment. It is a great and simple tool with good documentation to do just exactly what you need.
You can implement structured logging with RotatingFileHandler
import json
import logging
import logging.handlers
from datetime import datetime
class StructuredMessage:
def __init__(self, message, /, **kwargs):
self.message = message
self.kwargs = kwargs
def __str__(self):
return '%s >>> %s' % (self.message, json.dumps(self.kwargs))
_ = StructuredMessage # optional, to improve readability
log_json = {}
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
log_json['Created'] = datetime.utcnow().isoformat()
LOG_FILENAME = 'logging_rotatingfile_example.out'
# Set up a specific logger with our desired output level
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Add the log message handler to the logger
handler = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=20, backupCount=5)
bf = logging.Formatter('%(message)s')
handler.setFormatter(bf)
logger.addHandler(handler)
logger.info(_('INFO', **log_json))
Note: check here for more info about structured-logging-python
you can use also use json-logging-python with RotatingFileHandler
import logging
import json
import traceback
from datetime import datetime
import copy
import json_logging
import sys
json_logging.ENABLE_JSON_LOGGING = True
def extra(**kw):
'''Add the required nested props layer'''
return {'extra': {'props': kw}}
class CustomJSONLog(logging.Formatter):
"""
Customized logger
"""
def get_exc_fields(self, record):
if record.exc_info:
exc_info = self.format_exception(record.exc_info)
else:
exc_info = record.exc_text
return {'python.exc_info': exc_info}
#classmethod
def format_exception(cls, exc_info):
return ''.join(traceback.format_exception(*exc_info)) if exc_info else ''
def format(self, record):
json_log_object = {"#timestamp": datetime.utcnow().isoformat(),
"level": record.levelname,
"message": record.getMessage(),
"caller": record.filename + '::' + record.funcName
}
json_log_object['data'] = {
"python.logger_name": record.name,
"python.module": record.module,
"python.funcName": record.funcName,
"python.filename": record.filename,
"python.lineno": record.lineno,
"python.thread": record.threadName,
"python.pid": record.process
}
if hasattr(record, 'props'):
json_log_object['data'].update(record.props)
if record.exc_info or record.exc_text:
json_log_object['data'].update(self.get_exc_fields(record))
return json.dumps(json_log_object)
json_logging.init_non_web(custom_formatter=CustomJSONLog, enable_json=True)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
LOG_FILENAME = 'logging_rotating_json_example.out'
handler = logging.handlers.RotatingFileHandler(
LOG_FILENAME, maxBytes=20, backupCount=5)
logger.addHandler(handler)
log_json = {}
log_json["Data"] = {}
log_json['Data']['Key1'] = "value1"
log_json['Data']['alert'] = False
log_json['Data']['Key2'] = "N/A"
logger.info('Starting')
logger.debug('Working', extra={"props":log_json})
Note: check here for more info about json-logging-python
You can try this just before you write / append to the file. This should check to see if the file has reached the max lines size, then it will remove one line of code from the beginning of the file before you append as usual to the end of the file.
filename = 'file.txt'
maxLines = 100
count = len(open(filename).readlines())
if(count > maxLines) {
with open(filename, 'r') as fin:
data = fin.read().splitlines(True)
with open(filename, 'w') as fout:
fout.writelines(data[1:])
}
Related
So this code outputs level names with colors (default levels), and I'd like to add extra levels of my own, and then give them a custom color.
Code:
import logging
import re
import time
import sys
def set_colour(level):
"""
Sets colour of text for the level name in
logging statements using a dispatcher.
"""
escaped = "[\033[1;%sm%s\033[1;0m]"
return {
'INFO': lambda: logging.addLevelName(logging.INFO, escaped % ('94', level)),
'WARNING': lambda: logging.addLevelName(logging.ERROR, escaped % ('93', level)),
'ERROR': lambda: logging.addLevelName(logging.WARNING, escaped % ('91', level))
}.get(level, lambda: None)()
class NoColorFormatter(logging.Formatter):
"""
Log formatter that strips terminal colour
escape codes from the log message.
"""
# Regex for ANSI colour codes
ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def format(self, record):
"""Return logger message with terminal escapes removed."""
return "%s %s %s" % (
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
re.sub(self.ANSI_RE, "", record.levelname),
record.msg,
)
# Create logger
logger = logging.getLogger(__package__)
# Create formatters
logformatter = NoColorFormatter()
colorformatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
# Set logging colours
for level in 'INFO', 'ERROR', 'WARNING':
set_colour(level)
# Set logging level
logger.setLevel(logging.INFO)
# Set log handlers
loghandler = logging.FileHandler("log.txt", mode="a", encoding="utf8")
streamhandler = logging.StreamHandler(sys.stdout)
# Set log formatters
loghandler.setFormatter(logformatter)
streamhandler.setFormatter(colorformatter)
# Attach log handlers to logger
logger.addHandler(loghandler)
logger.addHandler(streamhandler)
# Example logging statements
logging.info("This is just an information for you")
logging.warning("This is just a warning for you")
logging.error("This is just an error for you")
What I want to accomplish is a new level, but with its unique color.
Here's the code on how I accomplish adding a new level:
def success(msg, *args, **kwargs):
if logging.getLogger().isEnabledFor(70):
logging.log(70, msg)
logging.addLevelName(70, "SUCCESS")
logging.success = success
logging.Logger.success = success
The above code works fine normally but does not include any color. How can I add this code to have a new level, but with a different color?
This requires the addition of a handful of lines, including this block (with the addition of another line to set the integer value for logging.SUCCESS:
def success(msg, *args, **kwargs):
if logging.getLogger().isEnabledFor(70):
logging.log(70, msg)
logging.addLevelName(70, "SUCCESS")
logging.SUCCESS = 70 # similar to logging.INFO -> 20
logging.success = success
logging.Logger.success = success
I've indicated the lines that have been added/modified here. To add further additional levels, defining the same structures for the new ones, and modifying the for loop and set_colour() functions should be enough.
import logging
import re
import time
import sys
def set_colour(level):
"""
Sets colour of text for the level name in
logging statements using a dispatcher.
"""
escaped = "[\033[1;%sm%s\033[1;0m]"
return {
'INFO': lambda: logging.addLevelName(logging.INFO, escaped % ('94', level)),
'WARNING': lambda: logging.addLevelName(logging.ERROR, escaped % ('93', level)),
'ERROR': lambda: logging.addLevelName(logging.WARNING, escaped % ('91', level)),
'SUCCESS': lambda: logging.addLevelName(logging.SUCCESS, escaped % ('31', level)) # new
}.get(level, lambda: None)()
class NoColorFormatter(logging.Formatter):
"""
Log formatter that strips terminal colour
escape codes from the log message.
"""
# Regex for ANSI colour codes
ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
def format(self, record):
"""Return logger message with terminal escapes removed."""
return "%s %s %s" % (
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
re.sub(self.ANSI_RE, "", record.levelname),
record.msg,
)
def success(msg, *args, **kwargs): # new
if logging.getLogger().isEnabledFor(70): # new
logging.log(70, msg) # new
# Create logger
logger = logging.getLogger(__package__)
# Create formatters
logformatter = NoColorFormatter()
colorformatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
# Create new level
logging.SUCCESS = 70 # new
logging.success = success # new
logging.Logger.success = success # new
# Set logging colours
for level in 'INFO', 'ERROR', 'WARNING', 'SUCCESS': # modified
set_colour(level)
# Set logging level
logger.setLevel(logging.INFO)
# Set log handlers
loghandler = logging.FileHandler("log.txt", mode="w", encoding="utf8")
streamhandler = logging.StreamHandler(sys.stdout)
# Set log formatters
loghandler.setFormatter(logformatter)
streamhandler.setFormatter(colorformatter)
# Attach log handlers to logger
logger.addHandler(loghandler)
logger.addHandler(streamhandler)
# Example logging statements
logging.info("This is just an information for you")
logging.warning("This is just an information for you")
logging.error("This is just an information for you")
logging.success("This is just an information for you")
I have a logging class that describes a base logger.
class logger:
def __init__(self):
self.filelocation = 'log/util.log'
self.loggers = {}
def init_logger(self,name="util",):
if self.loggers.get(name):
return self.loggers.get(name)
else:
module_logger = logging.getLogger(name)
module_logger.setLevel(logging.DEBUG)
module_logger.propagate = False
# create file handler which logs even debug messages
fh = logging.FileHandler(self.filelocation)
fh.setLevel(logging.DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.WARNING)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(module)s : %(levelname)s : %(asctime)s : %(name)s : %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
module_logger.addHandler(fh)
module_logger.addHandler(ch)
self.loggers[name] = module_logger
return module_logger
Now I have multiple modules referencing the above said logger. For ex
# mod1.py
logsession = logger().init_logger(name = "athena_s3")
dictsession = logger().init_logger(name = "dictinfo")
class mod1_class():
def __init__(self):
self.var1 = etc
def build_session(self):
"""
Build a session using the athena client
Input: None
Return: Active session
"""
if not self._session:
try:
self._session = Session(aws_access_key_id = self._aws_access_key_id,aws_secret_access_key = self._aws_secret_access_key)
logsession.info("Built new athena session")
Similarly I have another module that could reference the code from the above mod1.py. Now consider a test.py file that imports this mod1.py.
from mod1 import mod1_class
session = mod1_class().build_session()
### Do STUFF
How can I pass the logger from multiple test.py to mod1.py such that it maintains the same logger ?
So for example:
logs could be
test : INFO : time : athena_s3 : message
test : INFO : time : athena_s3 : athena_util : message
test2: INFO : time : athena_s2 : message
I'm trying log to different StringIOs. I would've expected that each logger would write to each StringIO that was instantiated and given to. But that's not working, only the first StringIO stores the logged info.
Any suggestions on what I'm missing?
import logging
from StringIO import StringIO
formatter = logging.Formatter('%(asctime)s %(name)-3s %(levelname)-4s %(message)s')
log_stream1 = StringIO()
log1 = logging.getLogger('a')
log1.setLevel(logging.DEBUG)
stream_handler1 = logging.StreamHandler(log_stream1)
stream_handler1.setLevel(logging.INFO)
stream_handler1.setFormatter(formatter)
log1.addHandler(stream_handler1)
log_stream2 = StringIO()
log2 = logging.getLogger('b')
log2.setLevel(logging.DEBUG)
stream_handler2 = logging.StreamHandler(log_stream2)
stream_handler2.setLevel(logging.INFO)
stream_handler2.setFormatter(formatter)
log2.addHandler(stream_handler1)
log1.info('log1')
log2.info('log2')
output:
In [6]: log_stream1.getvalue()
Out[6]: '2017-06-08 10:05:12,468 a INFO log1\n2017-06-08 10:05:12,468 b INFO log2\n'
In [7]: log_stream2.getvalue()
Out[7]: ''
Change
log2.addHandler(stream_handler1)
To
log2.addHandler(stream_handler2)
Entire code:
import logging
from StringIO import StringIO
formatter = logging.Formatter('%(asctime)s %(name)-3s %(levelname)-4s %(message)s')
log_stream1 = StringIO()
log1 = logging.getLogger('a')
log1.setLevel(logging.DEBUG)
stream_handler1 = logging.StreamHandler(log_stream1)
stream_handler1.setLevel(logging.INFO)
stream_handler1.setFormatter(formatter)
log1.addHandler(stream_handler1)
log_stream2 = StringIO()
log2 = logging.getLogger('b')
log2.setLevel(logging.DEBUG)
stream_handler2 = logging.StreamHandler(log_stream2)
stream_handler2.setLevel(logging.INFO)
stream_handler2.setFormatter(formatter)
log2.addHandler(stream_handler2)
log1.info('log1')
log2.info('log2')
You wrote log2.addHandler(stream_handler1).
You should write log2.addHandler(stream_handler2).
I have logging function as follows.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def printinfo(string):
if DEBUG:
logging.info(string)
def printerror(string):
if DEBUG:
logging.error(string)
print string
I need to login the line number, stack information. For example:
1: def hello():
2: goodbye()
3:
4: def goodbye():
5: printinfo()
---> Line 5: goodbye()/hello()
How can I do this with Python?
SOLVED
def printinfo(string):
if DEBUG:
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
logging.debug(stack_trace[:-1])
if LOG:
logging.info(string)
gives me this info which is exactly what I need.
DEBUG 2011-02-23 10:09:13,500 [
' File "/abc.py", line 553, in <module>\n runUnitTest(COVERAGE, PROFILE)\n',
' File "/abc.py", line 411, in runUnitTest\n printinfo(string)\n']
Current function name, module and line number you can do simply by changing your format string to include them.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(module)s:%(lineno)s %(funcName)s %(message)s",
level = logging.DEBUG
)
Most people only want the stack when logging an exception, and the logging module does that automatically if you call logging.exception(). If you really want stack information at other times then you will need to use the traceback module for extract the additional information you need.
import inspect
import traceback
def method():
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
print ''.join(stack_trace)
Use stack_trace[:-1] to avoid including method/printinfo in the stack trace.
As of Python 3.2, this can be simplified to passing the stack_info=True flag to the logging calls. However, you'll need to use one of the above answers for any earlier version.
Late answer, but oh well.
Another solution is that you can create your own formatter with a filter as specified in the docs here. This is a really great feature as you now no longer have to use a helper function (and have to put the helper function everywhere you want the stack trace). Instead, a custom formatted implements it directly into the logs themselves.
import logging
class ContextFilter(logging.Filter):
def __init__(self, trim_amount)
self.trim_amount = trim_amount
def filter(self, record):
import traceback
record.stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim_amount]
)
return True
# Now you can create the logger and apply the filter.
logger = logging.getLogger(__name__)
logger.addFilter(ContextFilter(5))
# And then you can directly implement a stack trace in the formatter.
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s \n %(stack)s')
Note: In the above code I trim the last 5 stack frames. This is just for convenience and so that we don't show stack frames from the python logging package itself.(It also might have to be adjusted for different versions of the logging package)
Use the traceback module.
logging.error(traceback.format_exc())
Here is an example that i hope it can help you:
import inspect
import logging
logging.basicConfig(
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def test():
caller_list = []
frame = inspect.currentframe()
this_frame = frame # Save current frame.
while frame.f_back:
caller_list.append('{0}()'.format(frame.f_code.co_name))
frame = frame.f_back
caller_line = this_frame.f_back.f_lineno
callers = '/'.join(reversed(caller_list))
logging.info('Line {0} : {1}'.format(caller_line, callers))
def foo():
test()
def bar():
foo()
bar()
Result:
INFO 2011-02-23 17:03:26,426 Line 28 : bar()/foo()/test()
Look at traceback module
>>> import traceback
>>> def test():
>>> print "/".join( str(x[2]) for x in traceback.extract_stack() )
>>> def main():
>>> test()
>>> main()
<module>/launch_new_instance/mainloop/mainloop/interact/push/runsource/runcode/<module>/main/test
This is based on #mouad's answer but made more useful (IMO) by including at each level the filename (but not its full path) and line number of the call stack, and by leaving the stack in most-recently-called-from (i.e. NOT reversed) order because that's the way I want to read it :-)
Each entry has file:line:func() which is the same sequence as the normal stacktrace, but all on the same line so much more compact.
import inspect
def callers(self):
caller_list = []
frame = inspect.currentframe()
while frame.f_back:
caller_list.append('{2}:{1}:{0}()'.format(frame.f_code.co_name,frame.f_lineno,frame.f_code.co_filename.split("\\")[-1]))
frame = frame.f_back
callers = ' <= '.join(caller_list)
return callers
You may need to add an extra f_back if you have any intervening calls to produce the log text.
frame = inspect.currentframe().f_back
Produces output like this:
file2.py:620:func1() <= file3.py:211:func2() <= file3.py:201:func3() <= main.py:795:func4() <= file4.py:295:run() <= main.py:881:main()
I only need this stacktrace in two key functions, so I add the output of callers into the text in the logger.debug() call, like htis:
logger.debug("\nWIRE: justdoit request -----\n"+callers()+"\n\n")
I want to configure my Python logger in such a way so that each instance of logger should log in a file having the same name as the name of the logger itself.
e.g.:
log_hm = logging.getLogger('healthmonitor')
log_hm.info("Testing Log") # Should log to /some/path/healthmonitor.log
log_sc = logging.getLogger('scripts')
log_sc.debug("Testing Scripts") # Should log to /some/path/scripts.log
log_cr = logging.getLogger('cron')
log_cr.info("Testing cron") # Should log to /some/path/cron.log
I want to keep it generic and dont want to hardcode all kind of logger names I can have. Is that possible?
How about simply wrap the handler code in a function:
import os
def myLogger(name):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(os.path.join('/some/path/', name + '.log'), 'w')
logger.addHandler(handler)
return logger
log_hm = myLogger('healthmonitor')
log_hm.info("Testing Log") # Should log to /some/path/healthmonitor.log
To prevent creating duplicate handlers, care needs to be taken to ensure that myLogger(name) is only called once per name. Usually that means putting myLogger(name) inside
if __name__ == '__main__':
log_hm = myLogger('healthmonitor')
of the main script.
import os
import logging
class MyFileHandler(object):
def __init__(self, dir, logger, handlerFactory, **kw):
kw['filename'] = os.path.join(dir, logger.name)
self._handler = handlerFactory(**kw)
def __getattr__(self, n):
if hasattr(self._handler, n):
return getattr(self._handler, n)
raise AttributeError, n
logger = logging.getLogger('test')
logger.setLevel(logging.INFO)
handler = MyFileHandler(os.curdir, logger, logging.FileHandler)
logger.addHandler(handler)
logger.info('hello mylogger')
The approach used in the above solution is correct, but that has issue of adding duplicate handlers when called more than once. Here is the improved version.
import os
def getLogger(name):
# logger.getLogger returns the cached logger when called multiple times
# logger.Logger created a new one every time and that avoids adding
# duplicate handlers
logger = logging.Logger(name)
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(os.path.join('/some/path/', name + '.log'), 'a')
logger.addHandler(handler)
return logger
def test(i):
log_hm = getLogger('healthmonitor')
log_hm.info("Testing Log %s", i) # Should log to /some/path/healthmonitor.log
test(1)
test(2)
I'm trying to implement this solution with both dynamic path and file name but nothing is written in the file.
class PaymentViewSet(viewsets.ModelViewSet):
serializer_class = PaymentSerializer
queryset = Payment.objects.all()
permission_classes = [IsAuthenticated]
def paymentLog(self, paymentSerializer):
# file : logs/${terminalName}/${%Y-%m}-payments.log
terminalName = TerminalSerializer(Terminal.objects.get(pk=paymentSerializer.data.get("terminal"))).data.get("name")
filePath = os.path.join(settings.LOG_PATH, terminalName)
if not os.path.exists(filePath):
os.makedirs(filePath)
fileName = filePath + "/" + datetime.now().strftime("%Y-%m") +'-payments.log'
handler = logging.FileHandler(fileName)
handler.setFormatter('%(asctime)s [PAYMENT]- %(message)s')
logger = logging.Logger("payment")
logger.setLevel(logging.INFO)
logger.addHandler(handler)
# logger.propagate = False
logging.info(paymentSerializer.data)
# printout()
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
self.perform_create(serializer)
# log here
self.paymentLog(serializer)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
The path and file are created like intended but the log never writes.