Develop Reference

Python Logging - Closing one file across multiple logs (ResourceWarning)

I created a custom logging.Logger that is used by several different objects in a script I'm running like so:
class TestLogger(logging.Logger):
def __init__(self, name, file=None):
super(TestLogger, self).__init__(name, level=logging.DEBUG)
self.log_file = file
...
def addLogFile(self, log_file):
self.log_file = log_file
self.setFormat()
# set the format of the log
def setFormat(self, default=True, end='\n'):
# remove any Handlers
self.removeStreamHandlers()
self.removeFileHandlers()
# get the log format string, default or message
format_str = DEFAULT_FORMAT if default else CUSTOM_FORMAT
std_formatter = logging.Formatter(format_str, datefmt=self.DATE_FORMAT)
# add the stream handler
console = logging.StreamHandler(sys.stdout)
console.setFormatter(std_formatter)
console.terminator = end
self.addHandler(console)
# add the file handler
if self.log_file:
file_formatter = logging.Formatter(format_str, datefmt=self.DATE_FORMAT)
logger = logging.FileHandler(self.log_file)
logger.setFormatter(file_formatter)
self.addHandler(logger)
# remove all stream handlers
def removeStreamHandlers(self):
stream_handlers = [h for h in self.handlers if isinstance(h, logging.StreamHandler)
and not isinstance(h, logging.FileHandler)]
for sh in stream_handlers:
self.removeHandler(sh)
# remove all file handlers
def removeFileHandlers(self):
file_handlers = [h for h in self.handlers if isinstance(h, logging.StreamHandler)
and isinstance(h, logging.FileHandler)]
for fh in file_handlers:
self.removeHandler(fh)
class Something:
def __init__(self):
self.log = TestLogger('Something')
...
def __del__(self):
self.log.removeFileHandlers()
self.log.removeStreamHandler()
class SomethingElse:
def __init__(self):
self.log = TestLogger('SomethingElse')
...
def __del__(self):
self.log.removeFileHandlers()
self.log.removeStreamHandler()
All of these objects are initialized and designed to share the same log file like so:
log_file = 'test.log'
s = Something()
se = SomethingElse()
s.addLogFile(log_file)
se.addLogFile(log_file)
...
del s, se
The problem seems to be that when I try to rerun my program, it throws a ResourceWarning every time I run setFormat(). It seems like the file isn't properly being closed and I'm not sure where this could be happening.

First of all, you should reuse the handlers. If you want to change the formatter just call setFormatter on the existing handlers and keep them.
If you really do want to throw away the handler and use a new one, there is a close() method on the FileHandler that is supposed to be called to clean up when the handler is done logging. So in your case you would change your code to look like this:
for fh in file_handlers:
fh.close()
self.removeHandler(fh)

How to use Python to log to a new directory each day?

I'm setting up a readout system that takes data from a number of instruments and needs to log the data to a log file. This system will be running for weeks at a time, and so each day should have a log file. Since these instruments are being manipulated over this time, they may also have log files associated with their status.
With this, I have a directory in which all of the logs are stored, for example 'C:/logs'. Since there will be multiple log files associated with each day, I'd like to automate the creation of a new subdirectory in the the logs folder each day, so the structure of the files are something like 'C:/logs/20190814' for August 14, 'C:/logs/20190815' for the 15th, and so on. Then, in each daily directory I would have a number of log files such as 'data.log', 'instrument1.log', 'instrument2.log', etc.
Ideally, these would roll over at midnight each day.
I have been using the Python Logging module to attempt to create these log files. I have been able to implement the TimedRotatingFileHandler, but the problem with this is
(1) I want to change the directory that the log files are in based on the day, but leave their titles the same (e.g. 'C:/logs/20190814/data.log', 'C:/logs/20190815/data.log')
(2) the TimedRotatingFileHandler saves the files not with a '%Y%m%d.log' extension, but rather '.log.%Y%m%d', which is inconvenient to work with. I'd like to create a new directory each day and start writing a new log in the new day's directory.

Using the framework from another StackOverflow question that's similar but not exactly what I needed, I was able to get the behavior that I wanted. Here's the custom class that updates the logging TimedRotatingFileHandler class.
class MyTimedRotatingFileHandler(logging.handlers.TimedRotatingFileHandler):
def __init__(self, log_title, whenTo="midnight", intervals=1):
self.when = whenTo.upper()
self.inter = intervals
self.log_file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "logs"))
if not os.path.isdir(self.log_file_path):
os.mkdir(self.log_file_path)
if self.when == "S":
self.extStyle = "%Y%m%d%H%M%S"
if self.when == "M":
self.extStyle = "%Y%m%d%H%M"
if self.when == "H":
self.extStyle = "%Y%m%d%H"
if self.when == "MIDNIGHT" or self.when == "D":
self.extStyle = "%Y%m%d"
self.dir_log = os.path.abspath(os.path.join(self.log_file_path, datetime.now().strftime(self.extStyle)))
if not os.path.isdir(self.dir_log):
os.mkdir(self.dir_log)
self.title = log_title
filename = os.path.join(self.dir_log, self.title)
logging.handlers.TimedRotatingFileHandler.__init__(self, filename, when=whenTo, interval=self.inter, backupCount=0, encoding=None)
self._header = ""
self._log = None
self._counter = 0
def doRollover(self):
"""
TimedRotatingFileHandler remix - rotates logs on daily basis, and filename of current logfile is time.strftime("%m%d%Y")+".txt" always
"""
self.stream.close()
# get the time that this sequence started at and make it a TimeTuple
t = self.rolloverAt - self.interval
timeTuple = time.localtime(t)
self.new_dir = os.path.abspath(os.path.join(self.log_file_path, datetime.now().strftime(self.extStyle)))
if not os.path.isdir(self.new_dir):
os.mkdir(self.new_dir)
self.baseFilename = os.path.abspath(os.path.join(self.new_dir, self.title))
if self.encoding:
self.stream = codecs.open(self.baseFilename, "w", self.encoding)
else:
self.stream = open(self.baseFilename, "w")
self.rolloverAt = self.rolloverAt + self.interval

Here is an example:
import logging
import time
from logging.handlers import TimedRotatingFileHandler
#----------------------------------------------------------------------
def create_timed_rotating_log(path):
""""""
logger = logging.getLogger("Rotating Log")
logger.setLevel(logging.INFO)
handler = TimedRotatingFileHandler(path,
when="m",
interval=1,
backupCount=5)
logger.addHandler(handler)
for i in range(6):
logger.info("This is a test!")
time.sleep(75)
#----------------------------------------------------------------------
if __name__ == "__main__":
log_file = "timed_test.log"
create_timed_rotating_log(log_file)
This example will rotate the log every minute with a back up count of 5. A more realistic rotation would probably be on the hour, so you would set the interval to 60 or the when to “h”. When this code is run, it too will create 6 files, but instead of appending integers to the log file name, it will append a timestamp using the strftime format %Y-%m-%d_%H-%M-%S.

The easiest way of logging

I would like to collect info with the help of logging.
The idea is simple. I have hash_value of some data, which I want to write to log. So, I set up my logging this way:
import logging
logger.setLevel(logging.DEBUG)
logging.basicConfig(format='%(asctime)s :%(message)s', level=logging.INFO)
As you can see, now timing and some message will automatically write to log file, for example I can use it like this:
logger.info('Initial data: {}'.format(data))
But what if I want to write hash_value of my data automatically? Like it is happening with time now.
I looked through documentation and find nothing useful. There is no attribute for variable in module logging.
So I am forced to do it awry. Like this:
hash_value = hash(data)
logger.info('Initial data: {} {}'.format(hash_value, data))
I would expect from this code:
logging.basicConfig(format='%(asctime)s: %(variable)s :%(message)s', level=logging.INFO)
and
logger.info('Initial data: {}'.format(hash_value, data))
to do the job. But it does not work (and it should not basically) and I did not find the solution in documentation.
So, how to avoid this awry code:
logger.info('Initial data: {} {}'.format(hash_value, data))
which I am having now?

import logging
import sys
MY_PARAMS = ("variable1", "var2", )
class ExtraFilter(logging.Filter):
def filter(self, record):
# this one used for second, simplier handler
# to avoid duplicate of logging entries if "extra" keyword was passed.
# Check all of your custom params:
# if all of them are present - record should be filtered
# * all because if any of them is missing - there would be silent exception and record wont be logged at all
# bellow is just an example how to check.
# You can use something like this:
# if all(hasattr(record, param) for param in MY_PARAMS): return False
if hasattr(record, "variable1"):
return False
return True
# init logging
log = logging.getLogger()
# init handlers and formatters
h1 = logging.StreamHandler(sys.stdout)
f1 = logging.Formatter('%(asctime)s: %(variable1)s: %(var2)s: %(message)s')
h2 = logging.StreamHandler(sys.stdout)
f2 = logging.Formatter('%(asctime)s: %(message)s')
h1.setFormatter(f1)
h2.setFormatter(f2)
h2.addFilter(ExtraFilter())
log.addHandler(h1)
log.addHandler(h2)
# example of data:
extra = {"variable1": "test1", "var2": "test2"}
log.setLevel(logging.DEBUG)
log.debug("debug message", extra=extra)
log.info("info message")
The above code will produce following output:
2017-11-04 09:16:36,787: test1: test2: debug message
2017-11-04 09:16:36,787: info message

It is not awry code, you want to add two informations, therefore you must either pass two parameters to format or concatenate the string more "manually"
You could go with
Logging.info("initial data " + hash_value + " " + data)
Or you could change the "data" object so its "str" or the repr method adds the hash by itself (preferably the repr in this case)
Class Data():
....
def __repr__(self):
Return self.hash() + " " self.data
Which in this case will print the hash and the string version of the parameter data( or simply whatever you want to show as string) passing only one parameter in the string format.
Anyway, you could make the formating string prettier with....
Logging.info("Initial data {hash} {data}".format(hash=hash_value, data=data))
By the way, in C++ and Java you would also need to declare two "entries" for those two atributes. In java would be something like this:
LOGGING.info("Initial data {} {}", hash, data);

How to use the Typed unmarshaller in suds?

I have existing code that processes the output from suds.client.Client(...).service.GetFoo(). Now that part of the flow has changed and we are no longer using SOAP, instead receiving the same XML through other channels. I would like to re-use the existing code by using the suds Typed unmarshaller, but so far have not been successful.
I came 90% of the way using the Basic unmarshaller:
tree = suds.umx.basic.Basic().process(xmlroot)
This gives me the nice tree of objects with attributes, so that the pre-existing code can access tree[some_index].someAttribute, but the value will of course always be a string, rather than an integer or date or whatever, so the code can still not be re-used as-is.
The original class:
class SomeService(object):
def __init__(self):
self.soap_client = Client(some_wsdl_url)
def GetStuff(self):
return self.soap_client.service.GetStuff()
The drop-in replacement that almost works:
class SomeSourceUntyped(object):
def __init__(self):
self.url = some_url
def GetStuff(self):
xmlfile = urllib2.urlopen(self.url)
xmlroot = suds.sax.parser.Parser().parse(xmlfile)
if xmlroot:
# because the parser creates a document root above the document root
tree = suds.umx.basic.Basic().process(xmlroot)[0]
else:
tree = None
return tree
My vain effort to understand suds.umx.typed.Typed():
class SomeSourceTyped(object):
def __init__(self):
self.url = some_url
self.schema_file_name =
os.path.realpath(os.path.join(os.path.dirname(__file__),'schema.xsd'))
with open(self.schema_file_name) as f:
self.schema_node = suds.sax.parser.Parser().parse(f)
self.schema = suds.xsd.schema.Schema(self.schema_node, "", suds.options.Options())
self.schema_query = suds.xsd.query.ElementQuery(('http://example.com/namespace/','Stuff'))
self.xmltype = self.schema_query.execute(self.schema)
def GetStuff(self):
xmlfile = urllib2.urlopen(self.url)
xmlroot = suds.sax.parser.Parser().parse(xmlfile)
if xmlroot:
unmarshaller = suds.umx.typed.Typed(self.schema)
# I'm still running into an exception, so obviously something is missing:
# " Exception: (document, None, ), must be qref "
# Do I need to call the Parser differently?
tree = unmarshaller.process(xmlroot, self.xmltype)[0]
else:
tree = None
return tree
This is an obscure one.
Bonus caveat: Of course I am in a legacy system that uses suds 0.3.9.
EDIT: further evolution on the code, found how to create SchemaObjects.

How can I log current line, and stack info with Python?

I have logging function as follows.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def printinfo(string):
if DEBUG:
logging.info(string)
def printerror(string):
if DEBUG:
logging.error(string)
print string
I need to login the line number, stack information. For example:
1: def hello():
2: goodbye()
3:
4: def goodbye():
5: printinfo()
---> Line 5: goodbye()/hello()
How can I do this with Python?
SOLVED
def printinfo(string):
if DEBUG:
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
logging.debug(stack_trace[:-1])
if LOG:
logging.info(string)
gives me this info which is exactly what I need.
DEBUG 2011-02-23 10:09:13,500 [
' File "/abc.py", line 553, in <module>\n runUnitTest(COVERAGE, PROFILE)\n',
' File "/abc.py", line 411, in runUnitTest\n printinfo(string)\n']

Current function name, module and line number you can do simply by changing your format string to include them.
logging.basicConfig(
filename = fileName,
format = "%(levelname) -10s %(asctime)s %(module)s:%(lineno)s %(funcName)s %(message)s",
level = logging.DEBUG
)
Most people only want the stack when logging an exception, and the logging module does that automatically if you call logging.exception(). If you really want stack information at other times then you will need to use the traceback module for extract the additional information you need.

import inspect
import traceback
def method():
frame = inspect.currentframe()
stack_trace = traceback.format_stack(frame)
print ''.join(stack_trace)
Use stack_trace[:-1] to avoid including method/printinfo in the stack trace.

As of Python 3.2, this can be simplified to passing the stack_info=True flag to the logging calls. However, you'll need to use one of the above answers for any earlier version.

Late answer, but oh well.
Another solution is that you can create your own formatter with a filter as specified in the docs here. This is a really great feature as you now no longer have to use a helper function (and have to put the helper function everywhere you want the stack trace). Instead, a custom formatted implements it directly into the logs themselves.
import logging
class ContextFilter(logging.Filter):
def __init__(self, trim_amount)
self.trim_amount = trim_amount
def filter(self, record):
import traceback
record.stack = ''.join(
str(row) for row in traceback.format_stack()[:-self.trim_amount]
)
return True
# Now you can create the logger and apply the filter.
logger = logging.getLogger(__name__)
logger.addFilter(ContextFilter(5))
# And then you can directly implement a stack trace in the formatter.
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s \n %(stack)s')
Note: In the above code I trim the last 5 stack frames. This is just for convenience and so that we don't show stack frames from the python logging package itself.(It also might have to be adjusted for different versions of the logging package)

Use the traceback module.
logging.error(traceback.format_exc())

Here is an example that i hope it can help you:
import inspect
import logging
logging.basicConfig(
format = "%(levelname) -10s %(asctime)s %(message)s",
level = logging.DEBUG
)
def test():
caller_list = []
frame = inspect.currentframe()
this_frame = frame # Save current frame.
while frame.f_back:
caller_list.append('{0}()'.format(frame.f_code.co_name))
frame = frame.f_back
caller_line = this_frame.f_back.f_lineno
callers = '/'.join(reversed(caller_list))
logging.info('Line {0} : {1}'.format(caller_line, callers))
def foo():
test()
def bar():
foo()
bar()
Result:
INFO 2011-02-23 17:03:26,426 Line 28 : bar()/foo()/test()

Look at traceback module
>>> import traceback
>>> def test():
>>> print "/".join( str(x[2]) for x in traceback.extract_stack() )
>>> def main():
>>> test()
>>> main()
<module>/launch_new_instance/mainloop/mainloop/interact/push/runsource/runcode/<module>/main/test

This is based on #mouad's answer but made more useful (IMO) by including at each level the filename (but not its full path) and line number of the call stack, and by leaving the stack in most-recently-called-from (i.e. NOT reversed) order because that's the way I want to read it :-)
Each entry has file:line:func() which is the same sequence as the normal stacktrace, but all on the same line so much more compact.
import inspect
def callers(self):
caller_list = []
frame = inspect.currentframe()
while frame.f_back:
caller_list.append('{2}:{1}:{0}()'.format(frame.f_code.co_name,frame.f_lineno,frame.f_code.co_filename.split("\\")[-1]))
frame = frame.f_back
callers = ' <= '.join(caller_list)
return callers
You may need to add an extra f_back if you have any intervening calls to produce the log text.
frame = inspect.currentframe().f_back
Produces output like this:
file2.py:620:func1() <= file3.py:211:func2() <= file3.py:201:func3() <= main.py:795:func4() <= file4.py:295:run() <= main.py:881:main()
I only need this stacktrace in two key functions, so I add the output of callers into the text in the logger.debug() call, like htis:
logger.debug("\nWIRE: justdoit request -----\n"+callers()+"\n\n")