I am trying to debugging multi-thread script. Once the exception is
raised I want to:
report it to monitoring system (just print in following example)
stop whole script (including all other threads)
call post mortem debugger prompt in a perspective raised exception
I prepare pretty complicated example to show how I tried to solve it:
#!/usr/bin/env python
import threading
import inspect
import traceback
import sys
import os
import time
def POST_PORTEM_DEBUGGER(type, value, tb):
traceback.print_exception(type, value, tb)
print
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
import rpdb
rpdb.pdb.pm()
else:
import pdb
pdb.pm()
sys.excepthook = POST_PORTEM_DEBUGGER
class MyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.exception = None
self.info = None
self.the_calling_script_name = os.path.abspath(inspect.currentframe().f_back.f_code.co_filename)
def main(self):
"Virtual method to be implemented by inherited worker"
return self
def run(self):
try:
self.main()
except Exception as exception:
self.exception = exception
self.info = traceback.extract_tb(sys.exc_info()[2])[-1]
# because of bug http://bugs.python.org/issue1230540
# I cannot use just "raise" under threading.Thread
sys.excepthook(*sys.exc_info())
def __del__(self):
print 'MyThread via {} catch "{}: {}" in {}() from {}:{}: {}'.format(self.the_calling_script_name, type(self.exception).__name__, str(self.exception), self.info[2], os.path.basename(self.info[0]), self.info[1], self.info[3])
class Worker(MyThread):
def __init__(self):
super(Worker, self).__init__()
def main(self):
""" worker job """
counter = 0
while True:
counter += 1
print self
time.sleep(1.0)
if counter == 3:
pass # print 1/0
def main():
Worker().start()
counter = 1
while True:
counter += 1
time.sleep(1.0)
if counter == 3:
pass # print 1/0
if __name__ == '__main__':
main()
The trick with
sys.excepthook = POST_PORTEM_DEBUGGER
works perfectly if no threads are involved. I found that in case of
multi-thread script I can use rpdb for debuggig by calling:
import rpdb; rpdb.set_trace()
It works perfectly for defined breakpoint but I want to debug
multi-thread script post mortem (after the uncatched exception is
raised). When I try to use rpdb in the POST_PORTEM_DEBUGGER function
with multi-thread application I get following:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
self.run()
File "./demo.py", line 49, in run
sys.excepthook(*sys.exc_info())
File "./demo.py", line 22, in POST_PORTEM_DEBUGGER
pdb.pm()
File "/usr/lib/python2.7/pdb.py", line 1270, in pm
post_mortem(sys.last_traceback)
AttributeError: 'module' object has no attribute 'last_traceback'
I looks like the
sys.excepthook(*sys.exc_info())
did not set up all what the raise command does.
I want the same behavior if the exception is raised in main() even
under started thread.
(I haven't tested my answer, but it seems to me that...)
The call to pdb.pm (pm="post mortem") fails simply because there had been no "mortem" prior to it. I.e. the program is still running.
Looking at the pdb source code, you find the implementation of pdb.pm:
def pm():
post_mortem(sys.last_traceback)
which makes me guess that what you actually want to do is call pdb.post_mortem() with no args. Looks like the default behavior does exactly what you need.
Some more source code (notice the t = sys.exc_info()[2] line):
def post_mortem(t=None):
# handling the default
if t is None:
# sys.exc_info() returns (type, value, traceback) if an exception is
# being handled, otherwise it returns None
t = sys.exc_info()[2]
if t is None:
raise ValueError("A valid traceback must be passed if no "
"exception is being handled")
p = Pdb()
p.reset()
p.interaction(None, t)
Building on #shx2's above, I now use the following pattern in the context of multithreading.
import sys, pdb
try:
... # logic that may fail
except exception as exc:
pdb.post_mortem(exc.__traceback__)
Here is a more verbose alternative:
import sys, pdb
try:
... # logic that may fail
except exception as exc:
if hasattr(sys, "last_traceback"):
pdb.pm()
else:
pdb.post_mortem(exc.__traceback__)
This can help:
import sys
from IPython.core import ultratb
sys.excepthook = ultratb.FormattedTB(mode='Verbose', color_scheme='Linux',
call_pdb=True, ostream=sys.__stdout__)
Related
Does Python support a way to display the same custom error message for every exception / raise / assert (no matter where the code broke)?
My current crack at it uses a decorator. I have a function main and it displays the traceback fine, but I want it to also print my_var (which is INSIDE the function scope) every time ANY error is thrown. So obviously there is a scope problem with this - it's just to illustrate what I want to do. Any ideas are appreciated.
import traceback
def exit_with_traceback(func, *args, **kwargs):
def wrap(*args, **kwargs):
try:
return func(*args, **kwargs)
except:
# how do I get this to print my_var AND the traceback?
print(traceback.format_exc())
return wrap
#exit_with_traceback
def main(bar=1):
my_var = 'hello world' # variable specific to main()
return bar + 1
main(bar=None) # run main() to throw the exception
You can try to override the excepthook function in sys module. From its documentation:
When an exception is raised and uncaught, the interpreter calls
sys.excepthook with three arguments, the exception class, exception
instance, and a traceback object.
So the code may look something like this (I've used your example):
import sys
# define custom exception hook
def custom_exception_hook(exc_type, value, traceback):
print('Custom exception hook')
print('Type:', exc_type)
print('Value:', value)
print('Traceback:', traceback)
lc = traceback.tb_next.tb_frame.f_locals
print(lc.get('my_var')) # this prints "hello world"
# override the default exception hook
sys.excepthook = custom_exception_hook
def main(bar=1):
my_var = 'hello world' # variable specific to main()
return bar + 1
main(bar=None) # run main() to throw the exception
The overriding of sys.excepthook won't work from IDLE, but it works just fine from command line. Hope this will be helpful.
I have a parent class who has a try clause in it, and a child class overrides a method inside of the try clause. Normal exceptions can be caught when the child raises it. However, the keyboard interrupt exception cannot be caught. Moreover, it can be caught inside the child method, but not the parent method.
Please see the example code like the following, where the interrupt cannot be caught in bar1, but can be caught after bar2 changes it into an assertion error. I produced this problem in python 3.6 in both Linux and Windows.
import time
class foo:
def body(self):
pass
def run(self):
try:
self.body()
except Exception as ex:
print("caught exception:", str(ex))
class bar1(foo):
def body(self):
while(1):
print(1)
time.sleep(0.1)
class bar2(foo):
def body(self):
interrupted = False
while(1):
assert not interrupted, "assert not interrupted"
try:
print(1)
time.sleep(0.1)
except KeyboardInterrupt as ex:
print("received interrupt")
interrupted = True
Interrupting the run method of class bar1 gets
1
....
1
Traceback (most recent call last):
File "tmp.py", line 34, in <module>
b.run()
File "tmp.py", line 7, in run
self.body()
File "tmp.py", line 15, in body
time.sleep(0.1)
KeyboardInterrupt
However, interrupting bar2 gets
1
...
1
received interrupt
caught exception: assert not interrupted
I have searched over StackOverflow and found some problems regarding the keyboard interruption handling with threads and stdIO, but I did not found some problems like this.
I am using python's unittest and would like to write a test that starts a few threads and waits for them to finish. The threads execute a function that has some unittest assertions. If any of the assertions fail, I wish the test to, well, fail. This does not seem to be the case.
EDIT: Minimal runnable example (python3)
import unittest
import threading
class MyTests(unittest.TestCase):
def test_sample(self):
t = threading.Thread(target=lambda: self.fail())
t.start()
t.join()
if __name__ == '__main__':
unittest.main()
and the output is:
sh-4.3$ python main.py -v
test_sample (__main__.MyTests) ... Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib64/python2.7/threading.py", line 813, in __bootstrap_inner
self.run()
File "/usr/lib64/python2.7/threading.py", line 766, in run
self.__target(*self.__args, **self.__kwargs)
File "main.py", line 7, in <lambda>
t = threading.Thread(target=lambda: self.fail())
File "/usr/lib64/python2.7/unittest/case.py", line 450, in fail
raise self.failureException(msg)
AssertionError: None
ok
----------------------------------------------------------------------
Ran 1 test in 0.002s
OK
use a concurrent.futures.ThreadPoolExecutor or https://docs.python.org/3/library/threading.html#threading.excepthook to collect exceptions thrown in threads
import unittest
import threading
from concurrent import futures
class catch_threading_exception:
"""
https://docs.python.org/3/library/test.html#test.support.catch_threading_exception
Context manager catching threading.Thread exception using
threading.excepthook.
Attributes set when an exception is catched:
* exc_type
* exc_value
* exc_traceback
* thread
See threading.excepthook() documentation for these attributes.
These attributes are deleted at the context manager exit.
Usage:
with support.catch_threading_exception() as cm:
# code spawning a thread which raises an exception
...
# check the thread exception, use cm attributes:
# exc_type, exc_value, exc_traceback, thread
...
# exc_type, exc_value, exc_traceback, thread attributes of cm no longer
# exists at this point
# (to avoid reference cycles)
"""
def __init__(self):
self.exc_type = None
self.exc_value = None
self.exc_traceback = None
self.thread = None
self._old_hook = None
def _hook(self, args):
self.exc_type = args.exc_type
self.exc_value = args.exc_value
self.exc_traceback = args.exc_traceback
self.thread = args.thread
def __enter__(self):
self._old_hook = threading.excepthook
threading.excepthook = self._hook
return self
def __exit__(self, *exc_info):
threading.excepthook = self._old_hook
del self.exc_type
del self.exc_value
del self.exc_traceback
del self.thread
class MyTests(unittest.TestCase):
def test_tpe(self):
with futures.ThreadPoolExecutor() as pool:
pool.submit(self.fail).result()
def test_t_excepthook(self):
with catch_threading_exception() as cm:
t = threading.Thread(target=self.fail)
t.start()
t.join()
if cm.exc_value is not None:
raise cm.exc_value
if __name__ == '__main__':
unittest.main()
on pytest these are collected for you: https://docs.pytest.org/en/latest/how-to/failures.html?highlight=unraisable#warning-about-unraisable-exceptions-and-unhandled-thread-exceptions
Your test isn't failing for the same reason that this code will print "no exception"
import threading
def raise_err():
raise Exception()
try:
t = threading.Thread(target=raise_err)
t.start()
t.join()
print('no exception')
except:
print('caught exception')
When unittest runs your test function, it determines pass/fail by seeing if the code execution results in some exception. If the exception occurs inside the thread, there still is no exception in the main thread.
You could do something like this if you think you HAVE to get a pass/fail result from running something in a thread. But this is really not how unittest is designed to work, and there's probably a much easier way to do what you're trying to accomplish.
import threading
import unittest
def raise_err():
raise Exception()
def no_err():
return
class Runner():
def __init__(self):
self.threads = {}
self.thread_results = {}
def add(self, target, name):
self.threads[name] = threading.Thread(target = self.run, args = [target, name])
self.threads[name].start()
def run(self, target, name):
self.thread_results[name] = 'fail'
target()
self.thread_results[name] = 'pass'
def check_result(self, name):
self.threads[name].join()
assert(self.thread_results[name] == 'pass')
runner = Runner()
class MyTests(unittest.TestCase):
#classmethod
def setUpClass(cls):
runner.add(raise_err, 'test_raise_err')
runner.add(no_err, 'test_no_err')
def test_raise_err(self):
runner.check_result('test_raise_err')
def test_no_err(self):
runner.check_result('test_no_err')
if __name__ == '__main__':
unittest.main()
Python unittest assertions are communicated by exceptions, so you have to ensure that the exceptions end up in the main thread. So for a thread that means you have to run .join(), as that will throw the exception from the thread over into the main thread:
t = threading.Thread(target=lambda: self.assertTrue(False))
t.start()
t.join()
Also make sure that you don't have any try/except blocks that might eat up the exception before the unittest can register them.
Edit: self.fail() is indeed not communicated when called from a thread, even if .join() is present. Not sure what's up with that.
I want to access the traceback of a python programm running in a subprocess.
The documentation says:
Exceptions raised in the child process, before the new program has started to execute, will be re-raised in the parent. Additionally, the exception object will have one extra attribute called child_traceback, which is a string containing traceback information from the child’s point of view.
Contents of my_sub_program.py:
raise Exception("I am raised!")
Contents of my_main_program.py:
import sys
import subprocess
try:
subprocess.check_output([sys.executable, "my_sub_program.py"])
except Exception as e:
print e.child_traceback
If I run my_main_program.py, I get the following error:
Traceback (most recent call last):
File "my_main_program.py", line 6, in <module>
print e.child_traceback
AttributeError: 'CalledProcessError' object has no attribute 'child_traceback'
How can I access the traceback of the subprocess without modifying the subprocess program code? This means, I want to avoid adding a large try/except clause around my whole sub-program code, but rather handle error logging from my main program.
Edit: sys.executable should be replaceable with an interpreter differing from the one running the main program.
As you're starting another Python process, you can also try to use the multiprocessing Python module ; by sub-classing the Process class it is quite easy to get exceptions from the target function:
from multiprocessing import Process, Pipe
import traceback
import functools
class MyProcess(Process):
def __init__(self, *args, **kwargs):
Process.__init__(self, *args, **kwargs)
self._pconn, self._cconn = Pipe()
self._exception = None
def run(self):
try:
Process.run(self)
self._cconn.send(None)
except Exception as e:
tb = traceback.format_exc()
self._cconn.send((e, tb))
# raise e # You can still rise this exception if you need to
#property
def exception(self):
if self._pconn.poll():
self._exception = self._pconn.recv()
return self._exception
p = MyProcess(target=functools.partial(execfile, "my_sub_program.py"))
p.start()
p.join() #wait for sub-process to end
if p.exception:
error, traceback = p.exception
print 'you got', traceback
The trick is to have the target function executing the Python sub-program, this is done by using functools.partial.
I am using the unittest framework to automate integration tests of multi-threaded python code, external hardware and embedded C. Despite my blatant abuse of a unittesting framework for integration testing, it works really well. Except for one problem: I need the test to fail if an exception is raised from any of the spawned threads. Is this possible with the unittest framework?
A simple but non-workable solution would be to either a) refactor the code to avoid multi-threading or b) test each thread separately. I cannot do that because the code interacts asynchronously with the external hardware. I have also considered implementing some kind of message passing to forward the exceptions to the main unittest thread. This would require significant testing-related changes to the code being tested, and I want to avoid that.
Time for an example. Can I modify the test script below to fail on the exception raised in my_thread without modifying the x.ExceptionRaiser class?
import unittest
import x
class Test(unittest.TestCase):
def test_x(self):
my_thread = x.ExceptionRaiser()
# Test case should fail when thread is started and raises
# an exception.
my_thread.start()
my_thread.join()
if __name__ == '__main__':
unittest.main()
At first, sys.excepthook looked like a solution. It is a global hook which is called every time an uncaught exception is thrown.
Unfortunately, this does not work. Why? well threading wraps your run function in code which prints the lovely tracebacks you see on screen (noticed how it always tells you Exception in thread {Name of your thread here}? this is how it's done).
Starting with Python 3.8, there is a function which you can override to make this work: threading.excepthook
... threading.excepthook() can be overridden to control how uncaught exceptions raised by Thread.run() are handled
So what do we do? Replace this function with our logic, and voilà:
For python >= 3.8
import traceback
import threading
import os
class GlobalExceptionWatcher(object):
def _store_excepthook(self, args):
'''
Uses as an exception handlers which stores any uncaught exceptions.
'''
self.__org_hook(args)
formated_exc = traceback.format_exception(args.exc_type, args.exc_value, args.exc_traceback)
self._exceptions.append('\n'.join(formated_exc))
return formated_exc
def __enter__(self):
'''
Register us to the hook.
'''
self._exceptions = []
self.__org_hook = threading.excepthook
threading.excepthook = self._store_excepthook
def __exit__(self, type, value, traceback):
'''
Remove us from the hook, assure no exception were thrown.
'''
threading.excepthook = self.__org_hook
if len(self._exceptions) != 0:
tracebacks = os.linesep.join(self._exceptions)
raise Exception(f'Exceptions in other threads: {tracebacks}')
For older versions of Python, this is a bit more complicated.
Long story short, it appears that the threading nodule has an undocumented import which does something along the lines of:
threading._format_exc = traceback.format_exc
Not very surprisingly, this function is only called when an exception is thrown from a thread's run function.
So for python <= 3.7
import threading
import os
class GlobalExceptionWatcher(object):
def _store_excepthook(self):
'''
Uses as an exception handlers which stores any uncaught exceptions.
'''
formated_exc = self.__org_hook()
self._exceptions.append(formated_exc)
return formated_exc
def __enter__(self):
'''
Register us to the hook.
'''
self._exceptions = []
self.__org_hook = threading._format_exc
threading._format_exc = self._store_excepthook
def __exit__(self, type, value, traceback):
'''
Remove us from the hook, assure no exception were thrown.
'''
threading._format_exc = self.__org_hook
if len(self._exceptions) != 0:
tracebacks = os.linesep.join(self._exceptions)
raise Exception('Exceptions in other threads: %s' % tracebacks)
Usage:
my_thread = x.ExceptionRaiser()
# will fail when thread is started and raises an exception.
with GlobalExceptionWatcher():
my_thread.start()
my_thread.join()
You still need to join yourself, but upon exit, the with-statement's context manager will check for any exception thrown in other threads, and will raise an exception appropriately.
THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED
This is an undocumented, sort-of-horrible hack. I tested it on linux and windows, and it seems to work. Use it at your own risk.
I've come across this problem myself, and the only solution I've been able to come up with is subclassing Thread to include an attribute for whether or not it terminates without an uncaught exception:
from threading import Thread
class ErrThread(Thread):
"""
A subclass of Thread that will log store exceptions if the thread does
not exit normally
"""
def run(self):
try:
Thread.run(self)
except Exception as self.err:
pass
else:
self.err = None
class TaskQueue(object):
"""
A utility class to run ErrThread objects in parallel and raises and exception
in the event that *any* of them fail.
"""
def __init__(self, *tasks):
self.threads = []
for t in tasks:
try:
self.threads.append(ErrThread(**t)) ## passing in a dict of target and args
except TypeError:
self.threads.append(ErrThread(target=t))
def run(self):
for t in self.threads:
t.start()
for t in self.threads:
t.join()
if t.err:
raise Exception('Thread %s failed with error: %s' % (t.name, t.err))
I've been using the accepted answer above for a while now, but since Python 3.8 the solution doesn't work anymore because the threading module doesn't have this _format_exc import anymore.
On the other hand the threading module now has a nice way to register custom except hooks in Python 3.8 so here is a simple solution to run unit tests which assert that some exceptions are raised inside threads:
def test_in_thread():
import threading
exceptions_caught_in_threads = {}
def custom_excepthook(args):
thread_name = args.thread.name
exceptions_caught_in_threads[thread_name] = {
'thread': args.thread,
'exception': {
'type': args.exc_type,
'value': args.exc_value,
'traceback': args.exc_traceback
}
}
# Registering our custom excepthook to catch the exception in the threads
threading.excepthook = custom_excepthook
# dummy function that raises an exception
def my_function():
raise Exception('My Exception')
# running the funciton in a thread
thread_1 = threading.Thread(name='thread_1', target=my_function, args=())
thread_1.start()
thread_1.join()
assert 'thread_1' in exceptions_caught_in_threads # there was an exception in thread 1
assert exceptions_caught_in_threads['thread_1']['exception']['type'] == Exception
assert str(exceptions_caught_in_threads['thread_1']['exception']['value']) == 'My Exception'