Develop Reference

How to define a global error handler in gRPC python

Im trying to catch any exception that is raised in any servicer so I can make sure that I only propagate known exceptions and not unexpected ones like ValueError, TypeError etc.
I'd like to be able to catch any raised error, and format them or convert them to other errors to better control the info that is exposed.
I don't want to have to enclose every servicer method with try/except.
I've tried with an interceptor, but im not able to catch the errors there.
Is there a way of specifying an error handler for the grpc Server? like what you do with flask or any other http server?

gRPC Python currently don't support server-side global error handler. The interceptor won't execute the server handler inside the intercept_service function, so there is no way to try/except.
Also, I found the gRPC Python server interceptor implementation is different from what they proposed original at L13-Python-Interceptors.md#server-interceptors. If the implementation stick to the original design, we can use interceptor as global error handler easily with handler and request/request_iterator.
# Current Implementation
intercept_service(self, continuation, handler_call_details)
# Original Design
intercept_unary_unary_handler(self, handler, method, request, servicer_context)
intercept_unary_stream_handler(self, handler, method, request, servicer_context)
intercept_stream_unary_handler(self, handler, method, request_iterator, servicer_context)
intercept_stream_stream_handler(self, handler, method, request_iterator, servicer_context)
Please submit a feature request issue to https://github.com/grpc/grpc/issues.

Maybe this will help you :)
def _wrap_rpc_behavior(handler, fn):
if handler is None:
return None
if handler.request_streaming and handler.response_streaming:
behavior_fn = handler.stream_stream
handler_factory = grpc.stream_stream_rpc_method_handler
elif handler.request_streaming and not handler.response_streaming:
behavior_fn = handler.stream_unary
handler_factory = grpc.stream_unary_rpc_method_handler
elif not handler.request_streaming and handler.response_streaming:
behavior_fn = handler.unary_stream
handler_factory = grpc.unary_stream_rpc_method_handler
else:
behavior_fn = handler.unary_unary
handler_factory = grpc.unary_unary_rpc_method_handler
return handler_factory(fn(behavior_fn,
handler.request_streaming,
handler.response_streaming),
request_deserializer=handler.request_deserializer,
response_serializer=handler.response_serializer)
class TracebackLoggerInterceptor(grpc.ServerInterceptor):
def intercept_service(self, continuation, handler_call_details):
def latency_wrapper(behavior, request_streaming, response_streaming):
def new_behavior(request_or_iterator, servicer_context):
try:
return behavior(request_or_iterator, servicer_context)
except Exception as err:
logger.exception(err, exc_info=True)
return new_behavior
return _wrap_rpc_behavior(continuation(handler_call_details), latency_wrapper)

As some of the previous comments suggested, I tried the meta-class approach which works quite well.
Attached is a simple example to demonstrate how to intercept the grpc calls.
You could extend this by providing the metaclass a list of decorators which you could apply on each function.
Also, it would be wise to be more selective regarding the methods you apply the wrapper to. A good option would be to list the methods of the autogenerated base class and only wrap those.
from types import FunctionType
from functools import wraps
def wrapper(method):
#wraps(method)
def wrapped(*args, **kwargs):
# do stuff here
return method(*args, **kwargs)
return wrapped
class ServicerMiddlewareClass(type):
def __new__(meta, classname, bases, class_dict):
new_class_dict = {}
for attribute_name, attribute in class_dict.items():
if isinstance(attribute, FunctionType):
# replace it with a wrapped version
attribute = wrapper(attribute)
new_class_dict[attribute_name] = attribute
return type.__new__(meta, classname, bases, new_class_dict)
# In order to use
class MyGrpcService(grpc.MyGrpcServicer, metaclass=ServicerMiddlewareClass):
...

Exception handled surprisingly in Pyside slots

Problem: When exceptions are raised in slots, invoked by signals, they do not seem to propagate as usual through Pythons call stack. In the example code below invoking:
on_raise_without_signal(): Will handle the exception as expected.
on_raise_with_signal(): Will print the exception and then unexpectedly print the success message from the else block.
Question: What is the reason behind the exception being handled surprisingly when raised in a slot? Is it some implementation detail/limitation of the PySide Qt wrapping of signals/slots? Is there something to read about in the docs?
PS: I initially came across that topic when I got surprising results upon using try/except/else/finally when implementing a QAbstractTableModels virtual methods insertRows() and removeRows().
# -*- coding: utf-8 -*-
"""Testing exception handling in PySide slots."""
from __future__ import unicode_literals, print_function, division
import logging
import sys
from PySide import QtCore
from PySide import QtGui
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
class ExceptionTestWidget(QtGui.QWidget):
raise_exception = QtCore.Signal()
def __init__(self, *args, **kwargs):
super(ExceptionTestWidget, self).__init__(*args, **kwargs)
self.raise_exception.connect(self.slot_raise_exception)
layout = QtGui.QVBoxLayout()
self.setLayout(layout)
# button to invoke handler that handles raised exception as expected
btn_raise_without_signal = QtGui.QPushButton("Raise without signal")
btn_raise_without_signal.clicked.connect(self.on_raise_without_signal)
layout.addWidget(btn_raise_without_signal)
# button to invoke handler that handles raised exception via signal unexpectedly
btn_raise_with_signal = QtGui.QPushButton("Raise with signal")
btn_raise_with_signal.clicked.connect(self.on_raise_with_signal)
layout.addWidget(btn_raise_with_signal)
def slot_raise_exception(self):
raise ValueError("ValueError on purpose")
def on_raise_without_signal(self):
"""Call function that raises exception directly."""
try:
self.slot_raise_exception()
except ValueError as exception_instance:
logger.error("{}".format(exception_instance))
else:
logger.info("on_raise_without_signal() executed successfully")
def on_raise_with_signal(self):
"""Call slot that raises exception via signal."""
try:
self.raise_exception.emit()
except ValueError as exception_instance:
logger.error("{}".format(exception_instance))
else:
logger.info("on_raise_with_signal() executed successfully")
if (__name__ == "__main__"):
application = QtGui.QApplication(sys.argv)
widget = ExceptionTestWidget()
widget.show()
sys.exit(application.exec_())

As you've already noted in your question, the real issue here is the treatment of unhandled exceptions raised in python code executed from C++. So this is not only about signals: it also affects reimplemented virtual methods as well.
In PySide, PyQt4, and all PyQt5 versions up to 5.5, the default behaviour is to automatically catch the error on the C++ side and dump a traceback to stderr. Normally, a python script would also automatically terminate after this. But that is not what happens here. Instead, the PySide/PyQt script just carries on regardless, and many people quite rightly regard this as a bug (or at least a misfeature). In PyQt-5.5, this behaviour has now been changed so that qFatal() is also called on the C++ side, and the program will abort like a normal python script would. (I don't know what the current situation is with PySide2, though).
So - what should be done about all this? The best solution for all versions of PySide and PyQt is to install an exception hook - because it will always take precedence over the default behaviour (whatever that may be). Any unhandled exception raised by a signal, virtual method or other python code will firstly invoke sys.excepthook, allowing you to fully customise the behaviour in whatever way you like.
In your example script, this could simply mean adding something like this:
def excepthook(cls, exception, traceback):
print('calling excepthook...')
logger.error("{}".format(exception))
sys.excepthook = excepthook
and now the exception raised by on_raise_with_signal can be handled in the same way as all other unhandled exceptions.
Of course, this does imply that best practice for most PySide/PyQt applications is to use largely centralised exception handling. This often includes showing some kind of crash-dialog where the user can report unexpected errors.

According to the Qt5 docs you need to handle exceptions within the slot being invoked.
Throwing an exception from a slot invoked by Qt's signal-slot connection mechanism is considered undefined behaviour, unless it is handled within the slot
State state;
StateListener stateListener;
// OK; the exception is handled before it leaves the slot.
QObject::connect(&state, SIGNAL(stateChanged()), &stateListener, SLOT(throwHandledException()));
// Undefined behaviour; upon invocation of the slot, the exception will be propagated to the
// point of emission, unwinding the stack of the Qt code (which is not guaranteed to be exception safe).
QObject::connect(&state, SIGNAL(stateChanged()), &stateListener, SLOT(throwUnhandledException()));
If the slot was invoked directly, like a regular function call,
exceptions may be used. This is because the connection mechanism is
bypassed when invoking slots directly
In the first case you call slot_raise_exception() directly, so this is fine.
In the second case you are invoking it via the raise_exception signal, so the exception will only propagate up to the point where slot_raise_exception() is called. You need to place the try/except/else inside slot_raise_exception() for the exception to be handled correctly.

Thanks for answering guys. I found ekhumoros answer particularly useful to understand where the exceptions are handled and because of the idea to utilize sys.excepthook.
I mocked up a quick solution via context manager to temporarily extend the current sys.excepthook to record any exception in the realm of "C++ calling Python" (as it seems to happen when slots are invoked by signals or virtual methods) and possibly re-raise upon exiting the context to achieve expected control flow in try/except/else/finally blocks.
The context manager allows on_raise_with_signal to maintain the same control flow as on_raise_without_signal with the surrounding try/except/else/finally block.
# -*- coding: utf-8 -*-
"""Testing exception handling in PySide slots."""
from __future__ import unicode_literals, print_function, division
import logging
import sys
from functools import wraps
from PySide import QtCore
from PySide import QtGui
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
class ExceptionHook(object):
def extend_exception_hook(self, exception_hook):
"""Decorate sys.excepthook to store a record on the context manager
instance that might be used upon leaving the context.
"""
#wraps(exception_hook)
def wrapped_exception_hook(exc_type, exc_val, exc_tb):
self.exc_val = exc_val
return exception_hook(exc_type, exc_val, exc_tb)
return wrapped_exception_hook
def __enter__(self):
"""Temporary extend current exception hook."""
self.current_exception_hook = sys.excepthook
sys.excepthook = self.extend_exception_hook(sys.excepthook)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Reset current exception hook and re-raise in Python call stack after
we have left the realm of `C++ calling Python`.
"""
sys.excepthook = self.current_exception_hook
try:
exception_type = type(self.exc_val)
except AttributeError:
pass
else:
msg = "{}".format(self.exc_val)
raise exception_type(msg)
class ExceptionTestWidget(QtGui.QWidget):
raise_exception = QtCore.Signal()
def __init__(self, *args, **kwargs):
super(ExceptionTestWidget, self).__init__(*args, **kwargs)
self.raise_exception.connect(self.slot_raise_exception)
layout = QtGui.QVBoxLayout()
self.setLayout(layout)
# button to invoke handler that handles raised exception as expected
btn_raise_without_signal = QtGui.QPushButton("Raise without signal")
btn_raise_without_signal.clicked.connect(self.on_raise_without_signal)
layout.addWidget(btn_raise_without_signal)
# button to invoke handler that handles raised exception via signal unexpectedly
btn_raise_with_signal = QtGui.QPushButton("Raise with signal")
btn_raise_with_signal.clicked.connect(self.on_raise_with_signal)
layout.addWidget(btn_raise_with_signal)
def slot_raise_exception(self):
raise ValueError("ValueError on purpose")
def on_raise_without_signal(self):
"""Call function that raises exception directly."""
try:
self.slot_raise_exception()
except ValueError as exception_instance:
logger.error("{}".format(exception_instance))
else:
logger.info("on_raise_without_signal() executed successfully")
def on_raise_with_signal(self):
"""Call slot that raises exception via signal."""
try:
with ExceptionHook() as exception_hook:
self.raise_exception.emit()
except ValueError as exception_instance:
logger.error("{}".format(exception_instance))
else:
logger.info("on_raise_with_signal() executed successfully")
if (__name__ == "__main__"):
application = QtGui.QApplication(sys.argv)
widget = ExceptionTestWidget()
widget.show()
sys.exit(application.exec_())

This way of handling expcetions is not a surprise taking into consideration that the Signal/Slot architecture proposes a loosely coupled intercation between the signals and the slots. This means that the signal should not be expecting anything to happen inside the slots.
Although timmwagener's solution was pretty clever, it should be used with caution. Probably the problem is not with how Exceptions are handled between Qt Connections, but that the signal/slot architecture is not ideal for your application. Also, that solution would not work if a slot from a different thread is connected, or a Qt.QueuedConnection is used.
A good way of tackling the problem of errors raised in slots is to determine that at the connection and not the emitting. Then the erros can be handled in a loosely coupled way.
class ExceptionTestWidget(QtGui.QWidget):
error = QtCore.Signal(object)
def abort_execution():
pass
def error_handler(self, err):
self.error.emit(error)
self.abort_execution()
(...)
def connect_with_async_error_handler(sig, slot, error_handler, *args,
conn_type=None, **kwargs):
#functools.wraps(slot)
def slot_with_error_handler(*args):
try:
slot(*args)
except Exception as err:
error_handler(err)
if conn_type is not None:
sig.connect(slot_with_error_handler, conn_type)
else:
sig.connect(slot_with_error_handler)
This way, we would be complying to the requirements in the Qt5 docs, stating that you need to handle exceptions within the slot being invoked.
Throwing an exception from a slot invoked by Qt's signal-slot
connection mechanism is considered undefined behaviour, unless it is
handled within the slot
PS: This is just a sugestion based on a very small overview of your use case. There is no right/wrong way of solving this, I just wanted to bring out a different point of view : )

Do the same change (try-except) on multiple lines - python

There are ways how programmer can make programming and refactoring easier and more simple, python is very good in this area.
I'm curious whether is there a more elegant way to solve my problem than brute-force writing the same code multiple times again and again.
Situation:
I'm writing a code. There are many equal methods calling with different arguments sequentially.
For example - I have this code:
...
...
my_method(1)
my_method(2)
my_method(3)
my_method(4)
...
my_method(10)
...
So I have this code written, everything works fine but suddenly I find out that I need to make a log file so I have to put try-except on everyone of this methods so the code will look like this:
...
...
try:
my_method(3)
except Exception as e:
print_to_file(log.txt,str(e))
...
...
try:
my_method(8)
except Exception as e:
print_to_file(log.txt,str(e))
...
...
Do I have a better option than changing every my_method(x) calling and putting it into try-except clause? I know that it is a mistake of the programmer who had to think about it at the beginning but these situations happens.
EDIT: According to the answer - the code above is the simple example. In real code there are no int arguments given but dates where there is no logic there so I can't put it into the loop. Assume that the arguments can't be generated.

If you're using the logger supplied by python, you can redirect exception output to the log as opposed to have to put a ton of try blocks everywhere:
import os, sys
import logging
logger = logging.getLogger(__name__)
handler = logging.StreamHandler(stream=sys.stdout)
logger.addHandler(handler)
def handle_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
logger.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
sys.excepthook = handle_exception
if __name__ == "__main__":
raise RuntimeError("Test unhandled")
Now is an exception is thrown, you won't need a try block, it will be written to the log regardless
ref

You can take advantage of the fact that a function, in python, is totally an object, and write a function that takes in another function, runs it, and logs any exceptions
def sloppyRun(func, *args, **kwargs):
"""Runs a function, catching all exceptions
and writing them to a log file."""
try:
return func(*args, **kwargs) #running function here
except:
logging.exception(func.__name__ + str(args) + str(kwargs))
#incidentally, the logging module is wonderful. I'd recommend using it.
#It'll even write the traceback to a file.
And then you can write something like
sloppyRun(my_method, 8) #note the lack of parens for my_method

You could have like a context manager or a decorator to log what you need, when you need to. if you intend to always log an exception when you use that function, I would suggest going the simple decorator rule or even a try and except inside that function. If it is functions not in your code or you dont want them to always log,then I would used a context manager (called as with ..:)
A context manager example code
import functools
class LoggerContext():
def __enter__(self):
# function that is called on enter of the with context
# we dont need this
pass
def __exit__(self, type, value, traceback):
# If there was an exception, it will be passed to the
# exit function.
# type = type of exception
# value = the string arg of the exception
# traceback object for you to extract the traceback if you need to
if traceback:
# do something with exception like log it etc
print(type, value, traceback)
# If the return value of the exit function is not True, python
# interpreter re-raises the exception. We dont want to re-raise
# the exception
return True
def __call__(self, f):
# this is just to make a context manager a decorator
# so that you could use the #context on a function
#functools.wraps(f)
def decorated(*args, **kwds):
with self:
return f(*args, **kwds)
return decorated
#LoggerContext()
def myMethod(test):
raise FileNotFoundError(test)
def myMethod2(test):
raise TypeError(test)
myMethod('asdf')
with LoggerContext():
myMethod2('asdf')
A simple decorator example:
import functools
def LoggerDecorator(f):
#functools.wraps(f)
def decorated(*args, **kwds):
try:
return f(*args, **kwds)
except Exception as e:
# do something with exception
print('Exception:', e)
return decorated
#LoggerDecorator
def myMethod3(test):
raise IOError(test)
myMethod3('asdf')

python custom exception handler for a class

I want to build my application with a redis cache. but maybe redis is not available all the time in our case,
so I hope, it redis works well, we use it. if it can't work, just logging and ignore it this time.
for example:
try:
conn.sadd('s', *array)
except :
...
since there are many place I will run some conn.{rediscommand}, I don't like to use try/except every place.
so the solution maybe :
class softcache(redis.StrictRedis):
def sadd(key, *p):
try:
super(redis.StrictRedis, self).sadd(key, p)
except:
..
but since redis have many commands, I have to warp them one by one.
is it possible to custom a exception handler for a class to handle all the exceptions which come from this class ?

Silencing per default all exceptions is probably the worst thing you can do.
Anyway, for your problem you can write a generic wrapper that just redirects to the connection object.
class ReddisWrapper(object):
conn = conn # Here your reddis object
def __getattr__(self, attr):
def wrapper(*args, **kwargs):
# Get the real reddis function
fn = getattr(self.conn, attr)
# Execute the function catching exceptions
try:
return fn(*args, **kwargs)
# Specify here the exceptions you expect
except:
log(...)
return wrapper
And then you would call like this:
reddis = ReddisWrapper()
reddis.do_something(4)
This has not been tested, and will only work with methods. For properties you should catch the non callable exception and react properly.

Is it always the same Exception?
If so you could write a custom, Exception catching and logging decorator.
Something like the following:
def exception_catcher(fn):
try:
fn()
except Exception as e:
log(e)
Then just use it around your code:
#exception_catcher
sadd('s', *array)
The comment and link to Exceptions for the whole class suggested by #idanshmu will offer more detailed handling of different Exceptions per method.

Make Python unittest fail on exception from any thread

I am using the unittest framework to automate integration tests of multi-threaded python code, external hardware and embedded C. Despite my blatant abuse of a unittesting framework for integration testing, it works really well. Except for one problem: I need the test to fail if an exception is raised from any of the spawned threads. Is this possible with the unittest framework?
A simple but non-workable solution would be to either a) refactor the code to avoid multi-threading or b) test each thread separately. I cannot do that because the code interacts asynchronously with the external hardware. I have also considered implementing some kind of message passing to forward the exceptions to the main unittest thread. This would require significant testing-related changes to the code being tested, and I want to avoid that.
Time for an example. Can I modify the test script below to fail on the exception raised in my_thread without modifying the x.ExceptionRaiser class?
import unittest
import x
class Test(unittest.TestCase):
def test_x(self):
my_thread = x.ExceptionRaiser()
# Test case should fail when thread is started and raises
# an exception.
my_thread.start()
my_thread.join()
if __name__ == '__main__':
unittest.main()

At first, sys.excepthook looked like a solution. It is a global hook which is called every time an uncaught exception is thrown.
Unfortunately, this does not work. Why? well threading wraps your run function in code which prints the lovely tracebacks you see on screen (noticed how it always tells you Exception in thread {Name of your thread here}? this is how it's done).
Starting with Python 3.8, there is a function which you can override to make this work: threading.excepthook
... threading.excepthook() can be overridden to control how uncaught exceptions raised by Thread.run() are handled
So what do we do? Replace this function with our logic, and voilà:
For python >= 3.8
import traceback
import threading
import os
class GlobalExceptionWatcher(object):
def _store_excepthook(self, args):
'''
Uses as an exception handlers which stores any uncaught exceptions.
'''
self.__org_hook(args)
formated_exc = traceback.format_exception(args.exc_type, args.exc_value, args.exc_traceback)
self._exceptions.append('\n'.join(formated_exc))
return formated_exc
def __enter__(self):
'''
Register us to the hook.
'''
self._exceptions = []
self.__org_hook = threading.excepthook
threading.excepthook = self._store_excepthook
def __exit__(self, type, value, traceback):
'''
Remove us from the hook, assure no exception were thrown.
'''
threading.excepthook = self.__org_hook
if len(self._exceptions) != 0:
tracebacks = os.linesep.join(self._exceptions)
raise Exception(f'Exceptions in other threads: {tracebacks}')
For older versions of Python, this is a bit more complicated.
Long story short, it appears that the threading nodule has an undocumented import which does something along the lines of:
threading._format_exc = traceback.format_exc
Not very surprisingly, this function is only called when an exception is thrown from a thread's run function.
So for python <= 3.7
import threading
import os
class GlobalExceptionWatcher(object):
def _store_excepthook(self):
'''
Uses as an exception handlers which stores any uncaught exceptions.
'''
formated_exc = self.__org_hook()
self._exceptions.append(formated_exc)
return formated_exc
def __enter__(self):
'''
Register us to the hook.
'''
self._exceptions = []
self.__org_hook = threading._format_exc
threading._format_exc = self._store_excepthook
def __exit__(self, type, value, traceback):
'''
Remove us from the hook, assure no exception were thrown.
'''
threading._format_exc = self.__org_hook
if len(self._exceptions) != 0:
tracebacks = os.linesep.join(self._exceptions)
raise Exception('Exceptions in other threads: %s' % tracebacks)
Usage:
my_thread = x.ExceptionRaiser()
# will fail when thread is started and raises an exception.
with GlobalExceptionWatcher():
my_thread.start()
my_thread.join()
You still need to join yourself, but upon exit, the with-statement's context manager will check for any exception thrown in other threads, and will raise an exception appropriately.
THE CODE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED
This is an undocumented, sort-of-horrible hack. I tested it on linux and windows, and it seems to work. Use it at your own risk.

I've come across this problem myself, and the only solution I've been able to come up with is subclassing Thread to include an attribute for whether or not it terminates without an uncaught exception:
from threading import Thread
class ErrThread(Thread):
"""
A subclass of Thread that will log store exceptions if the thread does
not exit normally
"""
def run(self):
try:
Thread.run(self)
except Exception as self.err:
pass
else:
self.err = None
class TaskQueue(object):
"""
A utility class to run ErrThread objects in parallel and raises and exception
in the event that *any* of them fail.
"""
def __init__(self, *tasks):
self.threads = []
for t in tasks:
try:
self.threads.append(ErrThread(**t)) ## passing in a dict of target and args
except TypeError:
self.threads.append(ErrThread(target=t))
def run(self):
for t in self.threads:
t.start()
for t in self.threads:
t.join()
if t.err:
raise Exception('Thread %s failed with error: %s' % (t.name, t.err))

I've been using the accepted answer above for a while now, but since Python 3.8 the solution doesn't work anymore because the threading module doesn't have this _format_exc import anymore.
On the other hand the threading module now has a nice way to register custom except hooks in Python 3.8 so here is a simple solution to run unit tests which assert that some exceptions are raised inside threads:
def test_in_thread():
import threading
exceptions_caught_in_threads = {}
def custom_excepthook(args):
thread_name = args.thread.name
exceptions_caught_in_threads[thread_name] = {
'thread': args.thread,
'exception': {
'type': args.exc_type,
'value': args.exc_value,
'traceback': args.exc_traceback
}
}
# Registering our custom excepthook to catch the exception in the threads
threading.excepthook = custom_excepthook
# dummy function that raises an exception
def my_function():
raise Exception('My Exception')
# running the funciton in a thread
thread_1 = threading.Thread(name='thread_1', target=my_function, args=())
thread_1.start()
thread_1.join()
assert 'thread_1' in exceptions_caught_in_threads # there was an exception in thread 1
assert exceptions_caught_in_threads['thread_1']['exception']['type'] == Exception
assert str(exceptions_caught_in_threads['thread_1']['exception']['value']) == 'My Exception'