This question already has answers here:
Find Out If a Function has been Called
(6 answers)
Closed 5 years ago.
I want to check if a function was called inside a Python script.
My current solution is the following:
def _print_error(msg):
print(msg)
print("Default error message.")
_print_error.was_called = True
_print_error.was_called = False
I can check if _print_error was called by checking _print_error.was_called.
Another possible way to do this is to set a class and define a static variable, that is changed whenever the class is instanced.The __init__ can print the messages and change the was_called variable.
Is there a better way of achieving the same results?
Use a decorator to set the flag:
from functools import wraps
def flag_when_called(func):
func.was_called = False
#wraps(func)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
func.was_called = True
return result
return wrapper
Note that I only set the flag when the call successfully completes (e.g. if an exception is raised the flag is not set). You can swap the order (and directly return func(*args, **kwargs) if you want to set the flag even if an exception was raised.
Just apply that to the function:
#flag_when_called
def _print_error(msg):
print(msg)
print("Default error message.")
It's okay to do what you do. If you do it often you can write a decorator for this:
import functools
def trackcalls(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
wrapper.was_called = True
return func(*args, **kwargs)
wrapper.was_called = False
return wrapper
#trackcalls
def _print_error(msg):
print(msg)
print("Default error message.")
_print_error()
# and then check it:
if _print_error.was_called:
print("Alarm!")
Related
I want to use a decorator (composer) that recevices as parameter n number of decorators, this decorators will be used to decorate a function. Also I want to pass some parameters from two origins, a parameter named "SKIP" in the composer and another parameter named "parameter" sent by the parameter_sender decorator. Here's what I tried:
def compose(*decorators, SKIP=None):
def something(func):
#wraps(func)
def func_wrap(parameter = None, **kwargs):
try:
if SKIP:
print("I'm here")
return func(parameter = parameter,**kwargs)
else:
for decorator in reversed(decorators):
func = decorator(func, parameter = parameter,**kwargs) # --------- This line is providing the error ------------------
return func
raise exception
except Exception as e:
print(e)
raise exception
return func_wrap
return something
And here is an example of where do I want to use it. In this example I want to SKIP the composing of all the decorators if the variable SKIP is true.
#application.route("/function/<id_something>", methods=['GET'])
#parameter_sender
#compose(decorator_1,decorator_2, SKIP=True)
def function (id_something, **kwargs):
try:
#TODO:
return jsonify("ok")
except Exception as e:
print(e)
But i've got an error that says this:
>>I'm here
>>local variable 'func' referenced before assignment
Even when the if statement is working. PD: It works without the line indicated in the composer.
The following code should do the thing.
You were trying to set a value for a variable from outer scope. In my example I used separate temp variable composition.
def compose(*decorators, SKIP=None):
def something(func):
#wraps(func)
def func_wrap(*args, **kwargs):
try:
if SKIP:
print("I'm here")
return func(*args, **kwargs)
else:
composition = func
for decorator in reversed(decorators):
composition = decorator(composition)
return composition(*args, **kwargs)
except Exception as e:
print(e)
raise
return func_wrap
return something
The below example is taken from python cookbook 3rd edition section 9.5.
I placed break points at each line to understand the flow of execution . Below is the code sample, its output and the questions I have . I have tried to explain my question , let me know if you need further info.
from functools import wraps, partial
import logging
# Utility decorator to attach a function as an attribute of obj
def attach_wrapper(obj, func=None):
if func is None:
return partial(attach_wrapper, obj)
setattr(obj, func.__name__, func)
return func
def logged(level, name=None, message=None):
def decorate(func):
logname = name if name else func.__module__
log = logging.getLogger(logname)
logmsg = message if message else func.__name__
#wraps(func)
def wrapper(*args, **kwargs):
log.log(level, logmsg)
return func(*args, **kwargs)
#attach_wrapper(wrapper)
def set_message(newmsg):
nonlocal logmsg
logmsg = newmsg
return wrapper
return decorate
# Example use
#logged(logging.DEBUG)
def add(x, y):
return x + y
logging.basicConfig(level=logging.DEBUG)
add.set_message('Add called')
#add.set_level(logging.WARNING)
print (add(2, 3))
output is
DEBUG:__main__:Add called
5
I understand the concept of decorators, but this is confusing a little.
scenario 1. When the following line is debugged #logged(logging.DEBUG) , we get
decorate = .decorate at 0x000000000< memoryaddress >>
Question : why would the control go back to execute the function " def decorate" ? Is it because the "decorate" function is on the top of the stack ?
scenario 2 :When executing #attach_wrapper(wrapper) , the control goes to execute attach_wrapper(obj, func=None) and partial function returns
func =
question : why would the control go back to execute def attach_wrapper(obj, func=None):
and how would this time the value for func is *.decorate..set_message at 0x000000000 >
being passed to the attach_wrapper ?
Scenario 1
This:
#logged(logging.DEBUG)
def add(x, y):
....
is the same as this:
def add(x, y):
....
add = logged(logging.DEBUG)(add)
Note that there are two calls there: first logged(logging.DEBUG) returns decorate and then decorate(add) is called.
Scenario 2
Same as in Scenario 1, this:
#attach_wrapper(wrapper)
def set_message(newmsg):
...
is the same as this:
def set_message(newmsg):
...
set_message = attach_wrapper(wrapper)(set_message)
Again, there are two calls: first attach_wrapper(wrapper) returns the partial object and then partial(set_message) is called.
In other words...
logged and attach_wrapper are not decorators. Those are functions which return decorators. That is why two calls are made: one to the function which returns the decorator and another the the decorator itself.
I'm trying to write a decorator to repeat an erroring function N times with increasingly sleeping times in between. This is my attempt so far:
def exponential_backoff(seconds=10, attempts=10):
def our_decorator(func):
def function_wrapper(*args, **kwargs):
for s in range(0, seconds*attempts, attempts):
sleep(s)
try:
return func(*args, **kwargs)
except Exception as e:
print(e)
return function_wrapper
return our_decorator
#exponential_backoff
def test():
for a in range(100):
if a - random.randint(0,1) == 0:
print('success count: {}'.format(a))
pass
else:
print('error count {}'.format(a))
'a' + 1
test()
I keep getting the error:
TypeError: our_decorator() missing 1 required positional argument: 'func'
Understand what decorator is:
#exponential_backoff
def test():
pass
equals to:
def test():
pass
test = exponential_backoff(test)
In this case, test is def our_decorator(func):. That's why you get TypeError when calling test().
So further:
#exponential_backoff()
def test():
pass
equals to:
def test():
pass
test = exponential_backoff()(test)
In this case, now test is what you need.
Further, functools.wraps helps you to copy all properties of original function to decorated function. Such as function's name or docstring:
from functools import wraps
def exponential_backoff(func):
# #wraps(func)
def function_wrapper(*args, **kwargs):
pass
return function_wrapper
#exponential_backoff
def test():
pass
print(test) # <function exponential_backoff.<locals>.function_wrapper at 0x7fcc343a4268>
# uncomment `#wraps(func)` line:
print(test) # <function test at 0x7fcc343a4400>
You should be using:
#exponential_backoff()
def test():
...
The overall decorator is not designed to have arguments be optional, so you must provide () when using it.
If want an example of how to make decorator allow argument list be optional, see:
https://wrapt.readthedocs.io/en/latest/decorators.html#decorators-with-optional-arguments
You might also consider using the wrapt package to make your decorators easier and more robust.
Either you go for the solution provided by #Graham Dumpleton or you can just modify your decorator like so:
from functools import wraps, partial
def exponential_backoff(func=None, seconds=10, attempts=10):
if func is None:
return partial(exponential_backoff, seconds=seconds, attempts=attempts)
#wraps(func)
def function_wrapper(*args, **kwargs):
for s in range(0, seconds*attempts, attempts):
sleep(s)
try:
return func(*args, **kwargs)
except Exception as e:
print(e)
return function_wrapper
#exponential_backoff
def test():
for a in range(100):
if a - random.randint(0,1) == 0:
print('success count: {}'.format(a))
pass
else:
print('error count {}'.format(a))
'a' + 1
test()
EDIT
My answer was not entirely correct, please see #GrahamDumpleton's answer which shows how to make my attempt of a solution viable (i.e. this link). Fixed it now, thank you #GrahamDumpleton !
I'm trying to wrap my mapper/reducer functions with something like:
def log_exceptions_to_sentry(sentry_id, raise_exception):
def decorator(fn):
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception, e:
client = Client(sentry_id)
client.captureException(
exc_info=sys.exc_info())
if raise_exception:
raise e
return wrapper
return decorator
and so my mapper/reducer functions look like:
#log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn(item):
logging.info(item)
But it doesn't seem to work. Without the decorator, I'd find INFO logs of item. But if I put the decorator, it seems the mapper/reducer functions don't get called at all.
I was hoping to make it easy to log any errors my functions might have so I can fix them, as trying to track down MapReduce via AppEngine's logs is almost impossible.
I could wrap the entire function body with try ... except block, but a decorator would be cleaner.
I believe you have an issue with the decorator structure. In partuclar, I think you want to replace
try:
return fn(*args, **kwargs)
with
try:
fn(*args, **kwargs)
I'm missing some of the functions to test this, but you can see simplified decorator examples here if you want to run one: http://simeonfranklin.com/blog/2012/jul/1/python-decorators-in-12-steps/
Try something like this to make sure your code works, then try the more complicated parameterized version after:
sentry_id = id
raise_exception = 1
def basic_decorator(function):
global sentry_id, raise_exception
def wrapper(*args,**kwargs):
try:
function(*args,**kwargs)
except Exception, e:
client = Client(sentry_id)
client.captureException(exc_info=sys.exc_info())
if raise_exception:
raise
return wrapper
#basic_decorator
def my_mapper_fn(item):
logging.info(item)
To parameterize sentry_id and raise_exception, wrap the decorator inside another decorator. The idea is that when the basic decorator is defined, sentry_id, raise_exception, and function will be defined ahead of time and enclosed within its scope. This should look something like
def log_exceptions_to_sentry(sentry_id,raise_exception=1):
def basic_decorator(function):
def wrapper(*args, **kwargs):
try:
function(*args,**kwargs)
except Exception, e:
client = Client(sentry_id)
client.captureException(exc_info=sys.exc_info())
if raise_exception:
raise
return wrapper
return basic_decorator
#log_exceptions_to_sentry(SENTRY_ID,RAISE_EXCEPTION)
def my_mapper_fn(item):
logging.info(item)
I don't know what SENTRY_ID or Client is, since you didn't post it. So I made up my own. Using your code exactly, everything appears to work as expected. I'm not sure what you're seeing that isn't working right.
SENTRY_ID = 1
class Client(object):
def __init__(self, sentry_id): pass
def captureException(self, **kwargs):
print('captureException, ', kwargs['exc_info'])
def log_exceptions_to_sentry(sentry_id, raise_exception):
def decorator(fn):
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except Exception as e:
client = Client(sentry_id)
client.captureException(
exc_info=sys.exc_info())
if raise_exception:
raise e
return wrapper
return decorator
def fn(item):
logging.debug(item)
logging.info(item)
logging.error(item)
#log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn(item):
logging.debug(item)
logging.info(item)
logging.error(item)
return 1
#log_exceptions_to_sentry(SENTRY_ID, False)
def my_mapper_fn2(item):
raise Exception()
logging.basicConfig(
level = logging.INFO,
format = '%(levelname)s:%(name)s:%(message)s',
#format = '%(message)s',
)
x = fn({'a':1})
print(x)
x = my_mapper_fn({'b':2})
print(x)
x = my_mapper_fn2({'c':3})
print(x)
Output:
INFO:root:{'a': 1}
ERROR:root:{'a': 1}
None
INFO:root:{'b': 2}
ERROR:root:{'b': 2}
1
captureException, (<type 'exceptions.Exception'>, Exception(), <traceback object at 0x1813cf8>)
None
In Celery, you can retry any task in case of exception. You can do it like so:
#task(max_retries=5)
def div(a, b):
try:
return a / b
except ZeroDivisionError, exc:
raise div.retry(exc=exc)
In this case, if you want to to divide by zero, task will be retied five times. But you have to check for errors in you code explicitly. Task will not be retied if you skip try-except block.
I want my functions to look like:
#celery.task(autoretry_on=ZeroDivisionError, max_retries=5)
def div(a, b):
return a / b
Celery (since version 4.0) has exactly what you were looking for:
#app.task(autoretry_for=(SomeException,))
def my_task():
...
See: http://docs.celeryproject.org/en/latest/userguide/tasks.html#automatic-retry-for-known-exceptions
I searched this issue for a while, but found only this feature request.
I decide to write my own decorator for doing auto-retries:
def task_autoretry(*args_task, **kwargs_task):
def real_decorator(func):
#task(*args_task, **kwargs_task)
#functools.wraps(func)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except kwargs_task.get('autoretry_on', Exception), exc:
wrapper.retry(exc=exc)
return wrapper
return real_decorator
With this decorator I can rewriter my previous task:
#task_autoretry(autoretry_on=ZeroDivisionError, max_retries=5)
def div(a, b):
return a / b
I've modified your answer to work with the existing Celery API (currently 3.1.17)
class MyCelery(Celery):
def task(self, *args_task, **opts_task):
def real_decorator(func):
sup = super(MyCelery, self).task
#sup(*args_task, **opts_task)
#functools.wraps(func)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except opts_task.get('autoretry_on', Exception) as exc:
logger.info('Yo! We did it!')
wrapper.retry(exc=exc, args=args, kwargs=kwargs)
return wrapper
return real_decorator
Then, in your tasks
app = MyCelery()
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
#app.task(autoretry_on=Exception)
def mytask():
raise Exception('Retrying!')
This allows you to add the autoretry_on functionality to your tasks without having to use a separate decorator to define tasks.
Here is an improved version of the existing answers.
This fully implements the Celery 4.2 behaviour (as documented here) but for Celery 3.1.25.
It also doesn't break the different task decorator forms (with/without parentheses) and returns/raises properly.
import functools
import random
from celery.app.base import Celery as BaseCelery
def get_exponential_backoff_interval(factor, retries, maximum, full_jitter=False):
"""
Calculate the exponential backoff wait time.
(taken from Celery 4 `celery/utils/time.py`)
"""
# Will be zero if factor equals 0
countdown = factor * (2 ** retries)
# Full jitter according to
# https://www.awsarchitectureblog.com/2015/03/backoff.html
if full_jitter:
countdown = random.randrange(countdown + 1)
# Adjust according to maximum wait time and account for negative values.
return max(0, min(maximum, countdown))
class Celery(BaseCelery):
def task(self, *args, **opts):
"""
Overridden to add a back-port of Celery 4's `autoretry_for` task args.
"""
super_method = super(Celery, self).task
def inner_create_task_cls(*args_task, **opts_task):
# http://docs.celeryproject.org/en/latest/userguide/tasks.html#Task.autoretry_for
autoretry_for = tuple(opts_task.get('autoretry_for', ())) # Tuple[Type[Exception], ...]
retry_backoff = int(opts_task.get('retry_backoff', False)) # multiplier, default if True: 1
retry_backoff_max = int(opts_task.get('retry_backoff_max', 600)) # seconds
retry_jitter = opts_task.get('retry_jitter', True) # bool
retry_kwargs = opts_task.get('retry_kwargs', {})
def real_decorator(func):
#super_method(*args_task, **opts_task)
#functools.wraps(func)
def wrapper(*func_args, **func_kwargs):
try:
return func(*func_args, **func_kwargs)
except autoretry_for as exc:
if retry_backoff:
retry_kwargs['countdown'] = get_exponential_backoff_interval(
factor=retry_backoff,
retries=wrapper.request.retries,
maximum=retry_backoff_max,
full_jitter=retry_jitter,
)
raise wrapper.retry(exc=exc, **retry_kwargs)
return wrapper
return real_decorator
# handle both `#task` and `#task(...)` decorator forms
if len(args) == 1:
if callable(args[0]):
return inner_create_task_cls(**opts)(*args)
raise TypeError('argument 1 to #task() must be a callable')
if args:
raise TypeError(
'#task() takes exactly 1 argument ({0} given)'.format(
sum([len(args), len(opts)])))
return inner_create_task_cls(**opts)
I have also written some unit tests for this as am using it in my project.
They can be found in this gist but note they are not easily runnable - treat more as documentation of how the above feature works (and validation that it works properly).