I'm new to python and trying to remove/trim gevent stacktrace output when an exception is raised. I read somewhere that I can make it happen by using AsyncResult, however it seems like I can't figure out how to use this.
Here is an example I started with and iterated over to make it similar to the real code I'm troubleshooting, but I got stuck in the last phase when I tried to add my_decor to work().
Any help fixing this is much appreciated.
from gevent.event import AsyncResult
import gevent
from functools import wraps
def my_decor(k, *args, **kwargs):
#wraps(k)
def wrapper(*args, **kwargs):
r = AsyncResult()
try:
value = k()
except Exception as e:
r.set_exception(e)
else:
r.set(value)
return r.exception or r.value
result = gevent.spawn(wrapper, k)
return result
def f():
def foo():
if True:
raise Exception('tttttttt')
return foo
def p():
def bar():
if True:
raise Exception('ppppppppppppp')
return bar
#my_decor
def work():
foo1 = gevent.spawn(f())
bar1 = gevent.spawn(p())
gevent.joinall([foo1, bar1])
return foo1.get() or bar1.get()
Found the answer, figured it might be a help to those with the same problem.
from gevent.event import AsyncResult
import gevent
from functools import wraps
def my_decor(k):
#wraps(k)
def wrapper(*args, **kwargs):
r = AsyncResult()
try:
value = k(*args, **kwargs)
except Exception as e:
r.set_exception(e)
else:
r.set(value)
return r.exception or r.value
return wrapper
def f(msg):
#my_decor
def foo():
if True:
raise Exception('tttttttt %s' % msg)
# print('test')
return foo
def p(msg):
#my_decor
def bar():
if True:
raise Exception('ppppppppppppp %s', msg)
return bar
def work():
test = "test"
seti = "set"
foo1 = gevent.spawn(f(test)) # returns a function that coroutine uses
bar1 = gevent.spawn(p(seti))
gevent.joinall([foo1, bar1])
return foo1.get() or bar1.get()
res = work()
print res
Related
I am trying to download some data from the web (web scraping); I have a list of URLs, a few of the URLs take too much time, and thus the loop gets stuck there; I am implementing a function that timeout after a certain period of threshold value and loop should be continued.
For example, if the downloading_source looks like this:
import time
import numpy as np
def downloading_source(x):
wt = np.random.randint(1,50)
print("waiting time", wt)
time.sleep(wt)
return x**2
for the demo, I am taking random values as time. sleep, and the timeout function looks like this
import error
import os
import signal
import functools
class TimeoutError(Exception):
pass
def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
def decorator(func):
def _handle_timeout(signum, frame):
raise TimeoutError(error_message)
#functools.wraps(func)
def wrapper(*args, **kwargs):
signal.signal(signal.SIGALRM, _handle_timeout)
signal.alarm(seconds)
try:
result = func(*args, **kwargs)
finally:
signal.alarm(0)
return result
return wrapper
return decorator
The download loop:
# Timeout after 5 seconds
#timeout(5)
def long_running_function2(x):
return downloading_source(x)
all_urls = list(range(1,100))
downloaded_data = []
for url in all_urls:
try:
print(url)
down_data = long_running_function2(url)
except Exception as e:
pass
downloaded_data.append(down_data)
It's working; I was wondering, is there any better way to do this?
I am new to Python and learning logging technique with Decorator.
For me the below code is not generating required log file. Debugged the code, getting correct message to logger statement but the file is not generating. From Test method i am call the required function where i have implemented Decorator. Please guide where i am doing mistake.
try:
import csv
import requests
import datetime
import os
import sys
import logging
except Exception as e:
print("Some Modules are missing {}".format(e))
class Meta(type):
""" Meta class"""
def __call__(cls, *args, **kwargs):
instance = super(Meta, cls).__call__(*args, **kwargs)
return instance
def __init__(cls, name, base, attr):
super(Meta, cls).__init__(name, base, attr)
class log(object):
def __init__(self, func):
self.func = func
def __call__(self, *args, **kwargs):
""" Wrapper Function"""
start = datetime.datetime.now() #start time
Tem = self.func(*args) #call Function
Argument = args
FunName = self.func.__name__ #get Function name
end = datetime.datetime.now() #end Time
message = """
Function : {}
Execustion Time : {}
Argument : {}
Memory : {} Bytes
Date : {}
""".format(FunName,
end-start,
Argument,
sys.getsizeof(self.func),
start
)
cwd = os.getcwd();
folder = 'Logs'
newPath = os.path.join(cwd, folder)
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
logging.basicConfig(filename='apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
return Tem
class APIHelper(metaclass=Meta):
def __init__(self, *args, **kwargs):
pass
#log
def star_wars_characters(url):
#self.url = url
api_response = requests.get(url)
people = []
if api_response.status_code == 200:
data = api_response.json()
for d in data['results']:
character = []
character.append(d['name'])
character.append(d['height'])
character.append(d['gender'])
people.append(character)
return people
else:
return "Bad Request"
My Test Method:
import unittest
import csv
from com.Script.APIHelper import APIHelper
class TestAPI(unittest.TestCase):
def _setUp(self, file_name):
self.api = APIHelper()
with open(file_name, "w") as self.fd:
self.csvfile = csv.writer(self.fd, delimiter = ',')
self.csvfile.writerow(['Name','Height','Gender'])
def tearDown(self):
self.fd.close()
def test_responseNotEmpty(self):
file_name = 'SWAPI.csv'
self._setUp(file_name)
people = self.api.star_wars_characters("https://swapi.dev/api/people/")
assert type(people) is list
Thanks you in Advance.
Add finally
Change filename='apiRun.log' to filename='{}/apiRun.log'
try:
"""Try to create a folder """
os.mkdir(newPath)
except:
"""Folder already exist """
finally:
logging.basicConfig(filename='{}/apiRun.log'.format(newPath), level=logging.DEBUG)
logging.debug(message)
except is executed only when an exception is raised from try.
finally is always executed.
Currently I have a piece of code that operates as such:
try:
function1()
except:
function2()
except:
function3()
But I get thrown the error: default 'except:' must be last. Basically if function1 fails, try function2. If function 2 fails, try function 3.
How would I achieve this?
You have to nest the exception handlers to match your handling logic.
try:
function1()
except:
try:
function2()
except:
function3()
If you control the function code, then I suggest that you have each one return a status code ... say, 0 for success and -1 for failure.
for f in [function1, function2, function3]:
if f(): break
Does that handle your use case? You keep executing functions until one succeeds.
If you would like to create longer chains of error handlers you might consider using a decorator.
def try_except_decorator(except_func):
def decorator(try_func):
def wrapped():
try:
try_func()
except:
except_func()
return wrapped
return decorator
def function3():
print('function3')
#try_except_decorator(function3)
def function2():
print('function2')
raise ValueError
#try_except_decorator(function2)
def function1():
print('function1')
raise ValueError
function1()
Assuming you don't control functionX, you could write a class that generically wraps functions and suppresses exceptions. This is similar to the decorator idea, but if you can change the source to add the decorator... why not just change the function itself?
class Supressor:
def __init__(self, func, exceptions=(Exception,), exc_val=None):
self._func = func
self._exc = exceptions
self._exc_val = exc_val
try:
self.__doc__ = func.__doc__
except AttributeError:
passs
def __call__(self, *args, **kw):
try:
retval = self._func(*args, **kw)
except Exception as e:
for okay in self._exc:
if isinstance(e, okay):
return self._exc_val
else:
raise
# defined in other module but placed here for test
def function1():
print('f1')
raise ValueError()
return 1
def function2():
print('f2')
raise KeyError()
return 2
def function3():
print('made it')
return 3
function1 = Supressor(function1)
function2 = Supressor(function2)
function3 = Supressor(function3)
if function1() is None:
if function2() is None:
function3()
I'm on python 2.7, tornado 4.5
The following code doesn't work: the except block doesn't get triggered. I don't understand why?
#gen.coroutine
def co_do_thing():
yield gen.Task(do_thing)
def do_thing(callback):
try:
a, b = ...
result = maybe_throw(a, b, callback)
except Exception as e:
# this block is not called
if a:
raise ApiError("called with A")
elif b:
raise ApiError("called with B")
else:
raise e
def maybe_throw(arg1, arg2, callback):
if random.random() < 0.5:
raise AssertionError("yikes")
callback("done")
Instead, I can catch the exception in co_do_thing around the call to gen.Task; but then I don't have the context of how I called maybe_throw. In my case, it makes more sense for maybe_throw to raise a lower-level exception, and for the caller to convert that to a human-readable error depending on the inputs.
Do I just need to refactor this to call gen.Task at a lower level? That would be annoying :/
As I tested it seems to work, a exception is raised. Below simple test suite:
import q # q.py is the file with question's code
import unittest
from mock import patch, Mock
from tornado.testing import gen_test, AsyncTestCase
class MyTest(AsyncTestCase):
def setUp(self):
self.mock_random = patch('q.random').start()
AsyncTestCase.setUp(self)
def tearDown(self):
AsyncTestCase.tearDown(self)
patch.stopall()
#gen_test
def test_no_error(self):
self.mock_random.return_value = 0.7
res = yield q.co_do_thing()
self.assertEqual(res, 'done')
#gen_test
def test_exception(self):
self.mock_random.return_value = 0.1
with self.assertRaises(Exception) as ctx:
yield q.co_do_thing()
self.assertEqual(ctx.exception.message, 'called with A')
if __name__ == '__main__':
unittest.main()
And tests passed:
..
----------------------------------------------------------------------
Ran 2 tests in 0.002s
OK
And here is q.py, I've added return statement to test it.
from random import random
from tornado import gen
#gen.coroutine
def co_do_thing():
res = yield gen.Task(do_thing)
# added: to enable to test it meaningfully
raise gen.Return(res)
def do_thing(callback):
try:
a, b = 22, 33
result = maybe_throw(a, b, callback)
except Exception as e:
if a:
raise Exception("called with A")
elif b:
raise Exception("called with B")
else:
raise e
def maybe_throw(arg1, arg2, callback):
if random() < 0.5:
raise AssertionError("yikes")
callback("done")
I have a bit of code that gets the title of a .MP3 file
def getTitle(fileName):
print "getTitle"
audio = MP3(fileName)
try:
sTitle = str(audio["TIT2"])
except KeyError:
sTitle = os.path.basename(fileName)
sTitle = replace_all(sTitle) #remove special chars
return sTitle
I would call this function with
sTitle = getTitle("SomeSong.mp3")
To solve another problem I wanted to spawn this on its own thread so I altered my call to
threadTitle = Thread(target=getTitle("SomeSong.mp3"))
threadTitle.start()
This correctly calls the function and solves my other problem, but now I can't figure out how to get the return value of sTitle from the function into Main.
I would make a new object that extends thread so that you can get anything you want out of it at any time.
from threading import Thread
class GetTitleThread(Thread):
def __init__(self, fileName):
self.sTitle = None
self.fileName = fileName
super(GetTitleThread, self).__init__()
def run(self):
print "getTitle"
audio = MP3(self.fileName)
try:
self.sTitle = str(audio["TIT2"])
except KeyError:
self.sTitle = os.path.basename(self.fileName)
self.sTitle = replace_all(self.sTitle) #remove special chars
if __name__ == '__main__':
t = GetTitleThread('SomeSong.mp3')
t.start()
t.join()
print t.sTitle
One way to do it is to use a wrapper storing the result:
def wrapper(func, args, res):
res.append(func(*args))
res = []
t = threading.Thread(
target=wrapper, args=(getTitle, ("SomeSong.mp3",), res))
t.start()
t.join()
print res[0]
This one comfortably makes any function running in a thread taking care of its return value or exception:
def threading_func(f):
"""Decorator for running a function in a thread and handling its return
value or exception"""
def start(*args, **kw):
def run():
try:
th.ret = f(*args, **kw)
except:
th.exc = sys.exc_info()
def get(timeout=None):
th.join(timeout)
if th.exc:
raise th.exc[0], th.exc[1], th.exc[2] # py2
##raise th.exc[1] #py3
return th.ret
th = threading.Thread(None, run)
th.exc = None
th.get = get
th.start()
return th
return start
Usage Examples
def f(x):
return 2.5 * x
th = threading_func(f)(4)
print("still running?:", th.is_alive())
print("result:", th.get(timeout=1.0))
#threading_func
def th_mul(a, b):
return a * b
th = th_mul("text", 2.5)
try:
print(th.get())
except TypeError:
print("exception thrown ok.")