I'm basically creating an object that needs to perform a number of tasks in async mode (+ other things but I've tried to simplify here). Here is a snippet of code for the object itself. It's successful (thanks to Lynn Root's example) at handling signals - but not at handling exceptions. Or at least not in the way I am hoping to be able to handle them.
class myobj(object):
def __init__(self, loop: asyncio.AbstractEventLoop):
self.shutdown = False
self.loop = loop
async def graceful_shutdown(self, s=None):
if s is not None:
logging.warning(f'Receiving signal {s.name}.')
else:
logging.warning(f'Shutting NOT via signal')
logging.warning(f'Initiating cancellation of {len(self.tasks)} tasks...')
[task.cancel() for task in self.tasks]
logging.warning(f'Gaterhing out put of cancellation of {len(self.tasks)} tasks...')
await asyncio.gather(*self.tasks, loop=self.loop, return_exceptions=True)
logging.warning('Done graceful shutdown of subtasks')
# Mark main task to shutdown
self.shutdown = True
async def run(self):
i = 0
taskx = self.loop.create_task(self.task_x())
self.tasks = [taskx]
while not self.shutdown:
await asyncio.sleep(1)
print(f'Main runner... {i}')
i += 1
logging.warning('Main runner is over.')
async def task_x(self):
logging.warning('Starting task X')
i = 0
while True:
await asyncio.sleep(2.25)
print(f'Doing task x... {i}')
if i == 2:
raise RuntimeError('BOOM X!')
i += 1
At this point, from the "main" I need to install a few things and create the loop :
def main():
try:
global newobj
loop = asyncio.get_event_loop()
logging.warning(f'Installing exception handler')
loop.set_exception_handler(handle_exception)
logging.warning(f'Creating main object')
newobj = myobj(loop)
logging.warning(f'Installing signal handlers')
signals = (signal.SIGINT, signal.SIGTERM)
for s in signals:
loop.add_signal_handler(s, lambda s=s: loop.create_task(newobj.graceful_shutdown(s)))
logging.warning(f'Running object...')
loop.run_until_complete(newobj.run())
finally:
loop.close()
logging.warning(f'object is Shutdown - Exiting program.')
sys.exit(0)
if __name__ == "__main__":
main()
But the handle_exception needs to be defined.
def handle_exception(loop, context):
# context["message"] will always be there; but context["exception"] may not
msg = context.get("exception", context["message"])
logging.error(f'Caught exception: {msg}')
logging.info(f'Shutting down from exception.')
loop.create_task(newobj.graceful_shutdown())
The problem is that it's never calling handle_exception. I need to be running this in python3.6 for some reason. What am I missing here?
Related
I am trying to create a process that would run indefinitely until being shut down (either from the command prompt or from another notebook cell) by changing a class member. My attempt looks like the follow:
def async_run(*awaitables: Awaitable,
timeout: float = None):
loop = asyncio.get_event_loop()
if not awaitables:
if loop.is_running():
return
loop.run_forever()
result = None
if sys.version_info >= (3, 7):
all_tasks = asyncio.all_tasks(loop) # type: ignore
else:
all_tasks = asyncio.Task.all_tasks() # type: ignore
if all_tasks:
# cancel pending tasks
f = asyncio.gather(*all_tasks)
f.cancel()
try:
loop.run_until_complete(f)
except asyncio.CancelledError:
pass
else:
if len(awaitables) == 1:
future = awaitables[0]
else:
future = asyncio.gather(*awaitables)
if timeout:
future = asyncio.wait_for(future, timeout)
task = asyncio.ensure_future(future)
def onError(_):
task.cancel()
global_error_event.connect(onError)
try:
result = loop.run_until_complete(task)
except asyncio.CancelledError as e:
raise global_error_event.value() or e
finally:
global_error_event.disconnect(onError)
return result
class Platform:
def __init__(self):
self.is_active: bool = True
async def kickstart(self):
while self.is_active:
await async_run(asyncio.sleep(0))
However it seems that the kickstart() function would simply finish running, even though my intention is for it to run indefinitely. If I remove the async, it does run indefinitely but then it becomes blocking. I'd appreciate some help on this, thanks in advance.
This is a test script I created to better understand task cancellation -
import asyncio
import random
import signal
import traceback
async def shutdown(signame, loop):
print("Shutting down")
tasks = [task for task in asyncio.Task.all_tasks()]
for task in tasks:
task.cancel()
try:
await task
except asyncio.CancelledError:
print("Task cancelled: %s", task)
loop.stop()
async def another():
await asyncio.sleep(2)
async def some_other_process():
await asyncio.sleep(5)
return "Me"
async def process(job, loop, i):
print(i)
task = loop.create_task(some_other_process())
value = await task
if i < 1:
another_task = loop.create_task(another())
await another_task
# await some_other_process()
def pull(loop):
i = 0
while True:
job = f"random-integer-{random.randint(0, 100)}"
try:
loop.run_until_complete(process(job, loop, i))
i += 1
except asyncio.CancelledError as e:
print("Task cancelled")
break
except Exception:
print(traceback.format_exc())
# asyncio.get_event_loop().stop()
def main():
try:
loop = asyncio.get_event_loop()
for signame in ['SIGINT']:
loop.add_signal_handler(
getattr(signal, signame),
lambda: asyncio.ensure_future(shutdown(signame, loop))
)
try:
pull(loop)
except Exception:
print(traceback.format_exc())
finally:
loop.close()
finally:
print("Done")
if __name__ == "__main__":
main()
And I can not understand why I see -
Task was destroyed but it is pending!
task: <Task cancelling coro=<shutdown() done, defined at test.py:6>>
loop.add_signal_handler(
getattr(signal, signame),
lambda: asyncio.ensure_future(shutdown(signame, loop))
)
Here using asyncio.ensure_future you create task for shutdown coroutine, but you don't await anywhere for this task to be finished. Later when you close event loop it warns you this task is pending.
Upd:
If you want to do some clenup, the best place for it is right before loop.close() regardless of reason your script ended (signal, exception, etc.)
Try to alter your code this way:
# ...
async def shutdown(loop): # remove `signal` arg
# ...
def main():
try:
loop = asyncio.get_event_loop()
try:
pull(loop)
except Exception:
print(traceback.format_exc())
finally:
loop.run_until_complete(shutdown(loop)) # just run until shutdown is done
loop.close()
finally:
print("Done")
# ...
Upd2:
In case you still want signal handler, you probably want to do something like this:
from functools import partial
loop.add_signal_handler(
getattr(signal, signame),
partial(cb, signame, loop)
)
def cb(signame, loop):
loop.stop()
loop.run_until_complete(shutdown(signame, loop))
I would like to stop a python asyncio task from another task and start it again when some condition in the second task happen.
Please note, than I don't want to cancel the coroutine of the first task (the state of that coroutine when it stopped should be available). Also, I don't care about the exact state the first task is in, I just want the event loop stop running the first task until told otherwise from the second.
I hope this example code helps understanding the problem:
import asyncio
async def coroutine1():
i = 0
while(True):
i += 1
print("coroutine1: " + str(i) )
await asyncio.sleep(1)
async def coroutine2(task1):
i = 0
while(True):
i += 1
if (i > 3) and (i<10):
pass #TODO: stop task1 here
else:
pass #TODO: Maybe check if task1 is running
#and start task1 again if it's not?
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
loop = asyncio.get_event_loop()
task1 = loop.create_task(coroutine1())
task2 = loop.create_task(coroutine2(task1))
done, pending = await asyncio.wait(
[task1, task2]
, return_when=asyncio.FIRST_COMPLETED,)
loop = asyncio.get_event_loop()
loop.run_until_complete(main_coroutine())
loop.close()
I would like to stop a python asyncio task from another task and start it again when some condition in the second task happen.
I assume you control the task creation, but don't want to touch the implementation of the coroutine. In your case, you control coroutine2 and main_coroutine, but not the insides of coroutine1.
In that case you can wrap the coroutine in a an __await__ that, instead of the normal yield from loop, checkes your stopped flag and waits for a future that tells it when to resume.
class Stoppable:
def __init__(self, coro):
self._coro_iter = coro.__await__()
self._stopped = None
def __await__(self):
while True:
while self._stopped:
print('awaiting stopped')
yield from self._stopped.__await__()
try:
v = next(self._coro_iter)
except StopIteration as e:
return v
yield v
def stop(self):
loop = asyncio.get_event_loop()
self._stopped = loop.create_future()
def start(self):
if self._stopped is not None:
self._stopped.set_result(None)
self._stopped = None
You can use the wrapper to modify coroutine2 to stop and resume the execution of coroutine1 at will:
async def coroutine2(s):
i = 0
while True:
i += 1
if i == 3:
print('stopping coroutine1')
s.stop()
elif i == 10:
print('restarting coroutine1')
s.start()
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
loop = asyncio.get_event_loop()
s = Stoppable(coroutine1())
fut1 = asyncio.ensure_future(s)
task2 = loop.create_task(coroutine2(s))
done, pending = await asyncio.wait(
[fut1, task2], return_when=asyncio.FIRST_COMPLETED)
The way wrapper works is by unrolling the loop inherent in yield from. For example, to just delegate __await__ to another coroutine, one would write:
def __await__(self):
yield from self._coro_iter
Written like this, you can't implement stopping because the yield from contains an implicit loop that yields all the values produced by the underlying iterator - something like:
def __await__(self):
while True:
try:
v = next(self._coro_iter)
except StopIteration as e:
return e.value
yield v
Taken like this, it is easy enough to add an if that checks for _stopped at each iteration pass, meaning each time we're resumed by the event loop. The remaining hurdle is that one cannot just busy-loop until _stopped is rescinded - we must yield something else to allow the event loop to resume running other coroutines. Fortunately that is easily achieved by making _stopped a future, and yielding from the future. When the future's result is set, we will be automatically resumed and continue executing the wrapped coroutine.
It seems it can't be done.
It's possible to cancel an ongoing task with task1.cancel() and it's possible to create a new task with asyncio.get_event_loop().create_task(newTask).
It's also possible to get the coroutine of a running task with task1._coro but if we try to create a task again with a previously scheduled coroutine we will get a RuntimeError exception. This the discussion where they decided it: https://bugs.python.org/issue25887
Finally, a possible way of accomplishing the desire effect is using a asyncio.Queue object:
import asyncio
async def coroutine1(stop_queue):
i = 0
while(True):
if stop_queue.empty(): #if the queue is empty keep working.
i += 1
print("coroutine1: " + str(i) )
await asyncio.sleep(1)
async def coroutine2(stop_queue):
i = 0
while(True):
i += 1
if i == 3:
await stop_queue.put("whatever..") #put something in the queue
if i == 11:
await stop_queue.get() #take something from the queue
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
stop_queue = asyncio.Queue()
done, pending = await asyncio.wait(
[coroutine1(stop_queue), coroutine2(stop_queue)]
, return_when=asyncio.ALL_COMPLETED,)
loop = asyncio.get_event_loop()
loop.run_until_complete(main_coroutine())
loop.close()
I have this async worker functionality using tornado's ioloop.
I'm trying to shutdown the loop gracefully on Ctrl+C but getting the following error
tornado.ioloop.TimeoutError: Operation timed out after None seconds
I know I can catch it but I do want to finish the process in a graceful way, how can I achieve that?
#!/usr/bin/env python
import time
import signal
import random
from tornado import gen, ioloop, queues
concurrency = 10
def sig_exit(signum, frame):
ioloop.IOLoop.current().add_callback_from_signal(shutdown)
def shutdown():
print('Will shutdown in few seconds ...')
io_loop = ioloop.IOLoop.current()
deadline = time.time() + 3
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
#gen.coroutine
def main():
q = queues.Queue()
q.put(1)
#gen.coroutine
def do_stuff():
print("doing stuff")
yield gen.Task(ioloop.IOLoop.instance().add_timeout, time.time() + random.randint(1, 5))
print("done doing stuff")
#gen.coroutine
def worker():
while True:
yield do_stuff()
for _ in range(concurrency):
worker()
yield q.join()
if __name__ == '__main__':
signal.signal(signal.SIGTERM, sig_exit)
signal.signal(signal.SIGINT, sig_exit)
io_loop = ioloop.IOLoop.instance()
io_loop.run_sync(main)
If you're using run_sync, you can no longer call IOLoop.stop - run_sync is now responsible for that. So if you want to make this shutdown "graceful" (instead of just raising a KeyboardInterrupt at the point where you now call stop() and exiting with a stack trace), you need to change the coroutine passed to run_sync so it exits.
One possible solution is a tornado.locks.Event:
# Create a global Event
shutdown_event = tornado.locks.Event()
def shutdown():
# Same as in the question, but instead of `io_loop.stop()`:
shutdown_event.set()
#gen.coroutine
def main():
# Use a WaitIterator to exit when either the queue
# is done or shutdown is triggered.
wait_iter = gen.WaitIterator(q.join(), shutdown_event.wait())
# In this case we just want to wait for the first one; we don't
# need to actually iterate over the WaitIterator.
yield wait_iter.next()
async def main():
tornado.options.parse_command_line()
...
app = Application(db)
app.listen(options.port)
shutdown_event = tornado.locks.Event()
def shutdown( signum, frame ):
print("shutdown database !!!!")
db.close()
shutdown_event.set()
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
await shutdown_event.wait()
print("\n\nshutdown -h now")
if __name__ == "__main__":
tornado.ioloop.IOLoop.current().run_sync(main)
I am trying to make a barebones skeleton fighting game with python asyncio.
class Skeleton(Creature):
pass
class SkeletonAI():
def __init__(self, skeleton,loop = None):
self.loop = loop or asyncio.new_event_loop()
self.skeleton = skeleton
self.action_task = None
async def run(self):
while True:
#print(self.action_task, )
if self.skeleton.alive and self.skeleton.target.alive:
if self.skeleton.state == 'idle':
#ATTACK
self.skeleton.begin_attack()
self.action_task = self.loop.call_later(3, self.skeleton.action_complete)
else:
break
class Player(Creature):
def attack_target(self, target):
target.take_damage(self.damage)
if target.state == 'attacking':
target.state = 'idle'
#interrupt attack
class Game():
#Super simple game
#The skeleton launches an attack, press anything to interrupt it
async def handle_sending(self):
loop = asyncio.get_event_loop()
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=1,
)
while True:
msg = await loop.run_in_executor(executor, input)
print('got a message')
if self.skeleton_ai.action_task:
print('cancelling attack')
self.skeleton_ai.action_task.cancel()
self.skeleton_ai.skeleton.machine.set_state('idle')
print('cancelled attack')
self.skeleton_ai.action_task = None
async def game_loop(self):
player_task = asyncio.ensure_future(self.handle_sending())
skeleton_task = asyncio.ensure_future(self.skeleton_ai.run())
def __init__(self):
self.task = None
self.loop = asyncio.get_event_loop()
self.player = Player(name='ply')
self.skeleton_ai = SkeletonAI(skeleton=Skeleton(name='bobby'))
self.skeleton_ai.skeleton.target = self.player
self.loop.run_until_complete(self.game_loop())
try:
self.loop.run_forever()
finally:
pass
loop.close()
Game()
Here's what I am trying to do:
Player input and game output are async, so input() doesn't block. This works.
The skeleton prepares an attack, if it's not interrupted in 3 seconds, the attack deals damage to the player.
The player can input any text to interrupt the skeleton attack.
How can I make the skeleton's attack? I want a task I can interrupt at will and call a callback later. Currently everything just gets stuck. The call_later never calls.
This is the pattern for a async function with timeout and callback function. The clue is to catch the asyncio.TimeoutError and do your timeout logic. The function that is cancelled will not continue after it's current await position.
import asyncio
async def slow_function(seconds):
print('starting slow computations')
await asyncio.sleep(seconds)
print('slow computations done')
async def timeout_callback():
print('timeout called')
async def timeout_with_cb(fut, timeout, timeout_fut):
try:
await asyncio.wait_for(fut, timeout)
except asyncio.TimeoutError:
await timeout_fut
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.ensure_future(
timeout_with_cb(slow_function(2), 1,
timeout_callback())))
This will print:
starting slow computations
timeout called
I guess this can help you to adapt your example (the provided example does not compile).