This is similar to Calling coroutines in asyncio.Protocol.data_received but I think it warrants a new question.
I have a simple server set up like this
loop.create_unix_server(lambda: protocol, path=serverSocket)
It works fine, if I do this
def data_received(self, data):
data = b'data reply'
self.send(data)
my client gets the reply. But I can't get it to work with any sort of asyncio call. I tried all of the following and none of them worked.
#asyncio.coroutine
def go(self):
yield from asyncio.sleep(1, result = b'data reply')
def data_received(self, data):
print('Data Received', flush=True)
task = asyncio.get_event_loop().create_task(self.go())
data = yield from asyncio.wait_for(task,10)
self.send(data)
that one hung and printed nothing (if I decorated data_received with #asyncio.coroutine I get that that is not yielded from) OK, I get that using yield in data_received isn't right.
If I try a new event loop, as below, that hangs in run_until_complete
loop = asyncio.new_event_loop()
task = loop.create_task(self.go())
loop.run_until_complete(task)
data = task.result()
self.send(data)
If I use a Future, that also hangs in run_until_complete
#asyncio.coroutine
def go(self, future):
yield from asyncio.sleep(1)
future.set_result(b'data reply')
def data_received(self, data):
print('Data Received', flush=True)
loop = asyncio.new_event_loop()
future = asyncio.Future(loop=loop)
asyncio.async(self.go(future))
loop.run_until_complete(future)
data = future.result()
self.send(data)
The following gets close, but it returns immediately and the result is of type asyncio.coroutines.CoroWrapper, implying that the wait_for line returned immediately with the unfinished task?
#asyncio.coroutine
def go(self):
return(yield from asyncio.sleep(3, result = b'data reply'))
#asyncio.coroutine
def go2(self):
task = asyncio.get_event_loop().create_task(self.go())
res = yield from asyncio.wait_for(task, 10)
return result
def data_received(self, data):
print('Data Received', flush=True)
data = self.go2()
self.send(data)
I'm a bit stuck really, and would appreciate some pointers about what to look at.
You need to add your coroutine to the event loop, and then use Future.add_done_callback to handle the result when the coroutine completes:
#asyncio.coroutine
def go(self):
return(yield from asyncio.sleep(3, result = b'data reply'))
def data_received(self, data):
print('Data Received', flush=True)
task = asyncio.async(self.go()) # or asyncio.get_event_loop().create_task()
task.add_done_callback(self.handle_go_result)
def handle_go_result(self, task):
data = task.result()
self.send(data)
Calling a coroutine directly in data_received just simply isn't allowed, since the caller isn't going to try to yield from it, and creating/running a new event loop inside of data_received will always end up blocking the main event loop until the inner event loop finishes its work.
You just want to schedule some work with your main event loop (asyncio.async/loop.create_task()), and schedule a callback to run when the work is done (add_done_callback).
Related
The code below is intended to send multiple HTTP requests asynchronously in a while loop, and depending on the response from each request(request "X" always returns "XXX", "Y" always returns "YYY" and so on), do something and sleep for interval seconds specified for each request.
However, it throws an error...
RuntimeError: cannot reuse already awaited coroutine
Could anyone help me how I could fix the code to realise the intended behaviour?
class Client:
def __init__(self):
pass
async def run_forever(self, coro, interval):
while True:
res = await coro
await self._onresponse(res, interval)
async def _onresponse(self, res, interval):
if res == "XXX":
# ... do something with the resonse ...
await asyncio.sleep(interval)
if res == "YYY":
# ... do something with the resonse ...
await asyncio.sleep(interval)
if res == "ZZZ":
# ... do something with the resonse ...
await asyncio.sleep(interval)
async def request(something):
# ... HTTP request using aiohttp library ...
return response
async def main():
c = Client()
await c.run_forever(request("X"), interval=1)
await c.run_forever(request("Y"), interval=2)
await c.run_forever(request("Z"), interval=3)
# ... and more
As the error says, you can't await a coroutine more than once. Instead of passing a coroutine into run_forever and then awaiting it in a loop, passing the coroutine's argument(s) instead and await a new coroutine each iteration of the loop.
class Client:
async def run_forever(self, value, interval):
while True:
res = await rqequest(value)
await self._response(response, interval)
You also need to change how you await run_forever. await is blocking, so when you await something with an infinite loop, you'll never reach the next line. Instead, you want to gather multiple coroutines as once.
async def main():
c = Client()
await asyncio.gather(
c.run_forever("X", interval=1),
c.run_forever("Y", interval=2),
c.run_forever("Z", interval=3),
)
I'm basically creating an object that needs to perform a number of tasks in async mode (+ other things but I've tried to simplify here). Here is a snippet of code for the object itself. It's successful (thanks to Lynn Root's example) at handling signals - but not at handling exceptions. Or at least not in the way I am hoping to be able to handle them.
class myobj(object):
def __init__(self, loop: asyncio.AbstractEventLoop):
self.shutdown = False
self.loop = loop
async def graceful_shutdown(self, s=None):
if s is not None:
logging.warning(f'Receiving signal {s.name}.')
else:
logging.warning(f'Shutting NOT via signal')
logging.warning(f'Initiating cancellation of {len(self.tasks)} tasks...')
[task.cancel() for task in self.tasks]
logging.warning(f'Gaterhing out put of cancellation of {len(self.tasks)} tasks...')
await asyncio.gather(*self.tasks, loop=self.loop, return_exceptions=True)
logging.warning('Done graceful shutdown of subtasks')
# Mark main task to shutdown
self.shutdown = True
async def run(self):
i = 0
taskx = self.loop.create_task(self.task_x())
self.tasks = [taskx]
while not self.shutdown:
await asyncio.sleep(1)
print(f'Main runner... {i}')
i += 1
logging.warning('Main runner is over.')
async def task_x(self):
logging.warning('Starting task X')
i = 0
while True:
await asyncio.sleep(2.25)
print(f'Doing task x... {i}')
if i == 2:
raise RuntimeError('BOOM X!')
i += 1
At this point, from the "main" I need to install a few things and create the loop :
def main():
try:
global newobj
loop = asyncio.get_event_loop()
logging.warning(f'Installing exception handler')
loop.set_exception_handler(handle_exception)
logging.warning(f'Creating main object')
newobj = myobj(loop)
logging.warning(f'Installing signal handlers')
signals = (signal.SIGINT, signal.SIGTERM)
for s in signals:
loop.add_signal_handler(s, lambda s=s: loop.create_task(newobj.graceful_shutdown(s)))
logging.warning(f'Running object...')
loop.run_until_complete(newobj.run())
finally:
loop.close()
logging.warning(f'object is Shutdown - Exiting program.')
sys.exit(0)
if __name__ == "__main__":
main()
But the handle_exception needs to be defined.
def handle_exception(loop, context):
# context["message"] will always be there; but context["exception"] may not
msg = context.get("exception", context["message"])
logging.error(f'Caught exception: {msg}')
logging.info(f'Shutting down from exception.')
loop.create_task(newobj.graceful_shutdown())
The problem is that it's never calling handle_exception. I need to be running this in python3.6 for some reason. What am I missing here?
I would like to stop a python asyncio task from another task and start it again when some condition in the second task happen.
Please note, than I don't want to cancel the coroutine of the first task (the state of that coroutine when it stopped should be available). Also, I don't care about the exact state the first task is in, I just want the event loop stop running the first task until told otherwise from the second.
I hope this example code helps understanding the problem:
import asyncio
async def coroutine1():
i = 0
while(True):
i += 1
print("coroutine1: " + str(i) )
await asyncio.sleep(1)
async def coroutine2(task1):
i = 0
while(True):
i += 1
if (i > 3) and (i<10):
pass #TODO: stop task1 here
else:
pass #TODO: Maybe check if task1 is running
#and start task1 again if it's not?
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
loop = asyncio.get_event_loop()
task1 = loop.create_task(coroutine1())
task2 = loop.create_task(coroutine2(task1))
done, pending = await asyncio.wait(
[task1, task2]
, return_when=asyncio.FIRST_COMPLETED,)
loop = asyncio.get_event_loop()
loop.run_until_complete(main_coroutine())
loop.close()
I would like to stop a python asyncio task from another task and start it again when some condition in the second task happen.
I assume you control the task creation, but don't want to touch the implementation of the coroutine. In your case, you control coroutine2 and main_coroutine, but not the insides of coroutine1.
In that case you can wrap the coroutine in a an __await__ that, instead of the normal yield from loop, checkes your stopped flag and waits for a future that tells it when to resume.
class Stoppable:
def __init__(self, coro):
self._coro_iter = coro.__await__()
self._stopped = None
def __await__(self):
while True:
while self._stopped:
print('awaiting stopped')
yield from self._stopped.__await__()
try:
v = next(self._coro_iter)
except StopIteration as e:
return v
yield v
def stop(self):
loop = asyncio.get_event_loop()
self._stopped = loop.create_future()
def start(self):
if self._stopped is not None:
self._stopped.set_result(None)
self._stopped = None
You can use the wrapper to modify coroutine2 to stop and resume the execution of coroutine1 at will:
async def coroutine2(s):
i = 0
while True:
i += 1
if i == 3:
print('stopping coroutine1')
s.stop()
elif i == 10:
print('restarting coroutine1')
s.start()
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
loop = asyncio.get_event_loop()
s = Stoppable(coroutine1())
fut1 = asyncio.ensure_future(s)
task2 = loop.create_task(coroutine2(s))
done, pending = await asyncio.wait(
[fut1, task2], return_when=asyncio.FIRST_COMPLETED)
The way wrapper works is by unrolling the loop inherent in yield from. For example, to just delegate __await__ to another coroutine, one would write:
def __await__(self):
yield from self._coro_iter
Written like this, you can't implement stopping because the yield from contains an implicit loop that yields all the values produced by the underlying iterator - something like:
def __await__(self):
while True:
try:
v = next(self._coro_iter)
except StopIteration as e:
return e.value
yield v
Taken like this, it is easy enough to add an if that checks for _stopped at each iteration pass, meaning each time we're resumed by the event loop. The remaining hurdle is that one cannot just busy-loop until _stopped is rescinded - we must yield something else to allow the event loop to resume running other coroutines. Fortunately that is easily achieved by making _stopped a future, and yielding from the future. When the future's result is set, we will be automatically resumed and continue executing the wrapped coroutine.
It seems it can't be done.
It's possible to cancel an ongoing task with task1.cancel() and it's possible to create a new task with asyncio.get_event_loop().create_task(newTask).
It's also possible to get the coroutine of a running task with task1._coro but if we try to create a task again with a previously scheduled coroutine we will get a RuntimeError exception. This the discussion where they decided it: https://bugs.python.org/issue25887
Finally, a possible way of accomplishing the desire effect is using a asyncio.Queue object:
import asyncio
async def coroutine1(stop_queue):
i = 0
while(True):
if stop_queue.empty(): #if the queue is empty keep working.
i += 1
print("coroutine1: " + str(i) )
await asyncio.sleep(1)
async def coroutine2(stop_queue):
i = 0
while(True):
i += 1
if i == 3:
await stop_queue.put("whatever..") #put something in the queue
if i == 11:
await stop_queue.get() #take something from the queue
print("coroutine2: " + str(i) )
await asyncio.sleep(1)
async def main_coroutine():
stop_queue = asyncio.Queue()
done, pending = await asyncio.wait(
[coroutine1(stop_queue), coroutine2(stop_queue)]
, return_when=asyncio.ALL_COMPLETED,)
loop = asyncio.get_event_loop()
loop.run_until_complete(main_coroutine())
loop.close()
I'm new to python's ascynio feature and I have a server that processes websocket requests from a browser. Here's a simplified version of how it works:
#asyncio.coroutine
def web_client_connected(self, websocket):
self.web_client_socket = websocket
while True:
request = yield from self.web_client_socket.recv()
json_val = process_request(request)
yield from self.socket_queue.put(json_val)
#asyncio.coroutine
def push_from_web_client_json_queue(self):
while True:
json_val = yield from self.socket_queue.get()
yield from self.web_client_socket.send(json_val)
You have one loop looking for web socket requests coming in from the client. When it gets one, it processes it and puts the value onto a queue. Another loop is looking for values on that queue and when it finds one it sends processed value back out on the web socket. Pretty straight forward and it works.
What I want to do now it introduce a timer. When requests comes and and is done processing, instead of putting a response back on the queue immediately, I want to start a timer for 1 minute. When the timer is finished, then I want to put the response on the queue.
I've tried something like:
#asyncio.coroutine
def web_client_connected(self, websocket):
self.web_client_socket = websocket
while True:
request = yield from self.web_client_socket.recv()
json_val = process_request(request)
t = threading.Timer(60, self.timer_done, json_val)
t.start()
#asyncio.coroutine
def timer_done(self, args):
yield from self.socket_queue.put(args)
It doesn't work though. The timer_done method is never called. If I removed the #asyncio.coroutine decorator and yield from, then timer_done does get called but then call to self.socket_queue.put(args) doesn't work.
I think I'm misunderstanding something fundamental here. How do you do this?
Insted of a timer, use asyncio.ensure_future() and asyncio.sleep():
#asyncio.coroutine
def web_client_connected(self, websocket):
self.web_client_socket = websocket
while True:
request = yield from self.web_client_socket.recv()
json_val = process_request(request)
asyncio.ensure_future(web_client_timer(json_val))
yield
#asyncio.coroutine
def web_client_timer(self, json_val):
yield from asyncio.sleep(60)
yield from self.socket_queue.put(json_val)
Working example:
import asyncio
#asyncio.coroutine
def foo():
print("enter foo")
timers = []
for i in range(10):
print("Start foo", i)
yield from asyncio.sleep(0.5)
print("Got foo", i)
timers.append(asyncio.ensure_future(timer(i)))
yield
print("foo waiting")
# wait for all timers to finish
yield from asyncio.wait(timers)
print("exit foo")
#asyncio.coroutine
def timer(i):
print("Setting timer", i)
yield from asyncio.sleep(2)
print("**** Timer", i)
loop = asyncio.get_event_loop()
resp = loop.run_until_complete(foo())
loop.close()
I'm trying to write an asynchronous consumer using asyncio/aioamqp. My problem is, the callback coroutine (below) is blocking. I set the channel to do a basic_consume(), and assign the callback as callback(). The callback has a "yield from asyncio.sleep" statement (to simulate "work"), which takes an integer from the publisher and sleeps for that amount of time before printing the message.
If I published two messages, one with a time of "10", immediately followed by one with a time of "1", I expected the second message would print first, since it has a shorter sleep time. Instead, the callback blocks for 10 seconds, prints the first message, and then prints the second.
It appears either basic_consume, or the callback, is blocking somewhere. Is there another way this could be handled?
#asyncio.coroutine
def callback(body, envelope, properties):
yield from asyncio.sleep(int(body))
print("consumer {} recved {} ({})".format(envelope.consumer_tag, body, envelope.delivery_tag))
#asyncio.coroutine
def receive_log():
try:
transport, protocol = yield from aioamqp.connect('localhost', 5672, login="login", password="password")
except:
print("closed connections")
return
channel = yield from protocol.channel()
exchange_name = 'cloudstack-events'
exchange_name = 'test-async-exchange'
queue_name = 'async-queue-%s' % random.randint(0, 10000)
yield from channel.exchange(exchange_name, 'topic', auto_delete=True, passive=False, durable=False)
yield from asyncio.wait_for(channel.queue(queue_name, durable=False, auto_delete=True), timeout=10)
binding_keys = ['mykey']
for binding_key in binding_keys:
print("binding", binding_key)
yield from asyncio.wait_for(channel.queue_bind(exchange_name=exchange_name,
queue_name=queue_name,
routing_key=binding_key), timeout=10)
print(' [*] Waiting for logs. To exit press CTRL+C')
yield from channel.basic_consume(queue_name, callback=callback)
loop = asyncio.get_event_loop()
loop.create_task(receive_log())
loop.run_forever()
For those interested, I figured out a way to do this. I'm not sure if it's best practice, but it's accomplishing what I need.
Rather than do the "work" (in this case, async.sleep) inside the callback, I create a new task on the loop, and schedule a separate co-routine to run do_work(). Presumably this is working, because it's freeing up callback() to return immediately.
I loaded up a few hundred events in Rabbit with different sleep timers, and they were interleaved when printed by the code below. So it seems to be working. Hope this helps someone!
#asyncio.coroutine
def do_work(envelope, body):
yield from asyncio.sleep(int(body))
print("consumer {} recved {} ({})".format(envelope.consumer_tag, body, envelope.delivery_tag))
#asyncio.coroutine
def callback(body, envelope, properties):
loop = asyncio.get_event_loop()
loop.create_task(do_work(envelope, body))
#asyncio.coroutine
def receive_log():
try:
transport, protocol = yield from aioamqp.connect('localhost', 5672, login="login", password="password")
except:
print("closed connections")
return
channel = yield from protocol.channel()
exchange_name = 'cloudstack-events'
exchange_name = 'test-async-exchange'
queue_name = 'async-queue-%s' % random.randint(0, 10000)
yield from channel.exchange(exchange_name, 'topic', auto_delete=True, passive=False, durable=False)
yield from asyncio.wait_for(channel.queue(queue_name, durable=False, auto_delete=True), timeout=10)
binding_keys = ['mykey']
for binding_key in binding_keys:
print("binding", binding_key)
yield from asyncio.wait_for(channel.queue_bind(exchange_name=exchange_name,
queue_name=queue_name,
routing_key=binding_key), timeout=10)
print(' [*] Waiting for logs. To exit press CTRL+C')
yield from channel.basic_consume(queue_name, callback=callback)
loop = asyncio.get_event_loop()
loop.create_task(receive_log())
loop.run_forever()