I made a dummy server to test my websockets application against. It listens to subscription messages, then gives information about those subscriptions through the socket.
The class' subscriptions attribute is empty at initialisation and should fill up as the listen() function receives subscription messages. However, it seems as though self.subscriptions in talk() is never appended to, leaving it stuck in its infinite while-loop and never transmitting messages.
The problem is solved by adding a line await asyncio.sleep(1) after the for-loop. But why? Shouldn't self.subscriptions be re-evaluated every time the for-loop is started?
Code below:
class DummyServer:
def __init__(self):
self.subscriptions = []
def start(self):
return websockets.serve(self.handle, 'localhost', 8765)
async def handle(self, websocket, path):
self.ws = websocket
listen_task = asyncio.ensure_future(self.listen())
talk_task = asyncio.ensure_future(self.talk())
done, pending = await asyncio.wait(
[listen_task, talk_task],
return_when=asyncio.FIRST_COMPLETED
)
for task in pending:
task.cancel()
async def listen(self):
while True:
try:
msg = await self.ws.recv()
msg = json.loads(msg)
await self.triage(msg) # handles subscriptions
except Exception as e:
await self.close()
break
async def talk(self):
while True:
for s in self.subscriptions:
dummy_data = {
'product_id': s
}
try:
await self.send(json.dumps(dummy_data))
except Exception as e:
await self.close()
break
await asyncio.sleep(1) # without this line, no message is ever sent
At the start of your function, subscriptions is empty and the for body is not evaluated. As a result, your coroutine is virtually the same as:
async def talk(self):
while True:
pass
The while-loop does not contain a "context switching point", meaning that the asyncio event loop basically hangs there, forever executing the blocking while loop.
Adding await sleep() breaks the magic circle; even await sleep(0) could help.
Clever code should probably make use of asyncio.Condition in combination with self.subscriptions, but this matter goes beyond the scope of your original question.
Related
I'm trying to run some asynchronous functions in her asynchronous function, the problem is, how did I understand that functions don't run like that, then how do I do it? I don't want to make the maze_move function asynchronous.
async def no_stop():
#some logic
await asyncio.sleep(4)
async def stop(stop_time):
await asyncio.sleep(stop_time)
#some logic
def maze_move():
no_stop()
stop(1.5)
async def main(websocket):
global data_from_client, data_from_server, power_l, power_r
get_params()
get_data_from_server()
get_data_from_client()
while True:
msg = await websocket.recv()
allow_data(msg)
cheker(data_from_client)
data_from_server['IsBrake'] = data_from_client['IsBrake']
data_from_server['powerL'] = power_l
data_from_server['powerR'] = power_r
await websocket.send(json.dumps(data_from_server))
print(data_from_client['IsBrake'])
start_server = websockets.serve(main, 'localhost', 8080)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
How about:
def maze_move():
loop = asyncio.get_event_loop()
loop.run_until_complete(no_stop())
loop.run_until_complete(stop(1.5))
If you wanted to run two coroutines concurrently, then:
def maze_move():
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(no_stop(), stop(1.5)))
Update Based on Updated Question
I am guessing what it is you want to do (see my comment to your question):
First, you cannot call from maze_move coroutines such as stop directly since stop() does not result in calling stop it just returns a coroutine object. So maze_move has to be modified. I will assume you do not want to make it a coroutine itself (why not as long as you already have to modify it?). And further assuming you want to invoke maze_move from a coroutine that wishes to run concurrently other coroutines, then you can create a new coroutine, e.g. maze_move_runner that will run maze_move in a separate thread so that it does not block other concurrently running coroutines:
import asyncio
import concurrent.futures
async def no_stop():
#some logic
print('no stop')
await asyncio.sleep(4)
async def stop(stop_time):
await asyncio.sleep(stop_time)
print('stop')
#some logic
async def some_coroutine():
print('Enter some_coroutine')
await asyncio.sleep(2)
print('Exit some_coroutine')
return 1
def maze_move():
# In case we are being run directly and not in a separate thread:
try:
loop = asyncio.get_running_loop()
except:
# This thread has no current event loop, so:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(no_stop())
loop.run_until_complete(stop(1.5))
return 'Done!'
async def maze_move_runner():
loop = asyncio.get_running_loop()
# Run in another thread:
return await loop.run_in_executor(None, maze_move)
async def main():
loop = asyncio.get_running_loop()
results = await (asyncio.gather(some_coroutine(), maze_move_runner()))
print(results)
asyncio.run(main())
Prints:
Enter some_coroutine
no stop
Exit some_coroutine
stop
[1, 'Done!']
But this would be the most straightforward solution:
async def maze_move():
await no_stop()
await stop(1.5)
return 'Done!'
async def main():
loop = asyncio.get_running_loop()
results = await (asyncio.gather(some_coroutine(), maze_move()))
print(results)
If you have an already running event loop, you can define an async function inside of a sync function and launch it as task:
def maze_move():
async def amaze_move():
await no_stop()
await stop(1.5)
return asyncio.create_task(amaze_move())
This function returns an asyncio.Task object which can be used in an await expression, or not, depending on requirements. This way you won't have to make maze_move itself an async function, although I don't know why that would be a goal. Only a async function can run no_stop and stop, so you've got to have an async function somewhere.
I have a program that has to unconditionally respond after X seconds, that response can either be the actual response i need if under X seconds or some sort of "Gathering resources, please wait" message if the time has passed.
How can i do something along the lines of
make request
start timer
if request.responded:
return request
if timer.hasPassed(X):
return False
While still having the request active, ready to respond as soon as it is finished?
You seem to be looking for asyncio.wait_for. It accepts a coroutine and a timeout parameter, awaits the coroutine and raises asyncio.TimeoutError (as well as cancelling the coroutine) if the timeout period is exceeded. More here.
async def eternity():
# Sleep for one hour
await asyncio.sleep(3600)
print('yay!')
async def main():
# Wait for at most 1 second
try:
await asyncio.wait_for(eternity(), timeout=1.0)
except asyncio.TimeoutError:
print('timeout!')
If you want to keep the coroutine running after the timeout and eventually get its result you need to create a task and shield it from cancellation:
async def long_running_task():
await asyncio.sleep(5)
return True
async def main():
task = asyncio.create_task(long_running_task())
try:
return await asyncio.wait_for(asyncio.shield(task), timeout=1)
except asyncio.TimeoutError:
print('Time out')
return await task
I need to run an event every 60 seconds to all my cars. My problem with the code below is the while loop doesn't end until the timeout (60) does and hence only the first car in cars is run.
class RunCars(BaseEvent):
def __init__(self):
interval_seconds = 60 # Set the interval for this event
super().__init__(interval_seconds)
# run() method will be called once every {interval_seconds} minutes
async def run(self, client, cars):
for car in cars:
channel = get_channel(client, "general")
await client.send_message(channel, 'Running this '+str(car))
await msg.add_reaction(str(get_emoji(':smiley:')))
reaction = None
while True:
if str(reaction) == str(get_emoji(':smiley:'))
await client.send_message(channel, 'Finished with this '+str(car))
try:
reaction, user = await client.wait_for('reaction_add', timeout=60, check=check)
except:
break
I tried changing the code into a multithreaded process but had trouble with async/await inside the function and problems with pickling the function itself.
I'd appreciate any suggestions for how to go about this..
The asyncio module lets you execute multiple async method concurrently using the gather method. I think you can achieve the behavior you want by defining a method that runs a single car, and then replacing your for-loop with a call to gather, which will execute multiple run_one coroutines (methods) concurrently:
import asyncio
class RunCars(BaseEvent):
def __init__(self):
interval_seconds = 60 # Set the interval for this event
super().__init__(interval_seconds)
async def run(self, client, cars):
coroutines = [self.run_one(client, car) for car in cars]
asyncio.gather(*coroutines)
async def run_one(self, client, car):
channel = get_channel(client, "general")
await client.send_message(channel, 'Running this '+str(car))
await msg.add_reaction(str(get_emoji(':smiley:')))
reaction = None
while True:
if str(reaction) == str(get_emoji(':smiley:'))
await client.send_message(channel, 'Finished with this '+str(car))
try:
reaction, user = await client.wait_for('reaction_add', timeout=60, check=check)
except:
break
In general, when writing async code, you should try to replace sequential calls to async methods - basically for-loops that call async methods - with gather statements so they execute concurrently.
I'm trying to figure out if it's possible throw a custom exception into a running asyncio task, similarly to what is achieved by Task.cancel(self) which schedules a CancelledError to be raised in the underlying coroutine.
I came across Task.get_coro().throw(exc), but calling it seems like opening a big can of worms as we may leave the task in a bad state. Especially considering all the machinery that happens when a task is throwing CancelledError into its coroutine.
Consider the following example:
import asyncio
class Reset(Exception):
pass
async def infinite():
while True:
try:
print('work')
await asyncio.sleep(1)
print('more work')
except Reset:
print('reset')
continue
except asyncio.CancelledError:
print('cancel')
break
async def main():
infinite_task = asyncio.create_task(infinite())
await asyncio.sleep(0) # Allow infinite_task to enter its work loop.
infinite_task.get_coro().throw(Reset())
await infinite_task
asyncio.run(main())
## OUTPUT ##
# "work"
# "reset"
# "work"
# hangs forever ... bad :(
Is what I try to do even feasible? It feels as if I shouldn't be manipulating the underlying coroutine like this. Any workaround?
There's no way to throw a custom exception into a running task. You shouldn't mess with .throw - it's a detail of implementation and altering it will probably break something.
If you want to pass information (about reset) into the task, do it trough an argument. Here's how it can be implemented:
import asyncio
from contextlib import suppress
async def infinite(need_reset):
try:
while True:
inner_task = asyncio.create_task(inner_job())
await asyncio.wait(
[
need_reset.wait(),
inner_task
],
return_when=asyncio.FIRST_COMPLETED
)
if need_reset.is_set():
print('reset')
await cancel(inner_task)
need_reset.clear()
except asyncio.CancelledError:
print('cancel')
raise # you should never suppress, see:
# https://stackoverflow.com/a/33578893/1113207
async def inner_job():
print('work')
await asyncio.sleep(1)
print('more work')
async def cancel(task):
# more info: https://stackoverflow.com/a/43810272/1113207
task.cancel()
with suppress(asyncio.CancelledError):
await task
async def main():
need_reset = asyncio.Event()
infinite_task = asyncio.create_task(infinite(need_reset))
await asyncio.sleep(1.5)
need_reset.set()
await asyncio.sleep(1.5)
await cancel(infinite_task)
asyncio.run(main())
Output:
work
more work
work
reset
work
more work
work
cancel
I'm trying to write some tests for some asynchronous Python code using the aioamqp message broker, but pytest and callbacks fail me.
Simply put, when the aioamqp basic_consume() function receives a message and calls the assigned asynchronous callback, inside the callback I can do whatever I like -- reference unassigned variables, assert something outrageous -- and pytest happily passes the test. Clearly an exception gets raised under the hood and the test is interrupted, since the callback function never runs further than the first failing line, but the failure never rises all the way to pytest.
Here's a code snippet to demonstrate:
import aioamqp
import asyncio
import pytest
MQ_HOST = '0.0.0.0'
MQ_PORT = 5672
MQ_LOGIN = 'login'
MQ_PASSWORD = 'password'
class MockMQ:
def __init__(self):
self.loop = asyncio.get_event_loop()
self.transport = None
self.protocol = None
async def connect(self):
try:
self.transport, self.protocol = await aioamqp.connect(
host=MQ_HOST, port=MQ_PORT, login=MQ_LOGIN, password=MQ_PASSWORD
)
self.channel = await self.protocol.channel()
except aioamqp.AmqpClosedConnection:
print('closed connection')
return
async def close(self):
await self.protocol.close()
self.transport.close()
async def publish(self, data, queue_name, exchange='', properties=None):
queue = await self.channel.queue_declare(queue_name)
await self.channel.publish(data, exchange, queue_name, properties=properties)
async def consume(self, callback, queue_name):
await self.channel.basic_consume(callback, queue_name=queue_name)
#pytest.mark.asyncio
async def test_mq():
"""Basic ping-pong test for RabbitMQ."""
QUEUE_NAME = 'my_queue'
#pytest.mark.asyncio
async def callback(channel, body, envelope, properties):
"""This is the callback called when a MQ message is consumed."""
print('we are here')
await channel.basic_client_ack(envelope.delivery_tag)
print(body) # this gets printed as well
foo = bar * 2 # this is where we fail
assert body == b'bar'
print('we never arrive here')
mq = MockMQ()
await mq.connect()
await mq.consume(callback, QUEUE_NAME)
await mq.publish(b'foo', QUEUE_NAME)
await asyncio.sleep(1.0)
await mq.close()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(test_mq())
Running this via the main program with IPython results correctly in an exception, since it doesn't get swallowed by pytest.
What is the proper way of writing tests for pytest in this case? pytest-asyncio does not seem to affect this issue in the least.
EDIT: I might as well add that my dev environment uses Django and pytest-django, but removing it doesn't change the result either.