I have the following scenario:
from time import sleep
async def do_a(a):
sleep(0.001)
return 1*a
async def do_b(a):
sleep(0.01)
return 2*a
async def do_c(b):
sleep(1)
return 3*b
async def my_func():
results = []
for i in range(3):
a = await do_a(i)
b = await do_b(a)
c = await do_c(b)
results.append(c)
return results
if __name__ == "__main__":
import asyncio
print(asyncio.run(my_func()))
Basically, I am calling asynchronous functions in a loop. Executing the above code shows it run in ~3s. I would like to call each procedure in parallel, so the expected time would drop to ~1s (I know this overhead is a little bit too optimistic, but to optimize the running time at least a bit). I have been looking into different python libraries that I think could help, but having trouble deciding which one is useful in this case. Python's multiprocessing, threading and concurrent.futures all seem to implement one form or another of parallelism/concurrency.
What should I do? Can you show me how you would proceed in this case?
You should use asyncio.sleep instead of time.sleep. If you want everything to run concurrently, this is one way you can do it with asyncio.gather:
import asyncio
async def do_a(a):
await asyncio.sleep(0.001)
return 1*a
async def do_b(a):
await asyncio.sleep(0.01)
return 2*a
async def do_c(b):
await asyncio.sleep(1)
return 3*b
async def do_abc(i):
a = await do_a(i)
b = await do_b(a)
return await do_c(b)
async def my_func():
return await asyncio.gather(*map(do_abc, range(3)))
if __name__ == "__main__":
import asyncio
print(asyncio.run(my_func()))
# [0, 6, 12]
If the actual code that runs instead of sleep is synchronous (blocking), you would do essentially the same, only you would have to defer that work to an executor.
Related
For example this code:
async def f1(num):
while True:
print(num)
await asyncio.sleep(2)
class ExampleClass:
def __init__():
self.tasks = []
async def main():
for i in range(10):
tasks.append(asyncio.create_task(f1(i)))
await asyncio.gather(*tasks)
def add_new_task(task):
self.tasks.append(task)
Then somewhere outside I call
ExampleClass.add_new_task(task)
What I need is to add new tasks and execute them asynchronously with the existing ones.
May be I should use any other constructions to implement what i want?
What is important is that my tasks probably need to execute forever(forever polling)
The asyncio.gather function is not really convenient for such task. However, you can take a look at asyncio.TaskGroup which was released in Python3.11 and allows to add new tasks dynamically more easily.
import asyncio
from concurrent.futures import wait, FIRST_COMPLETED
async def main():
async with asyncio.TaskGroup() as group:
# Create some tasks
tasks = [
group.create_task(asyncio.sleep(1.0))
for _ in range(10)
]
# Wait for some tasks to finish
done, tasks = await wait(tasks, return_when=FIRST_COMPLETED)
# Add more tasks (/!\ `tasks` is now a set)
tasks.add(group.create_task(asyncio.sleep(2.0)))
# Wait the rest to complete
await asyncio.gather(*tasks)
if __name__ == "__main__":
asyncio.run(main())
So, I come to possible solution for my case with the idea of Louis.
The approximate idea is to use fake async linfinite loop where all my task are executed and where I update new tasks within asyncio.TaskGroup :
async def fake_generator():
while True:
yield None
fake_gen = fake_generator()
async with asyncio.TaskGroup() as tg:
async for _ in fake_gen:
await add_new_data()
for element in self.data:
if not self.data[element]['in_use']:
tg.create_task(job(element))
self.data[element]['in_use'] = True
await asyncio.sleep(60 * 60)
I'm trying to run some asynchronous functions in her asynchronous function, the problem is, how did I understand that functions don't run like that, then how do I do it? I don't want to make the maze_move function asynchronous.
async def no_stop():
#some logic
await asyncio.sleep(4)
async def stop(stop_time):
await asyncio.sleep(stop_time)
#some logic
def maze_move():
no_stop()
stop(1.5)
async def main(websocket):
global data_from_client, data_from_server, power_l, power_r
get_params()
get_data_from_server()
get_data_from_client()
while True:
msg = await websocket.recv()
allow_data(msg)
cheker(data_from_client)
data_from_server['IsBrake'] = data_from_client['IsBrake']
data_from_server['powerL'] = power_l
data_from_server['powerR'] = power_r
await websocket.send(json.dumps(data_from_server))
print(data_from_client['IsBrake'])
start_server = websockets.serve(main, 'localhost', 8080)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
How about:
def maze_move():
loop = asyncio.get_event_loop()
loop.run_until_complete(no_stop())
loop.run_until_complete(stop(1.5))
If you wanted to run two coroutines concurrently, then:
def maze_move():
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(no_stop(), stop(1.5)))
Update Based on Updated Question
I am guessing what it is you want to do (see my comment to your question):
First, you cannot call from maze_move coroutines such as stop directly since stop() does not result in calling stop it just returns a coroutine object. So maze_move has to be modified. I will assume you do not want to make it a coroutine itself (why not as long as you already have to modify it?). And further assuming you want to invoke maze_move from a coroutine that wishes to run concurrently other coroutines, then you can create a new coroutine, e.g. maze_move_runner that will run maze_move in a separate thread so that it does not block other concurrently running coroutines:
import asyncio
import concurrent.futures
async def no_stop():
#some logic
print('no stop')
await asyncio.sleep(4)
async def stop(stop_time):
await asyncio.sleep(stop_time)
print('stop')
#some logic
async def some_coroutine():
print('Enter some_coroutine')
await asyncio.sleep(2)
print('Exit some_coroutine')
return 1
def maze_move():
# In case we are being run directly and not in a separate thread:
try:
loop = asyncio.get_running_loop()
except:
# This thread has no current event loop, so:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(no_stop())
loop.run_until_complete(stop(1.5))
return 'Done!'
async def maze_move_runner():
loop = asyncio.get_running_loop()
# Run in another thread:
return await loop.run_in_executor(None, maze_move)
async def main():
loop = asyncio.get_running_loop()
results = await (asyncio.gather(some_coroutine(), maze_move_runner()))
print(results)
asyncio.run(main())
Prints:
Enter some_coroutine
no stop
Exit some_coroutine
stop
[1, 'Done!']
But this would be the most straightforward solution:
async def maze_move():
await no_stop()
await stop(1.5)
return 'Done!'
async def main():
loop = asyncio.get_running_loop()
results = await (asyncio.gather(some_coroutine(), maze_move()))
print(results)
If you have an already running event loop, you can define an async function inside of a sync function and launch it as task:
def maze_move():
async def amaze_move():
await no_stop()
await stop(1.5)
return asyncio.create_task(amaze_move())
This function returns an asyncio.Task object which can be used in an await expression, or not, depending on requirements. This way you won't have to make maze_move itself an async function, although I don't know why that would be a goal. Only a async function can run no_stop and stop, so you've got to have an async function somewhere.
How do I remove the async-everywhere insanity in a program like this?
import asyncio
async def async_coro():
await asyncio.sleep(1)
async def sync_func_1():
# This is blocking and synchronous
await async_coro()
async def sync_func_2():
# This is blocking and synchronous
await sync_func_1()
if __name__ == "__main__":
# Async pollution goes all the way to __main__
asyncio.run(sync_func_2())
I need to have 3 async markers and asyncio.run at the top level just to call one async function. I assume I'm doing something wrong - how can I clean up this code to make it use async less?
FWIW, I'm interested mostly because I'm writing an API using asyncio and I don't want my users to have to think too much about whether their functions need to be def or async def depending on whether they're using a async part of the API or not.
After some research, one answer is to manually manage the event loop:
import asyncio
async def async_coro():
await asyncio.sleep(1)
def sync_func_1():
# This is blocking and synchronous
loop = asyncio.get_event_loop()
coro = async_coro()
loop.run_until_complete(coro)
def sync_func_2():
# This is blocking and synchronous
sync_func_1()
if __name__ == "__main__":
# No more async pollution
sync_func_2()
If you must do that, I would recommend an approach like this:
import asyncio, threading
async def async_coro():
await asyncio.sleep(1)
_loop = asyncio.new_event_loop()
threading.Thread(target=_loop.run_forever, daemon=True).start()
def sync_func_1():
# This is blocking and synchronous
return asyncio.run_coroutine_threadsafe(async_coro(), _loop).result()
def sync_func_2():
# This is blocking and synchronous
sync_func_1()
if __name__ == "__main__":
sync_func_2()
The advantage of this approach compared to one where sync functions run the event loop is that it supports nesting of sync functions. It also only runs a single event loop, so that if the underlying library wants to set up e.g. a background task for monitoring or such, it will work continuously rather than being spawned each time anew.
I mean what do I get from using async for. Here is the code I write with async for, AIter(10) could be replaced with get_range().
But the code runs like sync not async.
import asyncio
async def get_range():
for i in range(10):
print(f"start {i}")
await asyncio.sleep(1)
print(f"end {i}")
yield i
class AIter:
def __init__(self, N):
self.i = 0
self.N = N
def __aiter__(self):
return self
async def __anext__(self):
i = self.i
print(f"start {i}")
await asyncio.sleep(1)
print(f"end {i}")
if i >= self.N:
raise StopAsyncIteration
self.i += 1
return i
async def main():
async for p in AIter(10):
print(f"finally {p}")
if __name__ == "__main__":
asyncio.run(main())
The result I excepted should be :
start 1
start 2
start 3
...
end 1
end 2
...
finally 1
finally 2
...
However, the real result is:
start 0
end 0
finally 0
start 1
end 1
finally 1
start 2
end 2
I know I could get the excepted result by using asyncio.gather or asyncio.wait.
But it is hard for me to understand what I got by use async for here instead of simple for.
What is the right way to use async for if I want to loop over several Feature object and use them as soon as one is finished. For example:
async for f in feature_objects:
data = await f
with open("file", "w") as fi:
fi.write()
But it is hard for me to understand what I got by use async for here instead of simple for.
The underlying misunderstanding is expecting async for to automatically parallelize the iteration. It doesn't do that, it simply allows sequential iteration over an async source. For example, you can use async for to iterate over lines coming from a TCP stream, messages from a websocket, or database records from an async DB driver.
None of the above would work with an ordinary for, at least not without blocking the event loop. This is because for calls __next__ as a blocking function and doesn't await its result. You cannot manually await elements obtained by for because for expects __next__ to signal the end of iteration by raising StopIteration. If __next__ is a coroutine, the StopIteration exception won't be visible before awaiting it. This is why async for was introduced, not just in Python, but also in other languages with async/await and generalized for.
If you want to run the loop iterations in parallel, you need to start them as parallel coroutines and use asyncio.as_completed or equivalent to retrieve their results as they come:
async def x(i):
print(f"start {i}")
await asyncio.sleep(1)
print(f"end {i}")
return i
# run x(0)..x(10) concurrently and process results as they arrive
for f in asyncio.as_completed([x(i) for i in range(10)]):
result = await f
# ... do something with the result ...
If you don't care about reacting to results immediately as they arrive, but you need them all, you can make it even simpler by using asyncio.gather:
# run x(0)..x(10) concurrently and process results when all are done
results = await asyncio.gather(*[x(i) for i in range(10)])
(Adding on the accepted answer - for Charlie's bounty).
Assuming you want to consume each yielded value concurrently, a straightforward way would be:
import asyncio
async def process_all():
tasks = []
async for obj in my_async_generator:
# Python 3.7+. Use ensure_future for older versions.
task = asyncio.create_task(process_obj(obj))
tasks.append(task)
await asyncio.gather(*tasks)
async def process_obj(obj):
...
Explanation:
Consider the following code, without create_task:
async def process_all():
async for obj in my_async_generator:
await process_obj(obj))
This is roughly equivalent to:
async def process_all():
obj1 = await my_async_generator.__anext__():
await process_obj(obj1))
obj2 = await my_async_generator.__anext__():
await process_obj(obj1))
...
Basically, the loop cannot continue because its body is blocking. The way to go is to delegate the processing of each iteration to a new asyncio task which will start without blocking the loop. The, gather wait for all of the tasks - which means, for every iteration to be processed.
Code based on fantastic answer from #matan129, just missing the async generator to make it runnable, once I have that (or if someone wants to contributed it) will finilize this:
import time
import asyncio
async def process_all():
"""
Example where the async for loop allows to loop through concurrently many things without blocking on each individual
iteration but blocks (waits) for all tasks to run.
ref:
- https://stackoverflow.com/questions/56161595/how-to-use-async-for-in-python/72758067#72758067
"""
tasks = []
async for obj in my_async_generator:
# Python 3.7+. Use ensure_future for older versions.
task = asyncio.create_task(process_obj(obj)) # concurrently dispatches a coroutine to be executed.
tasks.append(task)
await asyncio.gather(*tasks)
async def process_obj(obj):
await asyncio.sleep(5) # expensive IO
if __name__ == '__main__':
# - test asyncio
s = time.perf_counter()
asyncio.run(process_all())
# - print stats
elapsed = time.perf_counter() - s
print(f"{__file__} executed in {elapsed:0.2f} seconds.")
print('Success, done!\a')
I want to make a decorator for a project of mine that every x seconds certain telethon task will be started.
I've asked in the telethon group and someone give me a small decorator but the problem here is that I need to start the loop using run_until_complete and I already use it when starting the client. Here is my code:
def periodic(period):
def scheduler(fcn):
async def wrapper():
while True:
asyncio.ensure_future(fcn())
await asyncio.sleep(period)
return wrapper
return scheduler
#periodic(2)
async def do_something():
await asyncio.sleep(5) # Do some heavy calculation
me = await client.get_me()
print(me.stingfy())
Now I already have a loop running in main:
if __name__ == "__main__":
async def start():
await client.start()
await client.get_me()
await client.run_until_disconnected()
loop = asyncio.get_event_loop()
loop.run_until_complete(start())
and I can't run another loop because if I do that it seems to close this one. Any ideas about how to make this work?
It looks like you desire to run multiple functions at same time. So you can use asyncio.Task or asyncio.create_task to create tasks and add them into list, then run them through using asyncio.wait or etc.
import asyncio
def periodic(period):
def scheduler(fcn):
async def wrapper():
while 1:
asyncio.ensure_future(fcn())
await asyncio.sleep(period)
return wrapper
return scheduler
#periodic(2)
async def do_something():
print("Im running")
async def client():
while 1:
await asyncio.sleep(1)
print("This is client")
if __name__ == "__main__":
async def start():
task = [asyncio.Task(client()),
asyncio.Task(do_something())]
done, pending = await asyncio.wait(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(start())