I was wondering how I could use asyncio to handle tasks similar to what nodeJS does. I want to run tasks at the same time without opening threads.
Example:
import asyncio
#asyncio.coroutine
def my_coroutine(task_name, seconds_to_sleep=3):
print('{0} sleeping for: {1} seconds'.format(task_name, seconds_to_sleep))
yield from asyncio.sleep(seconds_to_sleep)
print('{0} is finished'.format(task_name))
loop = asyncio.get_event_loop()
tasks = [
my_coroutine('task1', 4),
my_coroutine('task2', 3),
my_coroutine('task3', 2)]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
will output:
task1 sleeping for: 4 seconds
task2 sleeping for: 3 seconds
task3 sleeping for: 2 seconds
task3 is finished
task2 is finished
task1 is finished
but when I try to do it with a different task it won't work like that.
import asyncio
import timeit
#asyncio.coroutine
def my_coroutine(task_name):
print('order placed for ' + task_name)
print(timeit.timeit('1 + 3 ', number=50000000))
print(task_name + ' done')
loop = asyncio.get_event_loop()
tasks = [
my_coroutine('task1'),
my_coroutine('task2'),
my_coroutine('task3')]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
outputs
order placed for task2
0.6677237730912453
task2 done
order placed for task1
0.6627442526498016
task1 done
order placed for task3
0.665618849882418
task3 done
asyncio doesn't run things in parallel. It runs one task until it awaits, then moves on to the next. The sleeps in your first example are what make the tasks yield control to each other. Your second example doesn't await anything, so each task runs until completion before the event loop can give control to another task.
If you add something awaitable (e.g., asyncio.sleep) into your coroutine, each one will yield control and give the others a chance to run.
#asyncio.coroutine
def my_coroutine(task_name):
print('order placed for ' + task_name)
yield from asyncio.sleep(0) # Another coroutine will resume here.
print(timeit.timeit('1 + 3 ', number=50000000))
yield from asyncio.sleep(0) # Another coroutine will resume here.
print(task_name + ' done')
The asyncio documentation says below so asyncio tasks run concurrently but not parallelly.
asyncio is a library to write concurrent code using the async/await
syntax.
And, #asyncio.coroutine is deprecated since Python 3.7.14 and removed since Python 3.11.0 so instead, you should use async as shown below:
# #asyncio.coroutine
async def test():
print("Test")
And for example, with this code below:
import asyncio
async def test1():
for _ in range(0, 3):
print("Test1")
async def test2():
for _ in range(0, 3):
print("Test2")
async def test3():
for _ in range(0, 3):
print("Test3")
async def call_tests():
await asyncio.gather(test1(), test2(), test3())
asyncio.run(call_tests())
test1(), test2() and test3() are run serially as shown below:
Test1 # 0 second
Test1 # 0 second
Test1 # 0 second
Test2 # 0 second
Test2 # 0 second
Test2 # 0 second
Test3 # 0 second
Test3 # 0 second
Test3 # 0 second
And, if using await asyncio.sleep(1) in them as shown below:
import asyncio
async def test1():
for _ in range(0, 3):
print("Test1")
await asyncio.sleep(1) # Here
async def test2():
for _ in range(0, 3):
print("Test2")
await asyncio.sleep(1) # Here
async def test3():
for _ in range(0, 3):
print("Test3")
await asyncio.sleep(1) # Here
async def call_tests():
await asyncio.gather(test1(), test2(), test3())
asyncio.run(call_tests())
They are run alternately sleeping 1 second each time as shown below:
Test1 # 1 second
Test2 # 1 second
Test3 # 1 second
Test1 # 2 seconds
Test2 # 2 seconds
Test3 # 2 seconds
Test1 # 3 seconds
Test2 # 3 seconds
Test3 # 3 seconds
And, if using await asyncio.sleep(0) in them as shown below:
import asyncio
async def test1():
for _ in range(0, 3):
print("Test1")
await asyncio.sleep(0) # Here
async def test2():
for _ in range(0, 3):
print("Test2")
await asyncio.sleep(0) # Here
async def test3():
for _ in range(0, 3):
print("Test3")
await asyncio.sleep(0) # Here
async def call_tests():
await asyncio.gather(test1(), test2(), test3())
asyncio.run(call_tests())
They are run alternately without sleeping as shown below:
Test1 # 0 second
Test2 # 0 second
Test3 # 0 second
Test1 # 0 second
Test2 # 0 second
Test3 # 0 second
Test1 # 0 second
Test2 # 0 second
Test3 # 0 second
Related
I'm trying to run 2 async functions test1() and test2() with loop.run_until_complete() alternately in Python as shown below:
import asyncio
async def test1():
for _ in range(3):
print("Test1")
await asyncio.sleep(1)
async def test2():
for _ in range(3):
print("Test2")
await asyncio.sleep(1)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(test1()) # Here
loop.run_until_complete(test2()) # Here
But as shown below, they don't run with loop.run_until_complete() alternately:
Test1
Test1
Test1
Test2
Test2
Test2
I know that if I use loop.run_forever() with loop.create_task() as shown below:
import asyncio
async def test1(loop):
for _ in range(3):
print("Test1")
await asyncio.sleep(1)
loop.stop() # Extra code to stop "loop.run_forever()"
async def test2(loop):
for _ in range(3):
print("Test2")
await asyncio.sleep(1)
loop.stop() # Extra code to stop "loop.run_forever()"
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.create_task(test1(loop)) # Here
loop.create_task(test2(loop)) # Here
loop.run_forever() # Here
I can run them alternately as shown below but loop.run_forever() runs forever so to stop loop.run_forever(), the extra code loop.stop() is needed which is troublesome. In addition, I know that asyncio.gather() can also run them alternately but it needs await which I don't want:
Test1
Test2
Test1
Test2
Test1
Test2
So, how can I run them with loop.run_until_complete() alternately?
If you wouldn't insist on the loop.run_until_complete you can achieve what you want to get by using the asyncio.gather functionality, just like so:
import asyncio
async def test1():
for _ in range(3):
print("Test1")
await asyncio.sleep(1)
async def test2():
for _ in range(3):
print("Test2")
await asyncio.sleep(1)
async def main():
tasks = [test1(), test2()]
new_items = await asyncio.gather(*tasks)
return new_items
if __name__ == '__main__':
results = asyncio.run(main())
and the results would be as you'd expect-
==> python3 stack_overflow.py
Test1
Test2
Test1
Test2
Test1
Test2
You can run 2 async functions test1() and test2() alternately by calling the intermediate async function call_tests() with loop.run_until_complete() as shown below. You also need to use asyncio.get_running_loop() and loop.create_task() in call_tests() and await is needed for the last loop.create_task() to run them alternately as shown below:
import asyncio
async def test1():
for _ in range(3):
print("Test1")
await asyncio.sleep(1)
async def test2():
for _ in range(3):
print("Test2")
await asyncio.sleep(1)
async def call_tests(): # Here
loop = asyncio.get_running_loop() # Here
loop.create_task(test1()) # Here
await loop.create_task(test2()) # "await" is needed
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(call_tests()) # Call "call_tests()"
Finally, you can run them alternately as shown below:
Test1
Test2
Test1
Test2
Test1
Test2
Be careful, if using await for the first and last loop.create_task() both as shown below:
# ...
async def call_tests():
loop = asyncio.get_running_loop()
await loop.create_task(test1()) # Here
await loop.create_task(test2()) # Here
# ...
You cannot run them alternately as shown below:
Test1
Test1
Test1
Test2
Test2
Test2
And, if using await for the first loop.create_task() as shown below:
# ...
async def call_tests():
loop = asyncio.get_running_loop()
await loop.create_task(test1()) # Here
loop.create_task(test2())
# ...
You cannot run them alternately and the last loop.create_task() is exited without completed as shown below:
Test1
Test1
Test1
Test2
And, if not using await for the first and last loop.create_task() both as shown below:
# ...
async def call_tests():
loop = asyncio.get_running_loop()
loop.create_task(test1()) # No "await"
loop.create_task(test2()) # No "await"
# ...
You can run them alternately but the first and last loop.create_task() both are exited without completed as shown below:
Test1
Test2
In addition, if using asyncio.gather(), call_tests() is not needed as shown below:
import asyncio
async def test1():
for _ in range(3):
print("Test1")
await asyncio.sleep(1)
async def test2():
for _ in range(3):
print("Test2")
await asyncio.sleep(1)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(asyncio.gather(test1(), test2()))
Then, you can run them alternately as shown below:
Test1
Test2
Test1
Test2
Test1
Test2
When I go to the asyncio page, the first example is a hello world program. When I run it on python 3.73, I can't see any different from the normal one. can anyone tell me the difference and give a non-trivial example?
In [1]: import asyncio
...:
...: async def main():
...: print('Hello ...')
...: await asyncio.sleep(5)
...: print('... World!')
...:
...: # Python 3.7+
...: asyncio.run(main())
Hello ...
... World!
In [2]:
In [2]: import time
...:
...: def main():
...: print('Hello ...')
...: time.sleep(5)
...: print('... World!')
...:
...: # Python 3.7+
...: main()
Hello ...
... World!
I intentionally increase the time from 1s to 5s, hope to see something special but I didn't.
You aren't seeing anything special because there's nothing much asynchronous work in your code. However, the main difference is that time.sleep(5) is blocking, and asyncio.sleep(5) is non-blocking.
When time.sleep(5) is called, it will block the entire execution of the script and it will be put on hold, just frozen, doing nothing. But when you call await asyncio.sleep(5), it will ask the event loop to run something else while your await statement finishes its execution.
Here's an improved example.
import asyncio
async def hello():
print('Hello ...')
await asyncio.sleep(1)
print('... World!')
async def main():
await asyncio.gather(hello(), hello())
asyncio.run(main())
Will output:
~$ python3.7 async.py
Hello ...
Hello ...
... World!
... World!
You can see that await asyncio.sleep(1) is not blocking the execution of the script.
In contrast, replacing the line await asyncio.sleep(1) with time.sleep(1), the output will be
Hello ...
... World!
Hello ...
... World!
because time.sleep is blocking and the first call of hello() has to finish first before the second call of hello() starts running.
With time.sleep(1) below, first, test1() is run every one second, then test2() is run every one second:
import asyncio
import time
async def test1():
for _ in range(0, 3):
print('Test1')
time.sleep(1) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
time.sleep(1) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 6 seconds are taken to run test1() and test2() in total:
Test1 # 1 second
Test1 # 2 seconds
Test1 # 3 seconds
Test2 # 4 seconds
Test2 # 5 seconds
Test2 # 6 seconds
With asyncio.sleep(1) below, test1() and test2() are run every one second alternately:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
await asyncio.sleep(1) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
await asyncio.sleep(1) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, only 3 seconds are taken to run test1() and test2() in total:
Test1 # 1 second
Test2 # 1 second
Test1 # 2 seconds
Test2 # 2 seconds
Test1 # 3 seconds
Test2 # 3 seconds
And, with time.sleep(0) below, first, test1() is run at once, then test2() is run at once:
import asyncio
import time
async def test1():
for _ in range(0, 3):
print('Test1')
time.sleep(0) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
time.sleep(0) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test1 # 0 second
Test1 # 0 second
Test2 # 0 second
Test2 # 0 second
Test2 # 0 second
And, with asyncio.sleep(0) below, test1() and test2() are run at once alternately:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
await asyncio.sleep(0) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
await asyncio.sleep(0) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, only 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test2 # 0 second
Test1 # 0 second
Test2 # 0 second
Test1 # 0 second
Test2 # 0 second
Lastly, without time.sleep() or asyncio.sleep() below, first, test1() is run at once, then test2() is run at once:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
async def test2():
for _ in range(0, 3):
print('Test2')
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test1 # 0 second
Test1 # 0 second
Test2 # 0 second
Test2 # 0 second
Test2 # 0 second
How to get the someone task's return immediately not until all task completed in [asyncio] ?
import asyncio
import time
print(f"now: {time.strftime('%X')}")
async def test1():
print(f"test1 started at {time.strftime('%X')}")
await asyncio.sleep(5)
with open('test.txt', 'w') as f:
f.write('...')
return 'end....'
async def test2(num):
print(f"test2 started at {time.strftime('%X')}")
return num * num
async def main(num):
res = await asyncio.gather(test1(), test2(num))
return res
def my(num):
return asyncio.run(main(num))[1]
print(my(5))
print(f"all end at {time.strftime('%X')}")
From the above code(python 3.7+), I can only get test2 return after test1 and test2 are all completed.
How can I let the main function get test2 return after test2 is completed, instead of waiting until test1 is completed?, because test2 is executed more quickly. And test1 must be executed(generate test.txt file.)
Than means return (test2's return) to main function or my function as soon as possible when test1 and test2 is asynchronous.
To run a set of awaitables/coroutines until any Future/Task finishes or is canceled - you need asyncio.wait coroutine:
...
async def main(num):
done, pending = await asyncio.wait([test1(), test2(num)],
return_when=asyncio.FIRST_COMPLETED)
for coro in done:
return await coro
def my(num):
return asyncio.run(main(num))
print(my(5))
print(f"all end at {time.strftime('%X')}")
return_when indicates when this function should return
The output:
now: 18:50:16
test1 started at 18:50:16
test2 started at 18:50:16
25
all end at 18:50:16
But since you need all coroutines to be completed - use asyncio.as_completed approach OR print results from done set, then - await from pending set and change print(my(5)) to just my(5):
...
async def main(num):
done, pending = await asyncio.wait([test1(), test2(num)],
return_when=asyncio.FIRST_COMPLETED)
for coro in done:
print(await coro)
for coro in pending:
await coro
def my(num):
return asyncio.run(main(num))
my(5)
print(f"all end at {time.strftime('%X')}")
When I go to the asyncio page, the first example is a hello world program. When I run it on python 3.73, I can't see any different from the normal one. can anyone tell me the difference and give a non-trivial example?
In [1]: import asyncio
...:
...: async def main():
...: print('Hello ...')
...: await asyncio.sleep(5)
...: print('... World!')
...:
...: # Python 3.7+
...: asyncio.run(main())
Hello ...
... World!
In [2]:
In [2]: import time
...:
...: def main():
...: print('Hello ...')
...: time.sleep(5)
...: print('... World!')
...:
...: # Python 3.7+
...: main()
Hello ...
... World!
I intentionally increase the time from 1s to 5s, hope to see something special but I didn't.
You aren't seeing anything special because there's nothing much asynchronous work in your code. However, the main difference is that time.sleep(5) is blocking, and asyncio.sleep(5) is non-blocking.
When time.sleep(5) is called, it will block the entire execution of the script and it will be put on hold, just frozen, doing nothing. But when you call await asyncio.sleep(5), it will ask the event loop to run something else while your await statement finishes its execution.
Here's an improved example.
import asyncio
async def hello():
print('Hello ...')
await asyncio.sleep(1)
print('... World!')
async def main():
await asyncio.gather(hello(), hello())
asyncio.run(main())
Will output:
~$ python3.7 async.py
Hello ...
Hello ...
... World!
... World!
You can see that await asyncio.sleep(1) is not blocking the execution of the script.
In contrast, replacing the line await asyncio.sleep(1) with time.sleep(1), the output will be
Hello ...
... World!
Hello ...
... World!
because time.sleep is blocking and the first call of hello() has to finish first before the second call of hello() starts running.
With time.sleep(1) below, first, test1() is run every one second, then test2() is run every one second:
import asyncio
import time
async def test1():
for _ in range(0, 3):
print('Test1')
time.sleep(1) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
time.sleep(1) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 6 seconds are taken to run test1() and test2() in total:
Test1 # 1 second
Test1 # 2 seconds
Test1 # 3 seconds
Test2 # 4 seconds
Test2 # 5 seconds
Test2 # 6 seconds
With asyncio.sleep(1) below, test1() and test2() are run every one second alternately:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
await asyncio.sleep(1) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
await asyncio.sleep(1) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, only 3 seconds are taken to run test1() and test2() in total:
Test1 # 1 second
Test2 # 1 second
Test1 # 2 seconds
Test2 # 2 seconds
Test1 # 3 seconds
Test2 # 3 seconds
And, with time.sleep(0) below, first, test1() is run at once, then test2() is run at once:
import asyncio
import time
async def test1():
for _ in range(0, 3):
print('Test1')
time.sleep(0) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
time.sleep(0) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test1 # 0 second
Test1 # 0 second
Test2 # 0 second
Test2 # 0 second
Test2 # 0 second
And, with asyncio.sleep(0) below, test1() and test2() are run at once alternately:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
await asyncio.sleep(0) # Here
async def test2():
for _ in range(0, 3):
print('Test2')
await asyncio.sleep(0) # Here
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, only 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test2 # 0 second
Test1 # 0 second
Test2 # 0 second
Test1 # 0 second
Test2 # 0 second
Lastly, without time.sleep() or asyncio.sleep() below, first, test1() is run at once, then test2() is run at once:
import asyncio
async def test1():
for _ in range(0, 3):
print('Test1')
async def test2():
for _ in range(0, 3):
print('Test2')
async def main():
await asyncio.gather(test1(), test2()) # Here
asyncio.run(main())
So, 0 second is taken to run test1() and test2() in total:
Test1 # 0 second
Test1 # 0 second
Test1 # 0 second
Test2 # 0 second
Test2 # 0 second
Test2 # 0 second
If I run the following code:
import asyncio
import time
import concurrent.futures
def cpu_bound(mul):
for i in range(mul*10**8):
i+=1
print('result = ', i)
return i
async def say_after(delay, what):
print('sleeping async...')
await asyncio.sleep(delay)
print(what)
# The run_in_pool function must not block the event loop
async def run_in_pool():
with concurrent.futures.ProcessPoolExecutor() as executor:
result = executor.map(cpu_bound, [1, 1, 1])
async def main():
task1 = asyncio.create_task(say_after(0.1, 'hello'))
task2 = asyncio.create_task(run_in_pool())
task3 = asyncio.create_task(say_after(0.1, 'world'))
print(f"started at {time.strftime('%X')}")
await task1
await task2
await task3
print(f"finished at {time.strftime('%X')}")
if __name__ == '__main__':
asyncio.run(main())
The output is:
started at 18:19:28
sleeping async...
result = 100000000
result = 100000000
result = 100000000
sleeping async...
hello
world
finished at 18:19:34
This shows that the event loop blocks until the cpu bound jobs (task2) finish and it continues afterwards with the task3.
If I run only one cpu bound job (the run_in_pool is the following one):
async def run_in_pool():
loop = asyncio.get_running_loop()
with concurrent.futures.ProcessPoolExecutor() as executor:
result = await loop.run_in_executor(executor, cpu_bound, 1)
Then it seems that the event loop doesn't block since the output is:
started at 18:16:23
sleeping async...
sleeping async...
hello
world
result = 100000000
finished at 18:16:28
How can I run many cpu bound jobs (in task2) in a process pool without blocking the event loop?
As you discovered, you need to use asyncio's own run_in_executor to wait for submitted tasks to finish without blocking the event loop. Asyncio doesn't provide the equivalent of map, but it's not hard to emulate it:
async def run_in_pool():
with concurrent.futures.ProcessPoolExecutor() as executor:
futures = [loop.run_in_executor(executor, cpu_bound, i)
for i in (1, 1, 1)]
result = await asyncio.gather(*futures)