I'm trying to leave threads and start using async. I tried to write something simple so I can get more comfortable with async; for some reason my async code isn't acting async.
I rewrote the same code in threads and it worked fast and concurrently, unlike the async code.
Normal code
import time
import random
def display(x: int) -> None:
time.sleep(random.randint(1, 8))
print(x)
def main():
for i in range(10):
display(i)
if __name__ == '__main__':
main()
Output
0
1
2
3
4
5
6
7
8
9
Async code
import time
import random
import asyncio
async def display(x: int) -> None:
await asyncio.sleep(random.randint(1, 8))
print(x)
async def main():
for i in range(10):
await display(i)
if __name__ == '__main__':
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(main())
event_loop.close()
Output
0
1
2
3
4
5
6
7
8
9
Thread code
import time
import random
import threading
def display(x: int) -> None:
time.sleep(random.randint(1, 8))
print(x)
def main():
threads = []
for i in range(10):
t = threading.Thread(target=display, args=[i])
threads.append(t)
t.start()
for t in threads:
t.join()
if __name__ == '__main__':
main()
Output
5
9
3
0
4
2
1
8
6
7
await display(i) runs display with argument i, which returns an awaitable. You then immediately wait for it with await, blocking the call right there.
If you want to schedule them all together and then wait at the end, you need to collect the awaitables in a list and then wait for all of them at once.
import time
import random
import asyncio
async def display(x: int) -> None:
await asyncio.sleep(random.randint(1, 8)/10)
print(x)
async def main():
awaitables = []
for i in range(10):
awaitables.append(display(i))
await asyncio.wait(awaitables)
if __name__ == '__main__':
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(main())
event_loop.close()
And yes, because otherwise someone will surely point it out, you can also write that in a list comprehension:
async def main():
await asyncio.wait([display(i) for i in range(10)])
Further note:
I'm sure you are aware, nonetheless I think it is important to mention it anyway. The code runs asynchronous, but not parallel. Running multiple compution-heavy functions with async or threading.Thread still only runs them on one single cpu core without any speedup. The Python interpreter is single-threaded.
Related
import asyncio
import random
async def producer(q: asyncio.Queue):
for _ in range(100):
await q.put(random.randint(1, 10))
async def consumer(q: asyncio.Queue):
while True:
num = await q.get()
print("Got From Queue: ", num)
q.task_done()
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
consumers = [asyncio.create_task(consumer(q)) for _ in range(10)]
await pr
await q.join()
for c in consumers:
c.cancel()
asyncio.run(main())
I created this script to replicate a simplified version of my problem.
so this is a very basic producer-consumer type of script, and this works just fine
however if an exception occurs inside the consumer, the script freezes.
for example, If I modify the consumer script like so:
async def consumer(q: asyncio.Queue):
raise Exception
while True:
num = await q.get()
print("Got From Queue: ", num)
q.task_done()
when I run the script now it freezes and no exceptions get thrown.
However, If I change the main script to generate all the consumer tasks in a for loop instead of generating them through a list comprehension like so:
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
# consumers = [asyncio.create_task(consumer(q)) for _ in range(10)]
for _ in range(10):
asyncio.create_task(consumer(q))
await pr
await q.join()
It throws 10 exceptions of type Exception, as expected behavior.
my 2 questions are:
Why does this strange behaviour happen?
In my actual script, I want to use exception handling, so I'm using now the for-loop approach. However, In this approach, I don't have references to all the consumer tasks, so I can't cancel them when I want. How can I do that? (Simply appending them to a list after creation replicates the original problem)
I am completely clueless, any help appreciated!
I don't think that the use of a list comprehension instead of a for loop has any impact on this.
You should handle the exception in the consumer and call task_done() for each get(), otherwise you'll experience the current behaviour where q.join() will block/wait for unfinished tasks:
task_done()
If a join() is currently blocking, it will resume when all items have
been processed (meaning that a task_done() call was received for every
item that had been put() into the queue).
test.py:
import asyncio
import random
async def producer(q):
for _ in range(10):
await q.put(random.randint(1, 10))
async def consumer(q):
while True:
num = await q.get()
try:
if num in (5, 8):
raise Exception("ERROR")
print(f"Working on: {num}")
except Exception as exc:
print(f"{num}: {exc}")
finally:
q.task_done()
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
tasks = []
for _ in range(10):
tasks.append(asyncio.create_task(consumer(q)))
await pr
await q.join()
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if __name__ == "__main__":
asyncio.run(main())
Test:
$ python test.py
Working on: 7
Working on: 4
Working on: 4
Working on: 4
Working on: 1
Working on: 3
8: ERROR
Working on: 6
5: ERROR
Working on: 4
I want speed up some API requests... for that I try to figure out how to do and copy some code which run but when I try my own code its no longer asynchrone. Maybe someone find the fail?
Copy Code (guess from stackoverflow):
#!/usr/bin/env python3
import asyncio
#asyncio.coroutine
def func_normal():
print('A')
yield from asyncio.sleep(5)
print('B')
return 'saad'
#asyncio.coroutine
def func_infinite():
for i in range(10):
print("--%d" % i)
return 'saad2'
loop = asyncio.get_event_loop()
tasks = func_normal(), func_infinite()
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
My "own" code (I need at the end a list returned and merge the results of all functions):
import asyncio
import time
#asyncio.coroutine
def say_after(start,count,say,yep=True):
retl = []
if yep:
time.sleep(5)
for x in range(start,count):
retl.append(x)
print(say)
return retl
def main():
print(f"started at {time.strftime('%X')}")
loop = asyncio.get_event_loop()
tasks = say_after(10,20,"a"), say_after(20,30,"b",False)
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
c = a + b
#print(c)
print(f"finished at {time.strftime('%X')}")
main()
Or I m completly wrong and should solve that with multithreading? What would be the best way for API requests that returns a list that I need to merge?
Added comment for each section that needs improvement. Removed some to simply code.
In fact, I didn't find any performance uplift with using range() wrapped in coroutine and using async def, might worth with heavier operations.
import asyncio
import time
# #asyncio.coroutine IS DEPRECATED since python 3.8
#asyncio.coroutine
def say_after(wait=True):
result = []
if wait:
print("I'm sleeping!")
time.sleep(5)
print("'morning!")
# This BLOCKs thread, but release GIL so other thread can run.
# But asyncio runs in ONE thread, so this still harms simultaneity.
# normal for is BLOCKING operation.
for i in range(5):
result.append(i)
print(i, end='')
print()
return result
def main():
start = time.time()
# Loop argument will be DEPRECATED from python 3.10
# Make main() as coroutine, then use asyncio.run(main()).
# It will be in asyncio Event loop, without explicitly passing Loop.
loop = asyncio.get_event_loop()
tasks = say_after(), say_after(False)
# As we will use asyncio.run(main()) from now on, this should be await-ed.
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print(f"Took {time.time() - start:5f}")
loop.close()
main()
Better way:
import asyncio
import time
async def say_after(wait=True):
result = []
if wait:
print("I'm sleeping!")
await asyncio.sleep(2) # 'await' a coroutine version of it instead.
print("'morning!")
# wrap iterator in generator - or coroutine
async def asynchronous_range(end):
for _i in range(end):
yield _i
# use it with async for
async for i in asynchronous_range(5):
result.append(i)
print(i, end='')
print()
return result
async def main():
start = time.time()
tasks = say_after(), say_after(False)
a, b = await asyncio.gather(*tasks)
print(f"Took {time.time() - start:5f}")
asyncio.run(main())
Result
Your code:
DeprecationWarning: "#coroutine" decorator is deprecated since Python 3.8, use "async def" instead
def say_after(wait=True):
I'm sleeping!
'morning!
01234
01234
Took 5.003802
Better async code:
I'm sleeping!
01234
'morning!
01234
Took 2.013863
Note that fixed code now finish it's job while other task is sleeping.
I have several http requests to fire simultaneously. I am trying to use async for to do this.
import asyncio
async def ticker(delay, to):
for i in range(to):
yield i
print(i)
await asyncio.sleep(delay) # instead of aiohttp request
print(i, ' fin')
async def main():
async for x in ticker(1,2):
pass
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I keep getting subsequent calls with the following:
0
0 fin
1
1 fin
Instead I need the output as shown below:
0
1
0 fin
1 fin
Could you please advise me on how to do this?
The problem is that async for is exactly what you don't need.
async for is designed to iterate while waiting for a task to complete between each iteration; you want to iterate (starting requests) without waiting for the previous task(s) to finish.
You'll want something like
async def do_request():
await asyncio.sleep(1)
async def main():
await asyncio.gather(*[
do_request() for i in range(10)
])
Comment with a follow-up if that doesn't answer your question.
I've been using asyncio for a bit but I'm still fairly unfamiliar with it. My current issue is that while trying to wait for a response from a function with asyncio, the waiting (while loop) blocks the function from happening. Here is the code that sums up the problem:
import asyncio
response = 0
async def handle(x):
await asyncio.sleep(0.1)
return x
async def run():
global response
for number in range(1, 21):
response = await handle(number)
print(response)
if response == 10:
await wait_for_next(response)
async def wait_for_next(x):
while response == x:
print('waiting',response,x)
await asyncio.sleep(0.5)
print('done')
tasks = [run()]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
wait_for_next is supposed to wait for the next response, but the while loop blocks the run() function. How could I stop this happening? Should I be using loop.run_in_executor, and if so, how?
(There were a couple of other examples of this I could find, but they were very specific and I didn't understand if our problems/solutions would be the same.)
As already noted, loop stuck because await wait_for_next(response) blocks execution flow until this coroutine wouldn't be finished.
If you want some of your coroutines to be started without blocking execution flow you can start it as asyncio.Task (more about tasks) using ensure_future function:
import asyncio
response = 0
async def handle(x):
await asyncio.sleep(0.1)
return x
async def run():
global response
for number in range(1, 21):
response = await handle(number)
print(response)
if response == 10:
# run wait_for_next "in background" instead of blocking flow:
asyncio.ensure_future(wait_for_next(response))
async def wait_for_next(x):
while response == x:
print('waiting',response,x)
await asyncio.sleep(0.5)
print('done')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(run())
Output:
1
2
3
4
5
6
7
8
9
10
waiting 10 10
11
12
13
14
done
15
16
17
18
19
20
I have a bunch of coroutines that doing some work
#asyncio.coroutine
def do_work():
global COUNTER
result = ...
if result.status == 'OK':
COUNTER += 1
and another one
COUNTER = 0
#asyncio.coroutine
def display_status():
while True:
print(COUNTER)
yield from asyncio.sleep(1)
which have to display how many coroutines have finished their work. How to properly implement this task? Following solution doesn't work
#asyncio.coroutine
def spawn_jobs():
coros = []
for i in range(10):
coros.append(asyncio.Task(do_work()))
yield from asyncio.gather(*coros)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.create_task(display_status())
loop.run_until_complete(spawn_jobs())
loop.close()
I expect that counter will be printed to the console every second no matter what do_work() coroutines do. But I have just two outputs: 0 and after a few seconds repeating 10.
But I have just two outputs: 0 and after a few seconds repeating 10.
I can't reproduce it. If I use:
import asyncio
import random
#asyncio.coroutine
def do_work():
global COUNTER
yield from asyncio.sleep(random.randint(1, 5))
COUNTER += 1
I get the output like this:
0
0
4
6
8
Task was destroyed but it is pending!
task: <Task pending coro=<display_status() running at most_wanted.py:16> wait_for=<Future pending cb=[Task._wakeup()] created at ../Lib/asyncio/tasks.py:490> created at most_wanted.py:27>
The infinite loop in display_status() causes the warning at the end. To avoid the warning; exit the loop when all tasks in the batch are done:
#!/usr/bin/env python3
import asyncio
import random
from contextlib import closing
from itertools import cycle
class Batch:
def __init__(self, n):
self.total = n
self.done = 0
async def run(self):
await asyncio.wait([batch.do_work() for _ in range(batch.total)])
def running(self):
return self.done < self.total
async def do_work(self):
await asyncio.sleep(random.randint(1, 5)) # do some work here
self.done += 1
async def display_status(self):
while self.running():
await asyncio.sleep(1)
print('\rdone:', self.done)
async def display_spinner(self, char=cycle('/|\-')):
while self.running():
print('\r' + next(char), flush=True, end='')
await asyncio.sleep(.3)
with closing(asyncio.get_event_loop()) as loop:
batch = Batch(10)
loop.run_until_complete(asyncio.wait([
batch.run(), batch.display_status(), batch.display_spinner()]))
Output
done: 0
done: 2
done: 3
done: 4
done: 10
Solution using threading module
SHOW_STATUS = True
def display_status_sync():
while SHOW_STATUS:
print(S_REQ)
time.sleep(1)
if __name__ == '__main__':
new_thread = threading.Thread(target=display_status_sync)
new_thread.start()
loop.run_until_complete(spawn_jobs())
SHOW_STATS = False
new_thread.join()
loop.close()
But I want to achieve similar functionality using asyncio coroutines. Is it possible to do that?