I want speed up some API requests... for that I try to figure out how to do and copy some code which run but when I try my own code its no longer asynchrone. Maybe someone find the fail?
Copy Code (guess from stackoverflow):
#!/usr/bin/env python3
import asyncio
#asyncio.coroutine
def func_normal():
print('A')
yield from asyncio.sleep(5)
print('B')
return 'saad'
#asyncio.coroutine
def func_infinite():
for i in range(10):
print("--%d" % i)
return 'saad2'
loop = asyncio.get_event_loop()
tasks = func_normal(), func_infinite()
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
My "own" code (I need at the end a list returned and merge the results of all functions):
import asyncio
import time
#asyncio.coroutine
def say_after(start,count,say,yep=True):
retl = []
if yep:
time.sleep(5)
for x in range(start,count):
retl.append(x)
print(say)
return retl
def main():
print(f"started at {time.strftime('%X')}")
loop = asyncio.get_event_loop()
tasks = say_after(10,20,"a"), say_after(20,30,"b",False)
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
c = a + b
#print(c)
print(f"finished at {time.strftime('%X')}")
main()
Or I m completly wrong and should solve that with multithreading? What would be the best way for API requests that returns a list that I need to merge?
Added comment for each section that needs improvement. Removed some to simply code.
In fact, I didn't find any performance uplift with using range() wrapped in coroutine and using async def, might worth with heavier operations.
import asyncio
import time
# #asyncio.coroutine IS DEPRECATED since python 3.8
#asyncio.coroutine
def say_after(wait=True):
result = []
if wait:
print("I'm sleeping!")
time.sleep(5)
print("'morning!")
# This BLOCKs thread, but release GIL so other thread can run.
# But asyncio runs in ONE thread, so this still harms simultaneity.
# normal for is BLOCKING operation.
for i in range(5):
result.append(i)
print(i, end='')
print()
return result
def main():
start = time.time()
# Loop argument will be DEPRECATED from python 3.10
# Make main() as coroutine, then use asyncio.run(main()).
# It will be in asyncio Event loop, without explicitly passing Loop.
loop = asyncio.get_event_loop()
tasks = say_after(), say_after(False)
# As we will use asyncio.run(main()) from now on, this should be await-ed.
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print(f"Took {time.time() - start:5f}")
loop.close()
main()
Better way:
import asyncio
import time
async def say_after(wait=True):
result = []
if wait:
print("I'm sleeping!")
await asyncio.sleep(2) # 'await' a coroutine version of it instead.
print("'morning!")
# wrap iterator in generator - or coroutine
async def asynchronous_range(end):
for _i in range(end):
yield _i
# use it with async for
async for i in asynchronous_range(5):
result.append(i)
print(i, end='')
print()
return result
async def main():
start = time.time()
tasks = say_after(), say_after(False)
a, b = await asyncio.gather(*tasks)
print(f"Took {time.time() - start:5f}")
asyncio.run(main())
Result
Your code:
DeprecationWarning: "#coroutine" decorator is deprecated since Python 3.8, use "async def" instead
def say_after(wait=True):
I'm sleeping!
'morning!
01234
01234
Took 5.003802
Better async code:
I'm sleeping!
01234
'morning!
01234
Took 2.013863
Note that fixed code now finish it's job while other task is sleeping.
Related
import asyncio
import random
async def producer(q: asyncio.Queue):
for _ in range(100):
await q.put(random.randint(1, 10))
async def consumer(q: asyncio.Queue):
while True:
num = await q.get()
print("Got From Queue: ", num)
q.task_done()
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
consumers = [asyncio.create_task(consumer(q)) for _ in range(10)]
await pr
await q.join()
for c in consumers:
c.cancel()
asyncio.run(main())
I created this script to replicate a simplified version of my problem.
so this is a very basic producer-consumer type of script, and this works just fine
however if an exception occurs inside the consumer, the script freezes.
for example, If I modify the consumer script like so:
async def consumer(q: asyncio.Queue):
raise Exception
while True:
num = await q.get()
print("Got From Queue: ", num)
q.task_done()
when I run the script now it freezes and no exceptions get thrown.
However, If I change the main script to generate all the consumer tasks in a for loop instead of generating them through a list comprehension like so:
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
# consumers = [asyncio.create_task(consumer(q)) for _ in range(10)]
for _ in range(10):
asyncio.create_task(consumer(q))
await pr
await q.join()
It throws 10 exceptions of type Exception, as expected behavior.
my 2 questions are:
Why does this strange behaviour happen?
In my actual script, I want to use exception handling, so I'm using now the for-loop approach. However, In this approach, I don't have references to all the consumer tasks, so I can't cancel them when I want. How can I do that? (Simply appending them to a list after creation replicates the original problem)
I am completely clueless, any help appreciated!
I don't think that the use of a list comprehension instead of a for loop has any impact on this.
You should handle the exception in the consumer and call task_done() for each get(), otherwise you'll experience the current behaviour where q.join() will block/wait for unfinished tasks:
task_done()
If a join() is currently blocking, it will resume when all items have
been processed (meaning that a task_done() call was received for every
item that had been put() into the queue).
test.py:
import asyncio
import random
async def producer(q):
for _ in range(10):
await q.put(random.randint(1, 10))
async def consumer(q):
while True:
num = await q.get()
try:
if num in (5, 8):
raise Exception("ERROR")
print(f"Working on: {num}")
except Exception as exc:
print(f"{num}: {exc}")
finally:
q.task_done()
async def main():
q = asyncio.Queue()
pr = asyncio.create_task(producer(q))
tasks = []
for _ in range(10):
tasks.append(asyncio.create_task(consumer(q)))
await pr
await q.join()
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if __name__ == "__main__":
asyncio.run(main())
Test:
$ python test.py
Working on: 7
Working on: 4
Working on: 4
Working on: 4
Working on: 1
Working on: 3
8: ERROR
Working on: 6
5: ERROR
Working on: 4
I get this error:
D:\pythonstuff\demo.py:28: DeprecationWarning: The explicit passing of coroutine objects to asyncio.wait() is deprecated since Python 3.8, and scheduled for removal in Python 3.11.
await asyncio.wait([
Waited 1 second!
Waited 5 second!
Time passed: 0hour:0min:5sec
Process finished with exit code 0
When I run the code:
import asyncio
import time
class class1():
async def function_inside_class(self):
await asyncio.sleep(1)
print("Waited 1 second!")
async def function_inside_class2(self):
await asyncio.sleep(5)
print("Waited 5 second!")
def tic():
global _start_time
_start_time = time.time()
def tac():
t_sec = round(time.time() - _start_time)
(t_min, t_sec) = divmod(t_sec,60)
(t_hour,t_min) = divmod(t_min,60)
print('Time passed: {}hour:{}min:{}sec'.format(t_hour,t_min,t_sec))
object = class1()
async def main():
tic()
await asyncio.wait([
object.function_inside_class(),
object.function_inside_class2()
])
tac()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Are there any good alternatives to asyncio.wait? I don't want a warning in console every time I launch my application.
Edit: I don't want to just hide the error, that's bad practice, and I'm looking for other ways to do the same or a similar thing, not another async library to restore the old functionality.
You can just call it this way as it recommends in the docs here
Example from the docs:
async def foo():
return 42
task = asyncio.create_task(foo())
done, pending = await asyncio.wait({task})
So your code would become:
await asyncio.wait([
asyncio.create_task(object.function_inside_class()),
asyncio.create_task(object.function_inside_class2())
])
I have 2 functions: The first one, def_a, is an asynchronous function and the second one is def_b which is a regular function and called with the result of def_a as a callback with the add_done_callback function.
My code looks like this:
import asyncio
def def_b(result):
next_number = result.result()
# some work on the next_number
print(next_number + 1)
async def def_a(number):
await some_async_work(number)
return number + 1
loop = asyncio.get_event_loop()
task = asyncio.ensure_future(def_a(1))
task.add_done_callback(def_b)
response = loop.run_until_complete(task)
loop.close()
And it's work perfectly.
The problem began when also the second function, def_b, became asynchronous. Now it looks like this:
async def def_b(result):
next_number = result.result()
# some asynchronous work on the next_number
print(next_number + 1)
But now I can not provide it to the add_done_callback function, because it's not a regular function.
My question is- Is it possible and how can I provide def_b to the add_done_callback function if def_b is asynchronous?
add_done_callback is considered a "low level" interface. When working with coroutines, you can chain them in many ways, for example:
import asyncio
async def my_callback(result):
print("my_callback got:", result)
return "My return value is ignored"
async def coro(number):
await asyncio.sleep(number)
return number + 1
async def add_success_callback(fut, callback):
result = await fut
await callback(result)
return result
loop = asyncio.get_event_loop()
task = asyncio.ensure_future(coro(1))
task = add_success_callback(task, my_callback)
response = loop.run_until_complete(task)
print("response:", response)
loop.close()
Keep in mind add_done_callback will still call the callback if your future raises an exception (but calling result.result() will raise it).
This only works for one future job, if you have multiple async jobs, they will blocks each other, a better way is using asyncio.as_completed() to iterate future list:
import asyncio
async def __after_done_callback(future_result):
# await for something...
pass
async def __future_job(number):
await some_async_work(number)
return number + 1
loop = asyncio.get_event_loop()
tasks = [asyncio.ensure_future(__future_job(x)) for x in range(100)] # create 100 future jobs
for f in asyncio.as_completed(tasks):
result = await f
await __after_done_callback(result)
loop.close()
You can try the aiodag library. It's a very lightweight wrapper around asyncio that abstracts away some of the async plumbing that you usually have to think about. From this example you won't be able to tell that things are running asynchronously since it's just 1 task that depends on another, but it is all running async.
import asyncio
from aiodag import task
#task
async def def_b(result):
# some asynchronous work on the next_number
print(result + 1)
#task
async def def_a(number):
await asyncio.sleep(number)
return number + 1
async def main():
a = def_a(1)
b = def_b(a) # this makes task b depend on task a
return await b
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
response = loop.run_until_complete(main())
I have a program with one producer and two slow consumers and I'd like to rewrite it with coroutines in such way that each consumer will handle only last value (i.e. skip new values generated during processing the old ones) produced for it (I used threads and threading.Queue() but with it blocks on put(), cause the queue will be full most of the time).
After reading answer to this question I decided to use asyncio.Event and asyncio.Queue. I wrote this prototype program:
import asyncio
async def l(event, q):
h = 1
while True:
# ready
event.set()
# get value to process
a = await q.get()
# process it
print(a * h)
h *= 2
async def m(event, q):
i = 1
while True:
# pass element to consumer, when it's ready
if event.is_set():
await q.put(i)
event.clear()
# produce value
i += 1
el = asyncio.get_event_loop()
ev = asyncio.Event()
qu = asyncio.Queue(2)
tasks = [
asyncio.ensure_future(l(ev, qu)),
asyncio.ensure_future(m(ev, qu))
]
el.run_until_complete(asyncio.gather(*tasks))
el.close()
and I have noticed that l coroutine blocks on q.get() line and doesn't print anything.
It works as I expect after adding asyncio.sleep() in both (I get 1,11,21,...):
import asyncio
import time
async def l(event, q):
h = 1
a = 1
event.set()
while True:
# await asyncio.sleep(1)
a = await q.get()
# process it
await asyncio.sleep(1)
print(a * h)
event.set()
async def m(event, q):
i = 1
while True:
# pass element to consumer, when it's ready
if event.is_set():
await q.put(i)
event.clear()
await asyncio.sleep(0.1)
# produce value
i += 1
el = asyncio.get_event_loop()
ev = asyncio.Event()
qu = asyncio.Queue(2)
tasks = [
asyncio.ensure_future(l(ev, qu)),
asyncio.ensure_future(m(ev, qu))
]
el.run_until_complete(asyncio.gather(*tasks))
el.close()
...but I'm looking for solution without it.
Why is it so? How can I fix it? I think I cannot call await l() from m as both of them have states (in original program the first draws solution with PyGame and the second plots results).
The code is not working as expected as the task running the m function is never stopped. The task will continue increment i in the case that event.is_set() == False. Because this task is never suspended, the task running function l will never be called. Therefore, you need a way to suspend the task running function m. One way of suspending is awaiting another coroutine, that is the reason why a asyncio.sleep works as expected.
I think the following code will work as you expect. The LeakyQueue will ensure that only the last value from the producer will be processed by the consumer. As the complexity is very symmetric, the consumer will consume all values produced by the producer. If you increase the delay argument, you can simulate that the consumer only processes the last value created by the producer.
import asyncio
class LeakyQueue(asyncio.Queue):
async def put(self, item):
if self.full():
await self.get()
await super().put(item)
async def consumer(queue, delay=0):
h = 1
while True:
a = await queue.get()
if delay:
await asyncio.sleep(delay)
print ('consumer', a)
h += 2
async def producer(queue):
i = 1
while True:
await asyncio.ensure_future(queue.put(i))
print ('producer', i)
i += 1
loop = asyncio.get_event_loop()
queue = LeakyQueue(maxsize=1)
tasks = [
asyncio.ensure_future(consumer(queue, 0)),
asyncio.ensure_future(producer(queue))
]
loop.run_until_complete(asyncio.gather(*tasks))
I want to use generator yield and async functions. I read this topic, and wrote next code:
import asyncio
async def createGenerator():
mylist = range(3)
for i in mylist:
await asyncio.sleep(1)
yield i*i
async def start():
mygenerator = await createGenerator()
for i in mygenerator:
print(i)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(start())
except KeyboardInterrupt:
loop.stop()
pass
But i got the error:
SyntaxError: 'yield' inside async function
How to use yield generator in async function?
Upd:
Starting with Python 3.6 we have asynchronous generators and able to use yield directly inside coroutines.
import asyncio
async def async_generator():
for i in range(3):
await asyncio.sleep(1)
yield i*i
async def main():
async for i in async_generator():
print(i)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
finally:
loop.run_until_complete(loop.shutdown_asyncgens()) # see: https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.shutdown_asyncgens
loop.close()
Old answer for Python 3.5:
You can't yield inside coroutines. Only way is to implement Asynchronous Iterator manually using __aiter__/__anext__ magic methods. In your case:
import asyncio
class async_generator:
def __init__(self, stop):
self.i = 0
self.stop = stop
async def __aiter__(self):
return self
async def __anext__(self):
i = self.i
self.i += 1
if self.i <= self.stop:
await asyncio.sleep(1)
return i * i
else:
raise StopAsyncIteration
async def main():
async for i in async_generator(3):
print(i)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Output:
0
1
4
Here're two more examples: 1, 2
New Python 3.6 comes with support for asynchronous generators.
PEP 0525
What's new in Python 3.6
PS: On the moment of writing Python 3.6 is still beta. If you are on GNU/Linux or OS X and you cannot wait you can try new Python with pyenv.
This should work with python 3.6 (tested with 3.6.0b1):
import asyncio
async def createGenerator():
mylist = range(3)
for i in mylist:
await asyncio.sleep(1)
yield i*i
async def start():
async for i in createGenerator():
print(i)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(start())
except KeyboardInterrupt:
loop.stop()
pass