I have a bunch of coroutines that doing some work
#asyncio.coroutine
def do_work():
global COUNTER
result = ...
if result.status == 'OK':
COUNTER += 1
and another one
COUNTER = 0
#asyncio.coroutine
def display_status():
while True:
print(COUNTER)
yield from asyncio.sleep(1)
which have to display how many coroutines have finished their work. How to properly implement this task? Following solution doesn't work
#asyncio.coroutine
def spawn_jobs():
coros = []
for i in range(10):
coros.append(asyncio.Task(do_work()))
yield from asyncio.gather(*coros)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.create_task(display_status())
loop.run_until_complete(spawn_jobs())
loop.close()
I expect that counter will be printed to the console every second no matter what do_work() coroutines do. But I have just two outputs: 0 and after a few seconds repeating 10.
But I have just two outputs: 0 and after a few seconds repeating 10.
I can't reproduce it. If I use:
import asyncio
import random
#asyncio.coroutine
def do_work():
global COUNTER
yield from asyncio.sleep(random.randint(1, 5))
COUNTER += 1
I get the output like this:
0
0
4
6
8
Task was destroyed but it is pending!
task: <Task pending coro=<display_status() running at most_wanted.py:16> wait_for=<Future pending cb=[Task._wakeup()] created at ../Lib/asyncio/tasks.py:490> created at most_wanted.py:27>
The infinite loop in display_status() causes the warning at the end. To avoid the warning; exit the loop when all tasks in the batch are done:
#!/usr/bin/env python3
import asyncio
import random
from contextlib import closing
from itertools import cycle
class Batch:
def __init__(self, n):
self.total = n
self.done = 0
async def run(self):
await asyncio.wait([batch.do_work() for _ in range(batch.total)])
def running(self):
return self.done < self.total
async def do_work(self):
await asyncio.sleep(random.randint(1, 5)) # do some work here
self.done += 1
async def display_status(self):
while self.running():
await asyncio.sleep(1)
print('\rdone:', self.done)
async def display_spinner(self, char=cycle('/|\-')):
while self.running():
print('\r' + next(char), flush=True, end='')
await asyncio.sleep(.3)
with closing(asyncio.get_event_loop()) as loop:
batch = Batch(10)
loop.run_until_complete(asyncio.wait([
batch.run(), batch.display_status(), batch.display_spinner()]))
Output
done: 0
done: 2
done: 3
done: 4
done: 10
Solution using threading module
SHOW_STATUS = True
def display_status_sync():
while SHOW_STATUS:
print(S_REQ)
time.sleep(1)
if __name__ == '__main__':
new_thread = threading.Thread(target=display_status_sync)
new_thread.start()
loop.run_until_complete(spawn_jobs())
SHOW_STATS = False
new_thread.join()
loop.close()
But I want to achieve similar functionality using asyncio coroutines. Is it possible to do that?
Related
This is my code:
async def fun1():
result=False
#Some time-consuming operations...
return result
async def fun2():
result=False
#Some time-consuming operations...
return result
if fun1() or fun2():
print("Success")
The if block runs all conditions by order, but I want run them simultaneously and wait until any of theme returns True. I know asyncio.wait(tasks,return_when=asyncio.FIRST_COMPLETED) but I don't know how to use it for my purpose.
You can use asyncio.ensure_future:
Example of code adapted from what you showed:
import asyncio, time, random
async def fun1():
result = False
# Some time-consuming operations...
result = random.randint(1, 10)
await asyncio.sleep(result)
return result
async def fun2():
result = False
# Some time-consuming operations...
result = random.randint(1, 10)
await asyncio.sleep(result)
return result
async def main():
t1 = asyncio.ensure_future(fun1())
t2 = asyncio.ensure_future(fun2())
count = 0
while not any([t1.done(), t2.done()]):
print(f'{count} - {t1.done()}, {t2.done()}')
await asyncio.sleep(1)
count += 1
print(f'At least one task is done: {t1.done()}, {t2.done()}')
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Only as a reminder that is possible for this code to trigger a message if the main ends before any task finish their execution. Example of a possible execution:
0 - False, False
1 - False, False
At least one task is done: False, True
Task was destroyed but it is pending!
task: <Task pending name='Task-2' coro=<fun1() running at /tmp:7> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x00000239C83B4B50>()]>>
I am still pretty new to Python asyncio, so I am trying to convert a simple problem I solved using multithreading, to using asyncio.
I made an example of what I want to achieve.
Each MiniBot instance can start at random times (those time.sleep() calls in main represent instantiations at unpredictable times.)
I am expecting each MiniBot to run in parallel if they start before others have finished.
All good with MultiThreading, but when I translated the problem with async coroutines, I can't get them to start together.
I could use gather, but this would require to have all the task at the beginning, which I don't.
Any suggestions?
Thanks
Oh yes, I am using Python 3.6
Multithreaded version
import threading
import time
class MiniBot(object):
def __init__(self, _id:str):
self._id = _id
self.alive = True
self.start_time = time.time()
self.th = threading.Thread(target=self.run)
self.internal_state = 0
def _foo(self):
self.internal_state += 1
def run(self):
while self.alive:
self._foo()
if time.time() - self.start_time > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
time.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
self.th.start()
def stop(self):
self.alive = False
if __name__ == "__main__":
# MiniBots activities start at random times but should coexist
MiniBot('a').start()
time.sleep(2)
MiniBot('b').start()
time.sleep(1.5)
MiniBot('c').start()
Output:
Starting Minibot a
Starting Minibot b
Starting Minibot c
Killing minibot: a
Var is: 40
Killing minibot: b
Var is: 40
Killing minibot: c
Var is: 40
Async version (Not behaving as I hoped)
import asyncio
import time
class MiniBot(object):
def __init__(self, _id:str):
self._id = _id
self.alive = True
self.last_event = time.time()
self.internal_state = 0
async def _foo(self):
self.internal_state += 1
asyncio.sleep(2)
async def run(self):
while self.alive:
await self._foo()
if time.time() - self.last_event > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
asyncio.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(self.run())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def stop(self):
self.alive = False
if __name__ == "__main__":
# MiniBots activities start at random times but should coexist
MiniBot('a').start()
time.sleep(2)
MiniBot('b').start()
time.sleep(1.5)
MiniBot('c').start()
Output:
Starting Minibot a
Killing minibot: a
Var is: 2839119
Starting Minibot b
Killing minibot: b
Var is: 2820634
Starting Minibot c
Killing minibot: c
Var is: 2804579
start cannot call run_until_complete because run_until_complete runs a coroutine through to the end, whereas you need multiple coroutines running in parallel. The asyncio equivalent of creating and starting a thread is asyncio.create_task(), so start() should call that, and return to the caller, like in the threaded version.
For example:
import asyncio, time
class MiniBot:
def __init__(self, _id):
self._id = _id
self.alive = True
self.start_time = time.time()
self.internal_state = 0
def _foo(self):
self.internal_state += 1
async def run(self):
while self.alive:
self._foo()
if time.time() - self.start_time > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
await asyncio.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
return asyncio.create_task(self.run())
def stop(self):
self.alive = False
async def main():
taska = MiniBot('a').start()
await asyncio.sleep(2)
taskb = MiniBot('b').start()
await asyncio.sleep(1.5)
taskc = MiniBot('c').start()
await asyncio.gather(taska, taskb, taskc)
if __name__ == "__main__":
#asyncio.run(main())
asyncio.get_event_loop().run_until_complete(main())
Don't let the call to gather() throw you off: gather simply gives back control to the event loop and returns when all the provided tasks/futures have finished. You can replace it with something like await asyncio.sleep(10) and have the same effect (in this example). And if you have an unpredictable number of futures, there are other other ways to signal the end-condition.
Also note that you need to await asyncio.sleep(), otherwise it has no effect.
I'm trying to leave threads and start using async. I tried to write something simple so I can get more comfortable with async; for some reason my async code isn't acting async.
I rewrote the same code in threads and it worked fast and concurrently, unlike the async code.
Normal code
import time
import random
def display(x: int) -> None:
time.sleep(random.randint(1, 8))
print(x)
def main():
for i in range(10):
display(i)
if __name__ == '__main__':
main()
Output
0
1
2
3
4
5
6
7
8
9
Async code
import time
import random
import asyncio
async def display(x: int) -> None:
await asyncio.sleep(random.randint(1, 8))
print(x)
async def main():
for i in range(10):
await display(i)
if __name__ == '__main__':
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(main())
event_loop.close()
Output
0
1
2
3
4
5
6
7
8
9
Thread code
import time
import random
import threading
def display(x: int) -> None:
time.sleep(random.randint(1, 8))
print(x)
def main():
threads = []
for i in range(10):
t = threading.Thread(target=display, args=[i])
threads.append(t)
t.start()
for t in threads:
t.join()
if __name__ == '__main__':
main()
Output
5
9
3
0
4
2
1
8
6
7
await display(i) runs display with argument i, which returns an awaitable. You then immediately wait for it with await, blocking the call right there.
If you want to schedule them all together and then wait at the end, you need to collect the awaitables in a list and then wait for all of them at once.
import time
import random
import asyncio
async def display(x: int) -> None:
await asyncio.sleep(random.randint(1, 8)/10)
print(x)
async def main():
awaitables = []
for i in range(10):
awaitables.append(display(i))
await asyncio.wait(awaitables)
if __name__ == '__main__':
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(main())
event_loop.close()
And yes, because otherwise someone will surely point it out, you can also write that in a list comprehension:
async def main():
await asyncio.wait([display(i) for i in range(10)])
Further note:
I'm sure you are aware, nonetheless I think it is important to mention it anyway. The code runs asynchronous, but not parallel. Running multiple compution-heavy functions with async or threading.Thread still only runs them on one single cpu core without any speedup. The Python interpreter is single-threaded.
I am implementing an asynchronous iterator to be used with async for which should return a new value at a (mostly) regular interval.
We can illustrate such iterator with a simple clock that will increment a counter every ~n seconds:
import asyncio
class Clock(object):
def __init__(self, interval=1):
self.counter = 0
self.interval = interval
self.tick = asyncio.Event()
asyncio.ensure_future(self.tick_tock())
async def tick_tock(self):
while True:
self.tick.clear()
await asyncio.sleep(self.interval)
self.counter = self.__next__()
self.tick.set()
def __next__(self):
self.counter += 1
return self.counter
def __aiter__(self):
return self
async def __anext__(self):
await self.tick.wait()
return self.counter
Is there a better or cleaner approach than using asyncio.Event? More than one coroutine will async for on this iterator.
In my opinion, your approach is fine. Note that since python 3.6, you can also use asynchronous generators:
async def clock(start=0, step=1, interval=1.):
for i in count(start, step):
yield i
await asyncio.sleep(interval)
However, you won't be able to share them between multiple coroutines. You would have to run the clock in a task and make the data available through an asynchronous iteration interface, which is essentially what you did in your code. Here's a possible implementation.
If you're using Python 3.6+ you can use asynchronous generators which are more readable.
async def Clock(interval=1):
counter = 0
while True:
await asyncio.sleep(interval)
counter += 1
yield counter
async def main():
async for i in Clock(1):
print(i)
if i == 4:
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
Python3.6 now asynchronous iterables. Is there builtin way to transform a asynchronous iterable to a synchronous iterable.
I currently have this helper function, but it feels very un-pythonic. Is there a better way to do this?
async def aiter_to_list(aiter):
l = []
async for i in aiter:
l.append(i)
return l
From Python 3.6 you can use Asynchronous Comprehensions
async def async_iter():
for i in range(0,5):
yield i
# async comprehension
sync_list = [gen async for gen in async_iter()]
print(sync_list) # [0, 1, 2, 3, 4]
You can use aiostream.stream.list:
from aiostream import stream
async def agen():
yield 1
yield 2
yield 3
async def main():
lst = await stream.list(agen())
print(lst) # prints [1, 2, 3]
More operators and examples in the documentation.
Your "asynchronous to synchronous" helper is itself asynchronous; not a big change at all. In general: no, you cannot make something asynchronous synchronous. An asynchronous value will be supplied "sometime later"; you cannot make that into "now" because the value doesn't exist "now" and you will have to wait for it, asynchronously.
These functions allow you to convert from / to iterable <==> async iterable, not just simple lists.
Basic imports
import asyncio
import threading
import time
DONE = object()
TIMEOUT = 0.001
The function to_sync_iterable will convert any async iterable to a sync iterable:
def to_sync_iterable(async_iterable, maxsize = 0):
def sync_iterable():
queue = asyncio.Queue(maxsize=maxsize)
loop = asyncio.get_event_loop()
t = threading.Thread(target=_run_coroutine, args=(loop, async_iterable, queue))
t.daemon = True
t.start()
while True:
if not queue.empty():
x = queue.get_nowait()
if x is DONE:
break
else:
yield x
else:
time.sleep(utils.TIMEOUT)
t.join()
return sync_iterable()
def _run_coroutine(loop, async_iterable, queue):
loop.run_until_complete(_consume_async_iterable(async_iterable, queue))
async def _consume_async_iterable(async_iterable, queue):
async for x in async_iterable:
await queue.put(x)
await queue.put(DONE)
You can use it like this:
async def slow_async_generator():
yield 0
await asyncio.sleep(1)
yield 1
await asyncio.sleep(1)
yield 2
await asyncio.sleep(1)
yield 3
for x in to_sync_iterable(slow_async_generator()):
print(x)
The function to_async_iterable will convert any sync iterable to an async iterable:
def to_async_iterable(iterable, maxsize = 0):
async def async_iterable():
queue = asyncio.Queue(maxsize=maxsize)
loop = asyncio.get_event_loop()
task = loop.run_in_executor(None, lambda: _consume_iterable(loop, iterable, queue))
while True:
x = await queue.get()
if x is DONE:
break
else:
yield x
await task
return async_iterable()
def _consume_iterable(loop, iterable, queue):
for x in iterable:
while True:
if not queue.full():
loop.call_soon_threadsafe(queue.put_nowait, x)
break
else:
time.sleep(TIMEOUT)
while True:
if not queue.full():
loop.call_soon_threadsafe(queue.put_nowait, DONE)
break
else:
time.sleep(TIMEOUT)
This one is specially useful for asyncio programs because it won't block the event loop even if the the sync iterable blocks. You can use it like this:
def slow_sync_generator():
yield 0
time.sleep(1)
yield 1
time.sleep(1)
yield 2
time.sleep(1)
yield 3
async def async_task():
async for x in to_async_iterable(slow_sync_generator()):
print(x)
asyncio.get_event_loop().run_until_complete(async_task())