Develop Reference

Multiple loop functions parallel running in Python

I've 3 loops i want to be executed in parallel until the exit of program.
Actually I used this solution but i think is not optimal:
class Window(tk.Tk):
def __init__(self, loop,app):
self.loop = loop
self.app = app
self.root = tk.Tk()
async def show(self):
while True:
self.root.update()
await asyncio.sleep(0.1)
class App:
async def exec(self):
self.window = Window(asyncio.get_event_loop(),self)
await asyncio.gather(self.window.show(), self.connectWS(),self.readSerial())
async def readSerial(self):
self.serial = serial.serial_for_url('/dev/cu.usbserial-1430', baudrate=9600, timeout=5)
self.serial.isOpen()
self.serial.flushInput() #flush input buffer, discarding all its contents
self.serial.flushOutput()
while True:
response = self.serial.read(1)
await asyncio.sleep(0.1)
async def connectWS(self):
try:
async with websockets.connect(
"ws://mysocket.com") as ws:
self.ws = WebSocketHandler(str(uuid.getnode()), ws,self)
await asyncio.gather(self.ws.start(), self.ws.send_boot_payload())
except websockets.exceptions.ConnectionClosedError:
print("DISCONNECTED")
call_later(10,self.connectWS) #reconnect
asyncio.run(App().exec())
Can someone suggest me the best practice to execute 3 (or more) loops (actually are a WebSocket, a serial reader and a Tkinter loop that update UI basing on serial and websocket)?

Running an async function in an worker thread/loop

I'm trying to write code that enables using asyncpg from mostly sync code (to avoid duplication).
For some very strange reason, the coroutine Database.test() will execute and return in my worker eventloop/thread. The future seems to work correctly. But connecting to a database with asyncpg will just hang. Any clue as to why?
Also, maybe I should use asyncio.run() instead.
from threading import Thread
import asyncio
import asyncpg
class AsyncioWorkerThread(Thread):
def __init__(self, *args, daemon=True, loop=None, **kwargs):
super().__init__(*args, daemon=daemon, **kwargs)
self.loop = loop or asyncio.new_event_loop()
self.running = False
def run(self):
self.running = True
self.loop.run_forever()
def submit(self, coro):
fut = asyncio.run_coroutine_threadsafe(coro, loop=self.loop)
return fut.result()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
self.join()
self.running = False
class Database:
async def test(self):
print('In test')
await asyncio.sleep(5)
async def connect(self):
# Put in your db credentials here
# pg_user = ''
# pg _password = ''
# pg_host = ""
# pg_port = 20
# pg_db
connection_uri = f'postgres://{pg_user}:{pg_password}#{pg_host}:{pg_port}/{pg_db}'
self.connection_pool = await asyncpg.create_pool(
connection_uri, min_size=5, max_size=10)
if __name__ == "__main__":
db = Database()
worker = AsyncioWorkerThread()
worker.start()
worker.submit(db.test()) # Works future returns correctly
worker.submit(db.connect()) # Hangs, thread never manages to acquire

set_exception_handler ignored in python3.6 with asyncio

I'm basically creating an object that needs to perform a number of tasks in async mode (+ other things but I've tried to simplify here). Here is a snippet of code for the object itself. It's successful (thanks to Lynn Root's example) at handling signals - but not at handling exceptions. Or at least not in the way I am hoping to be able to handle them.
class myobj(object):
def __init__(self, loop: asyncio.AbstractEventLoop):
self.shutdown = False
self.loop = loop
async def graceful_shutdown(self, s=None):
if s is not None:
logging.warning(f'Receiving signal {s.name}.')
else:
logging.warning(f'Shutting NOT via signal')
logging.warning(f'Initiating cancellation of {len(self.tasks)} tasks...')
[task.cancel() for task in self.tasks]
logging.warning(f'Gaterhing out put of cancellation of {len(self.tasks)} tasks...')
await asyncio.gather(*self.tasks, loop=self.loop, return_exceptions=True)
logging.warning('Done graceful shutdown of subtasks')
# Mark main task to shutdown
self.shutdown = True
async def run(self):
i = 0
taskx = self.loop.create_task(self.task_x())
self.tasks = [taskx]
while not self.shutdown:
await asyncio.sleep(1)
print(f'Main runner... {i}')
i += 1
logging.warning('Main runner is over.')
async def task_x(self):
logging.warning('Starting task X')
i = 0
while True:
await asyncio.sleep(2.25)
print(f'Doing task x... {i}')
if i == 2:
raise RuntimeError('BOOM X!')
i += 1
At this point, from the "main" I need to install a few things and create the loop :
def main():
try:
global newobj
loop = asyncio.get_event_loop()
logging.warning(f'Installing exception handler')
loop.set_exception_handler(handle_exception)
logging.warning(f'Creating main object')
newobj = myobj(loop)
logging.warning(f'Installing signal handlers')
signals = (signal.SIGINT, signal.SIGTERM)
for s in signals:
loop.add_signal_handler(s, lambda s=s: loop.create_task(newobj.graceful_shutdown(s)))
logging.warning(f'Running object...')
loop.run_until_complete(newobj.run())
finally:
loop.close()
logging.warning(f'object is Shutdown - Exiting program.')
sys.exit(0)
if __name__ == "__main__":
main()
But the handle_exception needs to be defined.
def handle_exception(loop, context):
# context["message"] will always be there; but context["exception"] may not
msg = context.get("exception", context["message"])
logging.error(f'Caught exception: {msg}')
logging.info(f'Shutting down from exception.')
loop.create_task(newobj.graceful_shutdown())
The problem is that it's never calling handle_exception. I need to be running this in python3.6 for some reason. What am I missing here?

Convert a simple multithreaded program with asyncio

I am still pretty new to Python asyncio, so I am trying to convert a simple problem I solved using multithreading, to using asyncio.
I made an example of what I want to achieve.
Each MiniBot instance can start at random times (those time.sleep() calls in main represent instantiations at unpredictable times.)
I am expecting each MiniBot to run in parallel if they start before others have finished.
All good with MultiThreading, but when I translated the problem with async coroutines, I can't get them to start together.
I could use gather, but this would require to have all the task at the beginning, which I don't.
Any suggestions?
Thanks
Oh yes, I am using Python 3.6
Multithreaded version
import threading
import time
class MiniBot(object):
def __init__(self, _id:str):
self._id = _id
self.alive = True
self.start_time = time.time()
self.th = threading.Thread(target=self.run)
self.internal_state = 0
def _foo(self):
self.internal_state += 1
def run(self):
while self.alive:
self._foo()
if time.time() - self.start_time > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
time.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
self.th.start()
def stop(self):
self.alive = False
if __name__ == "__main__":
# MiniBots activities start at random times but should coexist
MiniBot('a').start()
time.sleep(2)
MiniBot('b').start()
time.sleep(1.5)
MiniBot('c').start()
Output:
Starting Minibot a
Starting Minibot b
Starting Minibot c
Killing minibot: a
Var is: 40
Killing minibot: b
Var is: 40
Killing minibot: c
Var is: 40
Async version (Not behaving as I hoped)
import asyncio
import time
class MiniBot(object):
def __init__(self, _id:str):
self._id = _id
self.alive = True
self.last_event = time.time()
self.internal_state = 0
async def _foo(self):
self.internal_state += 1
asyncio.sleep(2)
async def run(self):
while self.alive:
await self._foo()
if time.time() - self.last_event > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
asyncio.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_until_complete(self.run())
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def stop(self):
self.alive = False
if __name__ == "__main__":
# MiniBots activities start at random times but should coexist
MiniBot('a').start()
time.sleep(2)
MiniBot('b').start()
time.sleep(1.5)
MiniBot('c').start()
Output:
Starting Minibot a
Killing minibot: a
Var is: 2839119
Starting Minibot b
Killing minibot: b
Var is: 2820634
Starting Minibot c
Killing minibot: c
Var is: 2804579

start cannot call run_until_complete because run_until_complete runs a coroutine through to the end, whereas you need multiple coroutines running in parallel. The asyncio equivalent of creating and starting a thread is asyncio.create_task(), so start() should call that, and return to the caller, like in the threaded version.
For example:
import asyncio, time
class MiniBot:
def __init__(self, _id):
self._id = _id
self.alive = True
self.start_time = time.time()
self.internal_state = 0
def _foo(self):
self.internal_state += 1
async def run(self):
while self.alive:
self._foo()
if time.time() - self.start_time > 4:
print(f"Killing minibot: {self._id}")
print(f"Var is: {self.internal_state}")
self.stop()
await asyncio.sleep(0.1)
def start(self):
print(f"Starting Minibot {self._id}")
return asyncio.create_task(self.run())
def stop(self):
self.alive = False
async def main():
taska = MiniBot('a').start()
await asyncio.sleep(2)
taskb = MiniBot('b').start()
await asyncio.sleep(1.5)
taskc = MiniBot('c').start()
await asyncio.gather(taska, taskb, taskc)
if __name__ == "__main__":
#asyncio.run(main())
asyncio.get_event_loop().run_until_complete(main())
Don't let the call to gather() throw you off: gather simply gives back control to the event loop and returns when all the provided tasks/futures have finished. You can replace it with something like await asyncio.sleep(10) and have the same effect (in this example). And if you have an unpredictable number of futures, there are other other ways to signal the end-condition.
Also note that you need to await asyncio.sleep(), otherwise it has no effect.

Python asyncio: interruptable task

I am trying to make a barebones skeleton fighting game with python asyncio.
class Skeleton(Creature):
pass
class SkeletonAI():
def __init__(self, skeleton,loop = None):
self.loop = loop or asyncio.new_event_loop()
self.skeleton = skeleton
self.action_task = None
async def run(self):
while True:
#print(self.action_task, )
if self.skeleton.alive and self.skeleton.target.alive:
if self.skeleton.state == 'idle':
#ATTACK
self.skeleton.begin_attack()
self.action_task = self.loop.call_later(3, self.skeleton.action_complete)
else:
break
class Player(Creature):
def attack_target(self, target):
target.take_damage(self.damage)
if target.state == 'attacking':
target.state = 'idle'
#interrupt attack
class Game():
#Super simple game
#The skeleton launches an attack, press anything to interrupt it
async def handle_sending(self):
loop = asyncio.get_event_loop()
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=1,
)
while True:
msg = await loop.run_in_executor(executor, input)
print('got a message')
if self.skeleton_ai.action_task:
print('cancelling attack')
self.skeleton_ai.action_task.cancel()
self.skeleton_ai.skeleton.machine.set_state('idle')
print('cancelled attack')
self.skeleton_ai.action_task = None
async def game_loop(self):
player_task = asyncio.ensure_future(self.handle_sending())
skeleton_task = asyncio.ensure_future(self.skeleton_ai.run())
def __init__(self):
self.task = None
self.loop = asyncio.get_event_loop()
self.player = Player(name='ply')
self.skeleton_ai = SkeletonAI(skeleton=Skeleton(name='bobby'))
self.skeleton_ai.skeleton.target = self.player
self.loop.run_until_complete(self.game_loop())
try:
self.loop.run_forever()
finally:
pass
loop.close()
Game()
Here's what I am trying to do:
Player input and game output are async, so input() doesn't block. This works.
The skeleton prepares an attack, if it's not interrupted in 3 seconds, the attack deals damage to the player.
The player can input any text to interrupt the skeleton attack.
How can I make the skeleton's attack? I want a task I can interrupt at will and call a callback later. Currently everything just gets stuck. The call_later never calls.

This is the pattern for a async function with timeout and callback function. The clue is to catch the asyncio.TimeoutError and do your timeout logic. The function that is cancelled will not continue after it's current await position.
import asyncio
async def slow_function(seconds):
print('starting slow computations')
await asyncio.sleep(seconds)
print('slow computations done')
async def timeout_callback():
print('timeout called')
async def timeout_with_cb(fut, timeout, timeout_fut):
try:
await asyncio.wait_for(fut, timeout)
except asyncio.TimeoutError:
await timeout_fut
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.ensure_future(
timeout_with_cb(slow_function(2), 1,
timeout_callback())))
This will print:
starting slow computations
timeout called
I guess this can help you to adapt your example (the provided example does not compile).