Develop Reference

Websocket freezes program if no clients are connected

I'm creating a real-time event detector. It reads a gaming stream and detects some events there like goals, timer updates and etc. All of the events should be sent via websocket, so I use python websockets and asyncio libraries. Code is the following:
main_stack = []
cap = cv2.VideoCapture("vid_2.mp4")
game = Game(main_stack)
async def handler(websocket):
while True:
ret, frame = cap.read()
await asyncio.sleep(0.05)
game.run_detectors(frame)
while main_stack:
elem = main_stack.pop()
await websocket.send(elem.get_json())
async def main():
async with websockets.serve(handler, "", 8765):
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())
Line game.run_detectors(frame) adds events to main stack if detected. Then if there are some events in the stack, they're sent via websocket. The problem is that when no client is connected, program freezes, missing events from translation which runs in real time. How can I change the code, so that the stack is updated independently of websocket. And websocket constantly checks updates and sends events if they're present in the stack?

You should run the game in a separate thread and add events to a queue, then pop them and send them in a websocket handler. Note that this will only work for a single connection, if you have multiple connections you need to write some kind of logic which holds the state (which events were already sent, etc.) for each of the connections. Without the additional logic only one of the connections will receive an event, not all of them.
Below is a basic example how it should look like.
from threading import Thread
from queue import Queue, Empty
import time
main_stack = Queue()
cap = cv2.VideoCapture("vid_2.mp4")
game = Game(main_stack) # Adapt logic in the Game class to put events in the queue
async def handler(websocket):
while True:
try:
elem = main_stack.get(timeout=1e-3) # Try to fetch an event from queue with a 1 ms timeout in order to prevent high CPU usage
except Empty:
continue # In case the queue is empty, repeat fetching
await websocket.send(elem.get_json())
async def main():
async with websockets.serve(handler, "", 8765):
await asyncio.Future()
def game_loop(game, cap):
while True:
ret, frame = cap.read()
game.run_detectors(frame)
time.sleep(0.05)
if __name__ == "__main__":
game_thread = Thread(target=game_loop, args=(game, cap), daemon=True)
game_thread.start()
asyncio.run(main())

Send message using websockets with another thread without RuntimeWarning: coroutine 'WebSocketCommonProtocol.send' was never awaited

Am trying to send motion sensor data in json string using websocket client in another thread to avoid execution blocking for the rest of the code down by an infinite loop in MotionSensor class. but apparently the ws.send() needs await keyword. And if i add it throught i get an error
RuntimeWarning: coroutine 'MotionSensors.run' was never awaited
self.run()
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
and it doesn't send anything to the server
# motionSensor.py
import threading
import time
from client.ClientRequest import Request
class MotionSensors(threading.Thread):
def __init__(self, ws):
threading.Thread.__init__(self)
self.ws = ws
self.open = True
async def run(self):
await self.SendData()
async def SendData(self):
while self.open:
print("Sending motion state....")
state = 1 # Motion state demo value
request = Request()
request.push("mcu/sensors/motion")
request.addBody({
"state_type": "single",
"devices": {"state": state, "device_no": "DVC-876435"}
})
await self.ws.send(request.getAsJsonString())
print("sleeping now for 2 seconds....")
time.sleep(2)
here is my main code
client.py
# client.py
import settings
import asyncio
import websockets
from client.ClientHandler import Devices
from client.Rounte import Route
from ClientRequest import Request
from client.dbHandler import mcuConfig
from client.devices.motionSensor import MotionSensors
def ResponseMSG(request):
print(request)
route = Route()
route.addRoute("/response", ResponseMSG)
def onMessage(request):
route.fireRequest(request)
async def WsClient():
uri = settings.WS_URL
async with websockets.connect(uri) as websocket:
#####################################
###INITIALIZE DEVICES
motion = MotionSensors(websocket)
motion.start()
while True:
print("waiting to recieve......")
message = await websocket.recv()
onMessage(message)
loop = asyncio.get_event_loop()
loop.run_until_complete(WsClient())
loop.run_forever()
Guys i need your help to send data in another thread with a while loop without blocking the execution of code down and without errors. Thank you so much in advance

Actually i changed the code of motionSensor.py
i created a new event loop and i set it to the new thread
and it worked for the case of even those using python 3.7 and below. it works. All thanks to #user4815162342
# motionSensor.py
import threading
import time
import asyncio
from client.ClientRequest import Request
class MotionSensor(threading.Thread):
def __init__(self, ws):
threading.Thread.__init__(self)
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self.ws = ws
self.open = True
def run(self):
self.loop.run_until_complete(self.SendData())
# #asyncio.coroutine
async def SendData(self):
while True:
print("Sending motion state....")
state = 0
request = Request()
request.push("mcu/sensors/motion")
request.addBody({
"state_type": "single",
"devices": {"state": state, "device_no": "DVC-876435"}
})
await self.ws.send(request.getAsJsonString())
print("sleeping now for 5 seconds....")
time.sleep(5)

serial_asyncio: read serial port into a queue and read that queue from main() not working

I have a function do_serial() that reads from the serial port and adds the data to a queue. I want to process that queue on main().
I have read the serial_asyncio docs and have a basic echo type proof that reads serial data and echoes it back from a single async function. What I cannot get to work is updating the queue and reading the queue from main().
Once I have this working I also want to update another queue on main() and have that get written to the serial port. This way do_serial() can run async and process the serial port and main() interacts with the queues.
Here is my code:
import queue
import asyncio
import serial_asyncio
incoming_serial_queue = queue.Queue()
async def do_serial():
reader, writer = await serial_asyncio.open_serial_connection(url='/dev/ttyS0', baudrate=9600)
while True:
# read serial port
data = await reader.readline()
# echo back to serial port for verification
writer.write(incoming_serial_queue.get)
# add data to queue
incoming_serial_queue.put(data)
async def main():
while True:
# read queued data from serial port
await print(incoming_serial_queue.get())
asyncio.run(do_serial())
asyncio.run(main())

I'm not specialist for asyncio but I created some working code - so don't ask why something doesn't work :)
I can't run serial_asyncio so I used datetime to generate some data for queue
You can't use run() to start two processes at the same time. run() starts one process and waits for its end - so run(do_serial()) starts function do_serial and waits for its end - and this way it never runs main. If you change order then run(main()) will wait for end of main and it will never run do_serial
So I used run() to start one function which use create_task() for do_serial() and main(). I had to first create task and later use them with await because in other way it didn't start main and I don't know why.
Other problem with asyncio: await can't work with every function - it will not work with standard print(). You would have to create special print() for it. Or maybe you have special print() from serial_asyncio - I don't know.
In both functions I used asyncio.sleep() to have some function with await
Problem with queue: when there is no data in queue then get() may block code and it is better to check first if it is not empty()
import queue
import asyncio
import datetime
incoming_serial_queue = queue.Queue()
async def do_serial():
print("do serial")
while True:
data = str(datetime.datetime.now())
print('put:', data)
# add data to queue
incoming_serial_queue.put(data)
await asyncio.sleep(2)
async def main():
print("main")
while True:
# read queued data from serial port
if not incoming_serial_queue.empty():
data = incoming_serial_queue.get()
print('get:', data)
await asyncio.sleep(1)
async def start():
print("start")
#await do_serial() # doesn't work
#await main() # doesn't work
# I had to create all tasks before running
task1 = asyncio.create_task(do_serial())
task2 = asyncio.create_task(main())
# running task
await task1
await task2
asyncio.run(start())
EDIT:
I could run two functions using asyncio.gather()
async def start():
print("start")
await asyncio.gather(
do_serial(),
main(),
)
Rest is the same as in my previous code
import queue
import asyncio
import datetime
incoming_serial_queue = queue.Queue()
async def do_serial():
print("do serial")
while True:
data = str(datetime.datetime.now())
print('put:', data)
# add data to queue
incoming_serial_queue.put(data)
await asyncio.sleep(2)
async def main():
print("main")
while True:
# read queued data from serial port
if not incoming_serial_queue.empty():
data = incoming_serial_queue.get()
print('get:', data)
await asyncio.sleep(1)
async def start():
print("start")
await asyncio.gather(
do_serial(),
main(),
)
asyncio.run(start())

Python: Websockets in Synchronous Program

I have a bog-standard synchronous python program that needs to be able to read data from websockets and update the GUI with the data. However, asyncio creep is constantly tripping me up.
How do I make a module that:
accepts multiple subscriptions to multiple sources
sends an update to the requester whenever there's data
opens exactly one websocket connection per URL
resets the websocket if it closes
Here's what I have already, but it's failing at many points:
run_forever() means that the loop gets stuck before the subscription completes and then handle() is stuck in the falsey while loop
it does not seem to want to restart sockets when they're down because a websockets object does not have a connected property (websocket without an s does, but I'm not clear on the differences and can't find info online either)
I'm absolutely not sure if my approach is remotely correct.
Been fighting with this for weeks. Would appreciate some pointers.
class WSClient():
subscriptions = set()
connections = {}
started = False
def __init__(self):
self.loop = asyncio.get_event_loop()
def start(self):
self.started = True
self.loop.run_until_complete(self.handle())
self.loop.run_until_forever() # problematic, because it does not allow new subscribe() events
async def handle(self):
while len(self.connections) > 0:
# listen to every websocket
futures = [self.listen(self.connections[url]) for url in self.connections]
done, pending = await asyncio.wait(futures)
# the following is apparently necessary to avoid warnings
# about non-retrieved exceptions etc
try:
data, ws = done.pop().result()
except Exception as e:
print("OTHER EXCEPTION", e)
for task in pending:
task.cancel()
async def listen(self, ws):
try:
async for data in ws:
data = json.loads(data)
# call the subscriber (listener) back when there's data
[s.listener._handle_result(data) for s in self.subscriptions if s.ws == ws]
except Exception as e:
print('ERROR LISTENING; RESTARTING SOCKET', e)
await asyncio.sleep(2)
self.restart_socket(ws)
def subscribe(self, subscription):
task = self.loop.create_task(self._subscribe(subscription))
asyncio.gather(task)
if not self.started:
self.start()
async def _subscribe(self, subscription):
try:
ws = self.connections.get(subscription.url, await websockets.connect(subscription.url))
await ws.send(json.dumps(subscription.sub_msg))
subscription.ws = ws
self.connections[subscription.url] = ws
self.subscriptions.add(subscription)
except Exception as e:
print("ERROR SUBSCRIBING; RETRYING", e)
await asyncio.sleep(2)
self.subscribe(subscription)
def restart_socket(self, ws):
for s in self.subscriptions:
if s.ws == ws and not s.ws.connected:
print(s)
del self.connections[s.url]
self.subscribe(s)

I have a bog-standard synchronous python program that needs to be able to read data from websockets and update the GUI with the data. However, asyncio creep is constantly tripping me up.
As you mentioned GUI, then it is probably not a "bog-standard synchronous python program". Usually a GUI program has a non-blocking event-driven main thread, which allows concurrent user behaviors and callbacks. That is very much similar to asyncio, and it is usually a common way for asyncio to work together with GUIs to use GUI-specific event loop to replace default event loop in asyncio, so that your asyncio coroutines just run in GUI event loop and you can avoid calling run_forever() blocking everything.
An alternative way is to run asyncio event loop in a separate thread, so that your program could at the same time wait for websocket data and wait for user clicks. I've rewritten your code as follows:
import asyncio
import threading
import websockets
import json
class WSClient(threading.Thread):
def __init__(self):
super().__init__()
self._loop = None
self._tasks = {}
self._stop_event = None
def run(self):
self._loop = asyncio.new_event_loop()
self._stop_event = asyncio.Event(loop=self._loop)
try:
self._loop.run_until_complete(self._stop_event.wait())
self._loop.run_until_complete(self._clean())
finally:
self._loop.close()
def stop(self):
self._loop.call_soon_threadsafe(self._stop_event.set)
def subscribe(self, url, sub_msg, callback):
def _subscribe():
if url not in self._tasks:
task = self._loop.create_task(
self._listen(url, sub_msg, callback))
self._tasks[url] = task
self._loop.call_soon_threadsafe(_subscribe)
def unsubscribe(self, url):
def _unsubscribe():
task = self._tasks.pop(url, None)
if task is not None:
task.cancel()
self._loop.call_soon_threadsafe(_unsubscribe)
async def _listen(self, url, sub_msg, callback):
try:
while not self._stop_event.is_set():
try:
ws = await websockets.connect(url, loop=self._loop)
await ws.send(json.dumps(sub_msg))
async for data in ws:
data = json.loads(data)
# NOTE: please make sure that `callback` won't block,
# and it is allowed to update GUI from threads.
# If not, you'll need to find a way to call it from
# main/GUI thread (similar to `call_soon_threadsafe`)
callback(data)
except Exception as e:
print('ERROR; RESTARTING SOCKET IN 2 SECONDS', e)
await asyncio.sleep(2, loop=self._loop)
finally:
self._tasks.pop(url, None)
async def _clean(self):
for task in self._tasks.values():
task.cancel()
await asyncio.gather(*self._tasks.values(), loop=self._loop)

You can try tornado and autobahn-twisted for websockets.

Python3 Asyncio Creating a secondary connection from an initial one

I am trying to add two coroutines to asyncio loop and getting an error:
RuntimeError: This event loop is already running
My objective is to communicate to a server (that I have no control of). This server expects an initial connection from the client. The server then provided a port to the client on this connection. The client has to use this port to create a second connection. This second connection is used by the server to send unsolicited messages to the client. The first connection remains up throughout for other two-way communications.
To recreate this scenario, I have some code that reproduces the error:
class Connection():
def __init__(self, ip, port, ioloop):
self.ip = ip
self.port = port
self.ioloop = ioloop
self.reader, self.writer = None, None
self.protocol = None
self.fileno = None
async def __aenter__(self):
# Applicable when doing 'with Connection(...'
log.info("Entering and Creating Connection")
self.reader, self.writer = (
await asyncio.open_connection(self.ip, self.port, loop=self.ioloop)
)
self.protocol = self.writer.transport.get_protocol()
self.fileno = self.writer.transport.get_extra_info('socket').fileno()
log.info(f"Created connection {self}")
return self
async def __aexit__(self, *args):
# Applicable when doing 'with Connection(...'
log.info(f"Exiting and Destroying Connection {self}")
if self.writer:
self.writer.close()
def __await__(self):
# Applicable when doing 'await Connection(...'
return self.__aenter__().__await__()
def __repr__(self):
return f"[Connection {self.ip}:{self.port}, {self.protocol}, fd={self.fileno}]"
async def send_recv_message(self, message):
log.debug(f"send: '{message}'")
self.writer.write(message.encode())
await self.writer.drain()
log.debug("awaiting data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv: '{data}'")
return data
class ServerConnection(Connection):
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
self.ioloop.run_until_complete(event_connection.recv_message())
class EventConnection(Connection):
async def recv_message(self):
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv only: '{data}'")
return data
async def main(loop):
client1 = await ServerConnection('127.0.0.1', 8888, loop)
await client1.setup_connection()
await client1.send_recv_message("Hello1")
await client1.send_recv_message("Hello2")
await asyncio.sleep(5)
if __name__ == '__main__':
#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
The error occurs in ServerConnection.setup_connection() method where run_until_complete is being called.
I am probably doing something wrong due to lack of understanding asyncio. Basically, how do I setup a secondary connection which will get event notifications (unsolicited) while setting up the first connection?
Thanks.
Followup
Since the code is very similar (a few changes to add more functionality to it), I hope it's not bad etiquette to followup to the original post as the resulting error is still the same.
The new issue is that when it receives the unsolicited message (which is received by EventConnection), the recv_message calls process_data method. I would like to make process_data be a future so that recv_message completes (ioloop should stop). The ensure_future would then pick it up and continue running again to use ServerConnection to do a request/response to the server. Before it does that though, it has to go to some user code (represented by external_command() and from whom I would prefer to hide the async stuff). This would make it synchronous again. Hence, once they've done what they need to, they should call execute_command on ServerConnection, which then kicks off the loop again.
The problem is, my expectation for using ensure_future didn't pan out as it seems the loop didn't stop from running. Hence, when the code execution reaches execute_command which does the run_until_complete, an exception with the error "This event loop is already running" occurs.
I have two questions:
How can I make it so that the ioloop can stop after process_data is
placed into ensure_future, and subsequently be able to run it again
in execute_command?
Once recv_message has received something, how can we make it so that
it can receive more unsolicited data? Is it enough/safe to just use
ensure_future to call itself again?
Here's the example code that simulates this issue.
client1 = None
class ServerConnection(Connection):
connection_type = 'Server Connection'
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
asyncio.ensure_future(event_connection.recv_message())
async def _execute_command(self, data):
return await self.send_recv_message(data)
def execute_command(self, data):
response_str = self.ioloop.run_until_complete(self._execute_command(data))
print(f"exec cmd response_str: {response_str}")
def external_command(self, data):
self.execute_command(data)
class EventConnection(Connection):
connection_type = 'Event Connection'
async def recv_message(self):
global client1
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv-only: '{data}'")
asyncio.ensure_future(self.process_data(data))
asyncio.ensure_future(self.recv_message())
async def process_data(self, data):
global client1
await client1.external_command(data)
async def main(ioloop):
global client1
client1 = await ServerConnection('127.0.0.1', 8888, ioloop)
await client1.setup_connection()
print(f"after connection setup loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello1")
print(f"after Hello1 loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello2")
print(f"after Hello2 loop running is {ioloop.is_running()}")
while True:
print(f"inside while loop running is {ioloop.is_running()}")
t = 10
print(f"asyncio sleep {t} sec")
await asyncio.sleep(t)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()

Try replacing:
self.ioloop.run_until_complete
With
await