Given the following program:
1 import asyncio
2 async def run():
3 try:
4 server = await asyncio.start_server(on_connected, '127.0.0.1', 15500)
5
6 async with server:
7 await server.serve_forever()
8 except:
9 print("exception!")
10
11 async def on_connected(reader, writer):
12 while True:
13 data = await reader.readline()
14 print(1 / 0)
15
16 asyncio.run(run())
When I run it, and use nc to connect to it & send data, it will raise an exception in line 14. However I am unable to handle it. Instead, I will get an exception printed and the program will hang.
nc test:
$ nc localhost 15500
test
program output:
$ python3 serv
Task exception was never retrieved
future: <Task finished coro=<on_connected() done, defined at serv:14> exception=ZeroDivisionError('division by zero')>
Traceback (most recent call last):
File "serv", line 14, in on_connected
print(1 / 0)
ZeroDivisionError: division by zero
While I of course could just add a try/except around line 14, I want to implement a general exception handling which shall handle all errors which can ever occur inside of on_connected.
How can I do this?
Asyncio provides a method to set a general exception handler
Let's apply it to this example:
Let's just define our loop exception handler: handle_exception
And inside this method let's manage exceptions. Next code just customize logging messages.
import asyncio
import logging
log = logging.getLogger(__name__)
async def run():
loop = asyncio.get_running_loop()
loop.set_exception_handler(handle_exception)
server = await asyncio.start_server(on_connected, '127.0.0.1', 5062)
async with server:
await server.serve_forever()
async def on_connected(reader, writer):
while True:
data = await reader.readline()
raise print(1 / 0)
def handle_exception(loop, context):
# context["message"] will always be there; but context["exception"] may not
msg = context.get("exception", context["message"])
if name := context.get("future").get_coro().__name__ == "on_connected":
if type(msg) == ZeroDivisionError:
log.error(f"Caught ZeroDivisionError from on_connected: {msg}")
return
log.info(f"Caught another minor exception from on_connected: {msg}")
else:
log.error(f"Caught exception from {name}: {msg}")
Output on this case is the following:
Caught ZeroDivisionError from on_connected: division by zero
Reference that I have used to look for further information:
https://www.roguelynn.com/words/asyncio-exception-handling/
https://github.com/econchick/mayhem/blob/master/part-3/mayhem_2.py
Please, let me know if this answer your question.
When developing server code, bug causing an exception often occurs and then the server process just hangs and must be killed, which is super anoying.
I wanted it to exit cleanly as soon as one exception is raised in the protocol code. To do so, I created a handler class that closes the server and registered it as the exception handler. The call to await server.serve_forever() will throw CancelledError, so you may want to catch it to exit cleanly.
import asyncio
class ExcHandler:
def __init__(self, server):
self._server = server
def __call__(self, loop, context):
print("exception occured, closing server")
loop.default_exception_handler(context)
self._server.close()
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
raise RuntimeError("oups")
async def main():
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
loop.set_exception_handler(ExcHandler(server))
async with server:
try:
await server.serve_forever()
except asyncio.exceptions.CancelledError:
print("server cancelled")
asyncio.run(main())
Apparently, the server catching exceptions is by design (See Python issue #42526). The rationale behind that is the server keeps processing other requests, even when one of them crash. I guess this is suitable for production where the show must go on even in case of glitches.
Related
I have a Python Tornado Websocket server that stores clients in a shared set() so that I know how many clients are connected.
The challenge is that calling on_close after WebSocketClosedError raises a KeyError and the client-instance is not removed from the set of connected clients. This error has caused my server to accumulate over 1000 clients even when the active clients are only around 5.
My Code:
import tornado.iostream
import tornado.websocket
import asyncio
class SocketHandler(tornado.websocket.WebSocketHandler):
socket_active_message = {"status": "Socket Connection Active"}
waiters = set()
def initialize(self):
self.client_name = "newly_connected"
def open(self):
print('connection opened')
# https://kite.com/python/docs/tornado.websocket.WebSocketHandler.set_nodelay
self.set_nodelay(True)
SocketHandler.waiters.add(self)
def on_close(self):
print("CLOSED!", self.client_name)
SocketHandler.waiters.remove(self)
def check_origin(self, origin):
# Override the origin check if needed
return True
async def send_updates(self, message):
print('starting socket service loop')
loop_counter = 0
while True:
try:
await self.write_message({'status': 82317581})
except tornado.websocket.WebSocketClosedError:
self.on_close()
except tornado.iostream.StreamClosedError:
self.on_close()
except Exception as e:
self.on_close()
print('Exception e:', self.client_name)
await asyncio.sleep(0.05)
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message)
def run_server():
# Create tornado application and supply URL routes
webApp = tornado.web.Application(
[
(
r"/",
SocketHandler,
{},
),
]
)
application = tornado.httpserver.HTTPServer(webApp)
webApp.listen(3433)
# Start IO/Event loop
tornado.ioloop.IOLoop.instance().start()
run_server()
The Stack-trace:
Traceback (most recent call last):
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/web.py", line 1699, in _execute
result = await result
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 278, in get
await self.ws_connection.accept_connection(self)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 881, in accept_connection
await self._accept_connection(handler)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 964, in _accept_connection
await self._receive_frame_loop()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1118, in _receive_frame_loop
await self._receive_frame()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1209, in _receive_frame
await handled_future
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 658, in <lambda>
self.stream.io_loop.add_future(result, lambda f: f.result())
File "ask_So.py", line 50, in on_message
await self.send_updates(message)
File "ask_So.py", line 39, in send_updates
self.on_close()
File "ask_So.py", line 26, in on_close
SocketHandler.waiters.remove(self)
KeyError: <__main__.SocketHandler object at 0x7ffef9f25520>
I have tried moving the waiters set outside the class but it still produces the same behaviour.
To simulate WebSocketClosedError: open many browser tabs as clients and close one browser tab at a time.
It seems like self.on_close() is being called twice. Once you're calling it manually from inside send_updates() and then later, when a connection is actually closed, Tornado is also calling self.on_close(). Since the self object was already removed from the set the first time, it raises a KeyError the second time.
If you want to close the connection, just call self.close(). The self.on_close() method will be called by Tornado automatically.
Also, you can handle the exception in a try...except block inside on_close.
Update
The previous part of this answer should fix the KeyError related problem. This update is regarding why the clients are not being removed from waiters set.
So, I tested your code and found a major problem with it here:
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message) # <- This is problematic
Whenever a client sends a message, it will run self.send_updates method. So even if there's only one client that sends a message, let's say, 10 times, send_updates will also be called 10 times and, as a result, you will have 10 while loops running simultaneously!
As the number of loops increase, it ultimately blocks the server. That means Tornado has no time to run other code as it's busy juggling so many while loops. Hence, the clients from the waiters are never removed.
Solution
Instead of calling send_updates everytime a message arrives, you can call it just one time. Just have a single while loop to send updates to all clients.
I'd update the code like this:
class SocketHandler(...):
# Make it a classmethod so that it can be
# called without an instance
#classmethod
async def send_updates(cls):
print('starting socket service loop')
loop_counter = 0
while True:
for waiter in cls.waiters:
# use `waiter` instead of `self`
try:
await waiter.write_message({'status': 82317581})
...
await asyncio.sleep(0.05)
Instead of calling send_updates from on_message, you'll have to tel IOLoop to call it once:
def run_server():
...
# schedule SocketHandler.send_updates to be run
tornado.ioloop.IOLoop.current().add_callback(SocketHandler.send_updates)
tornado.ioloop.IOLoop.current().start()
This will have only one while loop running for all clients.
I'm getting an Exception in a Tornado WebSocket server but It gives no information in the trace to know which line of code or which step in my program it is originating from. I would like to find out so that I try-catch the origin of the exception.
Error Trace: (No mention of any part of my files)
[E 200527 21:07:19 base_events:1608] Task exception was never retrieved
future: <Task finished coro=<WebSocketProtocol13.write_message.<locals>.wrapper() done, defined at /usr/local/lib/python3.7/site-packages/tornado/websocket.py:1102> exception=WebSocketClosedError()>
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/tornado/websocket.py", line 1104, in wrapper
await fut
tornado.iostream.StreamClosedError: Stream is closed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/tornado/websocket.py", line 1106, in wrapper
raise WebSocketClosedError()
tornado.websocket.WebSocketClosedError
[E 200527 21:07:19 base_events:1608] Task exception was never retrieved
That Same Group of Traceback repeats over 16 times.
Here is my code:
import tornado.iostream
import tornado.web
import tornado.gen
import tornado.websocket
import asyncio
class SocketHandler(tornado.websocket.WebSocketHandler):
waiters = set()
def initialize(self):
self.client_name = "newly_connected"
def get_compression_options(self):
# Non-None enables compression with default options.
return {}
def open(self):
print('connection opened')
# SocketHandler.waiters.add(self)
def on_close(self):
print("CLOSED!", self.client_name)
try:
SocketHandler.waiters.remove(self)
except KeyError:
print('tried removing new client')
def check_origin(self, origin):
# Override the origin check if needed
return True
#classmethod
async def send_updates(cls, message):
if len(cls.waiters) < 2:
while True:
chat = {}
# Prevent RuntimeError: Set changed size during iteration
waiters_copy = cls.waiters.copy()
for waiter in waiters_copy:
try:
await waiter.write_message(chat)
except tornado.websocket.WebSocketClosedError:
pass
except tornado.iostream.StreamClosedError:
pass
except Exception as e:
print('Exception e:', waiter.client_name)
pass
# sleep a bit
await asyncio.sleep(0.05)
else:
print('broadcast loop already running')
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
self.first_serve_cache_on_connnect()
SocketHandler.waiters.add(self)
await SocketHandler.send_updates(message)
def first_serve_cache_on_connnect(self):
print('serving cache on connect')
temp_calc_results = self.namespace.results
try:
self.write_message(temp_calc_results)
except Exception as e:
pass
I have tried catching the exceptions that may cause any error while sending messages to the websocket clients but this error still happens when clients connect to the server.
The message "task exception was never retrieved" is not about a missing try/except block, it's about a missing await. In first_serve_cache_on_connect, you call write_message without await, so first_serve_cache_on_connect is no longer running by the time the exception is raised and it has nowhere to go but the logs.
This is basically harmless, but if you want to clean up your logs, you need to make first_serve_cache_on_connect an async def coroutine, call it with await in on_message, and call write_message with await.
I have a bog-standard synchronous python program that needs to be able to read data from websockets and update the GUI with the data. However, asyncio creep is constantly tripping me up.
How do I make a module that:
accepts multiple subscriptions to multiple sources
sends an update to the requester whenever there's data
opens exactly one websocket connection per URL
resets the websocket if it closes
Here's what I have already, but it's failing at many points:
run_forever() means that the loop gets stuck before the subscription completes and then handle() is stuck in the falsey while loop
it does not seem to want to restart sockets when they're down because a websockets object does not have a connected property (websocket without an s does, but I'm not clear on the differences and can't find info online either)
I'm absolutely not sure if my approach is remotely correct.
Been fighting with this for weeks. Would appreciate some pointers.
class WSClient():
subscriptions = set()
connections = {}
started = False
def __init__(self):
self.loop = asyncio.get_event_loop()
def start(self):
self.started = True
self.loop.run_until_complete(self.handle())
self.loop.run_until_forever() # problematic, because it does not allow new subscribe() events
async def handle(self):
while len(self.connections) > 0:
# listen to every websocket
futures = [self.listen(self.connections[url]) for url in self.connections]
done, pending = await asyncio.wait(futures)
# the following is apparently necessary to avoid warnings
# about non-retrieved exceptions etc
try:
data, ws = done.pop().result()
except Exception as e:
print("OTHER EXCEPTION", e)
for task in pending:
task.cancel()
async def listen(self, ws):
try:
async for data in ws:
data = json.loads(data)
# call the subscriber (listener) back when there's data
[s.listener._handle_result(data) for s in self.subscriptions if s.ws == ws]
except Exception as e:
print('ERROR LISTENING; RESTARTING SOCKET', e)
await asyncio.sleep(2)
self.restart_socket(ws)
def subscribe(self, subscription):
task = self.loop.create_task(self._subscribe(subscription))
asyncio.gather(task)
if not self.started:
self.start()
async def _subscribe(self, subscription):
try:
ws = self.connections.get(subscription.url, await websockets.connect(subscription.url))
await ws.send(json.dumps(subscription.sub_msg))
subscription.ws = ws
self.connections[subscription.url] = ws
self.subscriptions.add(subscription)
except Exception as e:
print("ERROR SUBSCRIBING; RETRYING", e)
await asyncio.sleep(2)
self.subscribe(subscription)
def restart_socket(self, ws):
for s in self.subscriptions:
if s.ws == ws and not s.ws.connected:
print(s)
del self.connections[s.url]
self.subscribe(s)
I have a bog-standard synchronous python program that needs to be able to read data from websockets and update the GUI with the data. However, asyncio creep is constantly tripping me up.
As you mentioned GUI, then it is probably not a "bog-standard synchronous python program". Usually a GUI program has a non-blocking event-driven main thread, which allows concurrent user behaviors and callbacks. That is very much similar to asyncio, and it is usually a common way for asyncio to work together with GUIs to use GUI-specific event loop to replace default event loop in asyncio, so that your asyncio coroutines just run in GUI event loop and you can avoid calling run_forever() blocking everything.
An alternative way is to run asyncio event loop in a separate thread, so that your program could at the same time wait for websocket data and wait for user clicks. I've rewritten your code as follows:
import asyncio
import threading
import websockets
import json
class WSClient(threading.Thread):
def __init__(self):
super().__init__()
self._loop = None
self._tasks = {}
self._stop_event = None
def run(self):
self._loop = asyncio.new_event_loop()
self._stop_event = asyncio.Event(loop=self._loop)
try:
self._loop.run_until_complete(self._stop_event.wait())
self._loop.run_until_complete(self._clean())
finally:
self._loop.close()
def stop(self):
self._loop.call_soon_threadsafe(self._stop_event.set)
def subscribe(self, url, sub_msg, callback):
def _subscribe():
if url not in self._tasks:
task = self._loop.create_task(
self._listen(url, sub_msg, callback))
self._tasks[url] = task
self._loop.call_soon_threadsafe(_subscribe)
def unsubscribe(self, url):
def _unsubscribe():
task = self._tasks.pop(url, None)
if task is not None:
task.cancel()
self._loop.call_soon_threadsafe(_unsubscribe)
async def _listen(self, url, sub_msg, callback):
try:
while not self._stop_event.is_set():
try:
ws = await websockets.connect(url, loop=self._loop)
await ws.send(json.dumps(sub_msg))
async for data in ws:
data = json.loads(data)
# NOTE: please make sure that `callback` won't block,
# and it is allowed to update GUI from threads.
# If not, you'll need to find a way to call it from
# main/GUI thread (similar to `call_soon_threadsafe`)
callback(data)
except Exception as e:
print('ERROR; RESTARTING SOCKET IN 2 SECONDS', e)
await asyncio.sleep(2, loop=self._loop)
finally:
self._tasks.pop(url, None)
async def _clean(self):
for task in self._tasks.values():
task.cancel()
await asyncio.gather(*self._tasks.values(), loop=self._loop)
You can try tornado and autobahn-twisted for websockets.
I'm using the Autobahn asyncio system (to talk the Websocket WAMP protocol), which works fine and I can handle incoming RPC calls and pubsub.
My problem is I now have to connect TCP sockets and send information over these sockets as soon as an RPC call comes in through the Autobahn part.
The autobahn part works fine like this :
from autobahn.asyncio.component import Component, run
from asyncio import sleep
from autobahn.asyncio.wamp import ApplicationSession, ApplicationRunner
#comp.on_join
async def joined(session, details):
print("Connected to websocket")
def on_message(msg):
msg = json.loads(msg)
print(msg)
def some_rpc(with_data):
print("Doing something with the data")
return json.dumps({'status': 'OK'})
try:
session.subscribe(on_message, u'some_pubsub_topic')
session.register(some_rpc, u'some_rpc_call')
print("RPC and Pubsub initialized")
except Exception as e:
print("could not subscribe to topic: {0}".format(e))
if __name__ == "__main__":
run([comp])
However now I need to be able to connect to multiple regular TCP sockets :
class SocketClient(asyncio.Protocol):
def __init__(self, loop):
self.data = b''
self.loop = loop
def connection_made(self, transport):
self.transport = transport
print('connected')
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def send(self, data):
self.transport.write(data)
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
loop = asyncio.get_event_loop()
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
loop.run_until_complete(c)
loop.run_forever()
loop.close()
The problem is that, when I combine both and do this :
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
It barks at me and tells me :
StopIteration
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/usr/lib/python3.5/site-packages/autobahn/wamp/websocket.py", line
95, in onMessage
self._session.onMessage(msg) File "/usr/lib/python3.5/site-packages/autobahn/wamp/protocol.py", line
894, in onMessage
on_reply = txaio.as_future(endpoint.fn, *invoke_args, **invoke_kwargs) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 400, in
as_future
return create_future_error(create_failure()) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 393, in
create_future_error
reject(f, error) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 462, in reject
future.set_exception(error.value) File "/usr/lib64/python3.5/asyncio/futures.py", line 365, in set_exception
raise TypeError("StopIteration interacts badly with generators " TypeError: StopIteration interacts badly with generators and cannot be
raised into a Future
Does anyone have any idea on how to call the send function from within the RPC call function ?
In this code:
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
# [...]
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
create_connection is a coroutine function, so c contains a coroutine object. Such object does have a send method, but one that is entirely unrelated to sending things over the network. After calling create_connection, you probably want to get the resulting transport with something like:
transport, ignore = loop.run_until_complete(c)
and then use transport.write(), not c.send().
I am trying to implement a basic websocket client using asyncio and websockets with Python 3.5.2.
Basically, I want connect_to_dealer to be a blocking call, but wait for the websocket message on a different thread.
After reading some docs (I have very little exp with Python), I concluded that asyncio.ensure_future() passing a coroutine (listen_for_message) was the way to go.
Now, I get to run listen_for_message on a different thread, but from within the coroutine I can't seem to use await or any other mechanism to make the calls synchronous. If I do it, the execution waits forever (it hangs) even for a simple sleep.
I'd like to know what I'm doing wrong.
async def listen_for_message(self, future, websocket):
while (True):
try:
await asyncio.sleep(1) # It hangs here
print('Listening for a message...')
message = await websocket.recv() # If I remove the sleep, hangs here
print("< {}".format(message))
future.set_result(message)
future.done()
except websockets.ConnectionClosed as cc:
print('Connection closed')
except Exception as e:
print('Something happened')
def handle_connect_message(self, future):
# We must first remove the websocket-specific payload because we're only interested in the connect protocol msg
print(future.result)
async def connect_to_dealer(self):
print('connect to dealer')
websocket = await websockets.connect('wss://mywebsocket'))
hello_message = await websocket.recv()
print("< {}".format(hello_message))
# We need to parse the connection ID out of the message
connection_id = hello_message['connectionId']
print('Got connection id {}'.format(connection_id))
sub_response = requests.put('https://subscribetotraffic{user_id}?connection={connection_id}'.format(user_id='username', connection_id=connection_id), headers=headers)
if sub_response.status_code == 200:
print('Now we\'re observing traffic')
else:
print('Oops request failed with {code}'.format(code=sub_response.status_code))
# Now we need to handle messages but continue with the regular execution
try:
future = asyncio.get_event_loop().create_future()
future.add_done_callback(self.handle_connect_message)
asyncio.ensure_future(self.listen_for_message(future, websocket))
except Exception as e:
print(e)
Is there a specific reason you need to work with explicit futures?
With asyncio you can use a combination of coroutines and Tasks to achieve most purposes. Tasks are essentially wrapped coroutines that go about cranking themselves over in the background, independently of other async code, so you don't have to explicitly manage their flow or juggle them with other bits of code.
I am not entirely sure of your end goal, but perhaps the approach elaborated below gives you something to work with:
import asyncio
async def listen_for_message(websocket):
while True:
await asyncio.sleep(0)
try:
print('Listening for a message...')
message = await websocket.recv()
print("< {}".format(message))
except websockets.ConnectionClosed as cc:
print('Connection closed')
except Exception as e:
print('Something happened')
async def connect_to_dealer():
print('connect to dealer')
websocket = await websockets.connect('wss://mywebsocket')
hello_message = await websocket.recv()
print("< {}".format(hello_message))
# We need to parse the connection ID out of the message
connection_id = hello_message['connectionId']
print('Got connection id {}'.format(connection_id))
sub_response = requests.put('https://subscribetotraffic{user_id}?connection={connection_id}'.format(
user_id='username', connection_id=connection_id), headers=headers)
if sub_response.status_code == 200:
print('Now we\'re observing traffic')
else:
print('Oops request failed with {code}'.format(code=sub_response.status_code))
async def my_app():
# this will block until connect_to_dealer() returns
websocket = await connect_to_dealer()
# start listen_for_message() in its own task wrapper, so doing it continues in the background
asyncio.ensure_future(listen_for_message(websocket))
# you can continue with other code here that can now coexist with listen_for_message()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(my_app())
loop.run_forever()