I'm trying to get my head around the Python 3 asyncio module, in particular using the transport/protocol API. I want to create a publish/subscribe pattern, and use the asyncio.Protocol class to create my client and server.
At the moment I've got the server up and running, and listening for incoming client connections. The client is able to connect to the server, send a message and receive the reply.
I would like to be able to keep the TCP connection alive and maintain a queue that allows me to add messages. I've tried to find a way to do this using the low-level API (Transport/Protocols) but the limited asyncio docs/examples online all seem to go into the high level API - using streams, etc. Would someone be able to point me in the right direction on how to implement this?
Here's the server code:
#!/usr/bin/env python3
import asyncio
import json
class SubscriberServerProtocol(asyncio.Protocol):
""" A Server Protocol listening for subscriber messages """
def connection_made(self, transport):
""" Called when connection is initiated """
self.peername = transport.get_extra_info('peername')
print('connection from {}'.format(self.peername))
self.transport = transport
def data_received(self, data):
""" The protocol expects a json message containing
the following fields:
type: subscribe/unsubscribe
channel: the name of the channel
Upon receiving a valid message the protocol registers
the client with the pubsub hub. When succesfully registered
we return the following json message:
type: subscribe/unsubscribe/unknown
channel: The channel the subscriber registered to
channel_count: the amount of channels registered
"""
# Receive a message and decode the json output
recv_message = json.loads(data.decode())
# Check the message type and subscribe/unsubscribe
# to the channel. If the action was succesful inform
# the client.
if recv_message['type'] == 'subscribe':
print('Client {} subscribed to {}'.format(self.peername,
recv_message['channel']))
send_message = json.dumps({'type': 'subscribe',
'channel': recv_message['channel'],
'channel_count': 10},
separators=(',', ':'))
elif recv_message['type'] == 'unsubscribe':
print('Client {} unsubscribed from {}'
.format(self.peername, recv_message['channel']))
send_message = json.dumps({'type': 'unsubscribe',
'channel': recv_message['channel'],
'channel_count': 9},
separators=(',', ':'))
else:
print('Invalid message type {}'.format(recv_message['type']))
send_message = json.dumps({'type': 'unknown_type'},
separators=(',', ':'))
print('Sending {!r}'.format(send_message))
self.transport.write(send_message.encode())
def eof_received(self):
""" an EOF has been received from the client.
This indicates the client has gracefully exited
the connection. Inform the pubsub hub that the
subscriber is gone
"""
print('Client {} closed connection'.format(self.peername))
self.transport.close()
def connection_lost(self, exc):
""" A transport error or EOF is seen which
means the client is disconnected.
Inform the pubsub hub that the subscriber has
Disappeared
"""
if exc:
print('{} {}'.format(exc, self.peername))
loop = asyncio.get_event_loop()
# Each client will create a new protocol instance
coro = loop.create_server(SubscriberServerProtocol, '127.0.0.1', 10666)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
try:
server.close()
loop.until_complete(server.wait_closed())
loop.close()
except:
pass
And here's the client code:
#!/usr/bin/env python3
import asyncio
import json
class SubscriberClientProtocol(asyncio.Protocol):
def __init__(self, message, loop):
self.message = message
self.loop = loop
def connection_made(self, transport):
""" Upon connection send the message to the
server
A message has to have the following items:
type: subscribe/unsubscribe
channel: the name of the channel
"""
transport.write(self.message.encode())
print('Message sent: {!r}'.format(self.message))
def data_received(self, data):
""" After sending a message we expect a reply
back from the server
The return message consist of three fields:
type: subscribe/unsubscribe
channel: the name of the channel
channel_count: the amount of channels subscribed to
"""
print('Message received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
if __name__ == '__main__':
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: SubscriberClientProtocol(message,
loop),
'127.0.0.1', 10666)
loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print('Closing connection')
loop.close()
Your server is fine as-is for what you're trying to do; your code as written actually keeps the TCP connection alive, it's you just don't have the plumbing in place to continously feed it new messages. To do that, you need to tweak the client code so that you can feed new messages into it whenever you want, rather than only doing it when the connection_made callback fires.
This is easy enough; we'll add an internal asyncio.Queue to the ClientProtocol which can receive messages, and then run a coroutine in an infinite loop that consumes the messages from that Queue, and sends them on to the server. The final piece is to actually store the ClientProtocol instance you get back from the create_connection call, and then pass it to a coroutine that actually sends messages.
import asyncio
import json
class SubscriberClientProtocol(asyncio.Protocol):
def __init__(self, loop):
self.transport = None
self.loop = loop
self.queue = asyncio.Queue()
self._ready = asyncio.Event()
asyncio.async(self._send_messages()) # Or asyncio.ensure_future if using 3.4.3+
#asyncio.coroutine
def _send_messages(self):
""" Send messages to the server as they become available. """
yield from self._ready.wait()
print("Ready!")
while True:
data = yield from self.queue.get()
self.transport.write(data.encode('utf-8'))
print('Message sent: {!r}'.format(message))
def connection_made(self, transport):
""" Upon connection send the message to the
server
A message has to have the following items:
type: subscribe/unsubscribe
channel: the name of the channel
"""
self.transport = transport
print("Connection made.")
self._ready.set()
#asyncio.coroutine
def send_message(self, data):
""" Feed a message to the sender coroutine. """
yield from self.queue.put(data)
def data_received(self, data):
""" After sending a message we expect a reply
back from the server
The return message consist of three fields:
type: subscribe/unsubscribe
channel: the name of the channel
channel_count: the amount of channels subscribed to
"""
print('Message received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
#asyncio.coroutine
def feed_messages(protocol):
""" An example function that sends the same message repeatedly. """
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
while True:
yield from protocol.send_message(message)
yield from asyncio.sleep(1)
if __name__ == '__main__':
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: SubscriberClientProtocol(loop),
'127.0.0.1', 10666)
_, proto = loop.run_until_complete(coro)
asyncio.async(feed_messages(proto)) # Or asyncio.ensure_future if using 3.4.3+
try:
loop.run_forever()
except KeyboardInterrupt:
print('Closing connection')
loop.close()
Related
I trying to use Python's logging and asyncio libraries to send logs via WebSockets. For context Python's logging runs logging handlers in different threads.
In the code bellow the event loop and all WebSocket connections get passed to the log handler so it can broadcast the messages to everyone.
class ESSocketLoggerHandlerWS(logging.Handler, object):
def __init__(self, name, other_attr=None, **kwargs):
logging.Handler.__init__(self)
self.connections = []
self.loop = kwargs.get("loop")
def addConnection(self, connection):
self.connections.append(connection)
async def send(self, socketcon, msg):
print("Sending...")
await socketcon.send(msg)
def emit(self, record):
msg = (self.format(record) + "\n").encode("utf-8")
# Remove old clients no longer connected
self.connections[:] = [x for x in self.connections if not x.closed]
# Send logs to the available sockets
for socketcon in self.connections:
print("Sending message to all WS clients")
self.loop.call_soon_threadsafe(self.send(socketcon, msg))
eventLoop = asyncio.new_event_loop()
asyncio.set_event_loop(eventLoop);
esSocketHandlerWS = ESSocketLoggerHandlerWS(name='mainlogger', loop=eventLoop)
log.addHandler(esSocketHandlerWS)
async def handle_client_ws(websocket):
log.info('New websocket connection')
esSocketHandlerWS.addConnection(websocket)
async for message in websocket:
# do suff with incoming messages...
async def main():
serverWs = await websockets.serve(handle_client_ws, "127.0.0.1", 8765)
async with server, serverWs:
await asyncio.gather(
...,
serverWs.serve_forever()
)
asyncio.run(main())
Unfortunately it doesn't work nor gives me any errors. Clients can connect and send messages but no logs get to them. I get Sending message to all WS clients in stdout, however print("Sending...") is never called nor await socketcon.send(msg).
What's the proper way of doing this? Thank you.
I have a publisher/subscriber architecture running on my websocket server, where the publisher runs in one thread, and the websocket server in another. I connect to the server from the publisher over localhost, and the server distributes the published messages to any other connected clients on the /sub path. However, since the publisher thread not always has new data to publish, it has a tendency to disconnect after a timeout of 50 sec. To fix this, I implemented a heartbeat ping function:
async def ping(websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
This keeps the publisher from disconnecting. However, when I'm trying to run this concurrently with the coroutine that sends the actual data, I cannot get both ping() and send_data() to run in parallel. I've tried just awaiting both functions as well as asyncio.gather() (which according to documentation is supposed to run tasks concurrently) as well as flipping the order, but it seems like in all cases only the first function call is ran.
My thread class for reference:
class Publisher(threading.Thread):
"""
Thread acting as the websocket publisher
Pulls data from the data merger queue and publishes onto the websocket server
"""
def __init__(self, loop, q, addr, port):
threading.Thread.__init__(self)
self.loop = loop
self.queue = q
self.id = threading.get_ident()
self.addr = addr
self.port = port
self.name = 'publisher'
print("Publisher thread started (ID:%s)" % self.id)
def run(self):
self.loop.run_until_complete(self.publish())
asyncio.get_event_loop().run_forever()
async def ping(self, websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
async def send_data(self, websocket):
while True:
try:
msg = json.dumps(self.queue.get()) # Get the data from the queue
print(msg)
await asyncio.sleep(0.1)
if not msg:
print("No message")
break
await websocket.send(msg)
except websockets.exceptions.ConnectionClosedError:
print("Connection closed")
break
async def publish(self):
uri = 'ws://' + str(self.addr) + ':' + str(self.port) + '/pub'
async with websockets.connect(uri) as websocket:
await asyncio.gather(
self.ping(websocket),
self.send_data(websocket)
)
I'm working on a application. Where am using python websockets. Now I need UDP and WS asynchronously running and listening on different ports.
I'm unable to do it because WS recv() waits indefinitely untill a message is received. Message will be received and pushed into queue. I need UDP to receive and push to same queue. This below class implements only websockets. I need another class with UDP and both class instance run asynchronously.
import websockets
import json
from sinric.command.mainqueue import queue
from sinric.callback_handler.cbhandler
import CallBackHandler
from time import sleep
class SinricProSocket:
def __init__(self, apiKey, deviceId, callbacks):
self.apiKey = apiKey
self.deviceIds = deviceId
self.connection = None
self.callbacks = callbacks
self.callbackHandler = CallBackHandler(self.callbacks)
pass
async def connect(self): # Producer
self.connection = await websockets.client.connect('ws://2.5.2.2:301',
extra_headers={'Authorization': self.apiKey,
'deviceids': self.deviceIds},
ping_interval=30000, ping_timeout=10000)
if self.connection.open:
print('Client Connected')
return self.connection
async def sendMessage(self, message):
await self.connection.send(message)
async def receiveMessage(self, connection):
try:
message = await connection.recv()
queue.put(json.loads(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
async def handle(self):
# sleep(6)
while queue.qsize() > 0:
await self.callbackHandler.handleCallBacks(queue.get(), self.connection)
return
thanks for your time in the comments. I solved this issue by running instances of WS and UDP in 2 different daemon threads.
A good way to solve this issue would be to use threads. You could accept a message and put it into a queue, then handle the queue on a different thread.
I have asynsio server on Python 3.7.
For each connection, asyncio creates a new EchoServerProtocol() object. After receiving the first packet, the server closes the connection, but the EchoServerProtocol() object remains in memory. Can you please tell me how to correctly remove it? I understand that somewhere in asyncio there are links to it.
server.py
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def __init__(self):
self.__loop = asyncio.get_event_loop()
def connection_made(self, transport):
self.__loop.call_later(5, self.check_connection)
print('Connection made')
self.transport = transport
def connection_lost(self, exc):
print('Connection lost')
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def check_connection(self):
print('check connection here')
self.__loop.call_later(5, self.check_connection)
async def main():
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
client.py
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost, loop):
self.message = message
self.loop = loop
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost, loop),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
Output:
Connection made
Data received: 'Hello World!'
Send: 'Hello World!'
Close the client socket
Connection lost
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here
check connection here
After receiving the first packet, the server closes the connection, but the EchoServerProtocol() object remains in memory.
Looking at your code, it is check_connection that is keeping the object in memory. Specifically, the end of check_connection says:
self.__loop.call_later(5, self.check_connection)
This means: "after 5 seconds, invoke check_connection on self". The fact that self is a protocol that is no longer in use doesn't matter - the event loop is told to invoke a function 5 seconds later, and it will do exactly that.
If you want to have a monitoring task, you should make it a coroutine and cancel it when the connection is lost. For example:
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
loop = asyncio.get_event_loop()
self._checker = loop.create_task(self.check_connection())
print('Connection made')
self.transport = transport
def connection_lost(self, exc):
print('Connection lost')
self._checker.cancel()
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def check_connection(self):
while True:
print('check connection here')
await asyncio.sleep(5)
user4815162342 is right. Thanks for your reply. My stupid mistake.
If you want to have a monitoring task, you should make it a coroutine and cancel it when >the connection is lost. For example:
As for the check_connection method, I just changed my code and added the del handler to make sure that the object is being deleted.
def check_connection(self):
print('check connection here')
if not self.transport.is_closing():
self.__loop.call_later(5, self.check_connection)
def __del__(self):
print("destroy protocol")
I'm trying to build a websocket client on Python using websockets package from here: Websockets 4.0 API
I'm using this way instead of example code because I want to create a websocket client class object, and use it as gateway.
I'm having issues with my listener method (receiveMessage) on client side, which raises a ConnectionClose exception at execution. I think maybe there is any problem with the loop.
This is the simple webSocket client I've tried to build:
import websockets
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
# Enable listener
await self.receiveMessage()
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self):
'''
Receiving all server messages and handling them
'''
while True:
message = await self.connection.recv()
print('Received message from server: ' + str(message))
And this is the main:
'''
Main file
'''
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
loop.run_until_complete(client.connect())
loop.run_forever()
loop.close()
To test incoming messages listener, server sends two messages to client when it stablishes the connection.
Client connects correctly to server, and sends the greeting. However, when client receives both messages, it raises a ConnectionClosed exception with code 1000 (no reason).
If I remove the loop in the receiveMessage client method, client does not raise any exception, but it only receives one message, so I suppose I need a loop to keep listener alive, but I don't know exactly where or how.
Any solution?
Thanks in advance.
EDIT: I realize that client closes connection (and breaks loop) when it receives all pending messages from server. However, I want client keeps alive listening future messages.
In addition, I've tried to add another function whose task is to send a 'heartbeat' to server, but client closes connection anyway.
Finally, based on this post answer, I modified my client and main files this way:
WebSocket Client:
import websockets
import asyncio
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
return self.connection
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self, connection):
'''
Receiving all server messages and handling them
'''
while True:
try:
message = await connection.recv()
print('Received message from server: ' + str(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
async def heartbeat(self, connection):
'''
Sending heartbeat to server every 5 seconds
Ping - pong messages to verify connection is alive
'''
while True:
try:
await connection.send('ping')
await asyncio.sleep(5)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
Main:
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
# Start connection and get client connection protocol
connection = loop.run_until_complete(client.connect())
# Start listener and heartbeat
tasks = [
asyncio.ensure_future(client.heartbeat(connection)),
asyncio.ensure_future(client.receiveMessage(connection)),
]
loop.run_until_complete(asyncio.wait(tasks))
Now, client keeps alive listening all messages from server and sending 'ping' messages every 5 seconds to server.