Develop Reference

Python: asyncio websockets sending messages from different threads

I trying to use Python's logging and asyncio libraries to send logs via WebSockets. For context Python's logging runs logging handlers in different threads.
In the code bellow the event loop and all WebSocket connections get passed to the log handler so it can broadcast the messages to everyone.
class ESSocketLoggerHandlerWS(logging.Handler, object):
def __init__(self, name, other_attr=None, **kwargs):
logging.Handler.__init__(self)
self.connections = []
self.loop = kwargs.get("loop")
def addConnection(self, connection):
self.connections.append(connection)
async def send(self, socketcon, msg):
print("Sending...")
await socketcon.send(msg)
def emit(self, record):
msg = (self.format(record) + "\n").encode("utf-8")
# Remove old clients no longer connected
self.connections[:] = [x for x in self.connections if not x.closed]
# Send logs to the available sockets
for socketcon in self.connections:
print("Sending message to all WS clients")
self.loop.call_soon_threadsafe(self.send(socketcon, msg))
eventLoop = asyncio.new_event_loop()
asyncio.set_event_loop(eventLoop);
esSocketHandlerWS = ESSocketLoggerHandlerWS(name='mainlogger', loop=eventLoop)
log.addHandler(esSocketHandlerWS)
async def handle_client_ws(websocket):
log.info('New websocket connection')
esSocketHandlerWS.addConnection(websocket)
async for message in websocket:
# do suff with incoming messages...
async def main():
serverWs = await websockets.serve(handle_client_ws, "127.0.0.1", 8765)
async with server, serverWs:
await asyncio.gather(
...,
serverWs.serve_forever()
)
asyncio.run(main())
Unfortunately it doesn't work nor gives me any errors. Clients can connect and send messages but no logs get to them. I get Sending message to all WS clients in stdout, however print("Sending...") is never called nor await socketcon.send(msg).
What's the proper way of doing this? Thank you.

how do i switch between connected clients with asyncio sockets

I am trying to make a socket server that's able to have multiple clients connected using the asyncio sockets and is able to easily switch between which client it communicates while still having all the clients connected. I thought there would be some type of FD of the clients like there is in sockets, but I looked through the docs and did not find anything, or I missed it.
Here is my server code:
import socket
import asyncio
host = "localhost"
port = 9998
list_of_auths = ['desktop-llpeu0p\\tomiss', 'desktop-llpeu0p\\tomisss',
'desktop-llpeu0p\\tomissss', 'desktop-llpeu0p\\tomisssss']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print('socket initiated.')
confirmed = 'CONFIRMED'
deny = 'denied'
#(so i dont forget) to get recv in async do: var = (await reader.read(4096)).decode('utf-8') if -1 then it will read all
#(so i dont forget) to write sendall do: writer.write(var.encode('utf-8')) should be used with await writer.drain()
async def handle_client(reader, writer):
idrecv = (await reader.read(255)).decode('utf-8')
if idrecv in list_of_auths:
writer.write(confirmed.encode('utf-8'))
else:
writer.write(deny.encode('utf-8'))
writer.close()
request = None
while request != 'quit':
print("second checkpoint")
writer.close()
async def run_server():
print("first checkpoint")
server = await asyncio.start_server(handle_client, host, port)
async with server:
await server.serve_forever()
asyncio.run(run_server())
This code allows multiple clients to connect at once; However, it only lets me communicate with the last one that connected.

I would suggest to implement it like so:
class SocketHandler(asyncio.Protocol):
def __init__(self):
asyncio.Protocol.__init__(self)
self.transport = None
self.peername = None
# your other code
def connection_made(self, transport):
""" incoming connection """
global ALL_CONNECTIONS
self.transport = transport
self.peername = transport.get_extra_info('peername')
ALL_CONNECTIONS.append(self)
# your other code
def connection_lost(self, exception):
self.close()
# your other code
def data_received(self, data):
# your code handling incoming data
def close(self):
try:
self.transport.close()
except AttributeError:
pass
# global list to store all connections
ALL_CONNECTIONS = []
def send_to_all(message):
""" sending a message to all connected clients """
global ALL_CONNECTIONS
for sh in ALL_CONNECTIONS:
# here you can also check sh.peername to know which client it is
if sh.transport is not None:
sh.transport.write(message)
port = 5060
loop = asyncio.get_event_loop()
coro = loop.create_server(SocketHandler, '', port)
server = loop.run_until_complete(coro)
loop.run_forever()
This way, each connection to the server is represented by an instance of SocketHandler. Whenever you process some data inside this instance, you know which client connection it is.

Python Asyncio & Websocket how to avoid global variable?

I have a python socket server using asyncio and websockets. When the websocket is active 100+ devices will connect and hold their connection waiting for commands/messages.
There are two threads the first thread accepts connections and adds their details to a global variable then waits for messages from the device:
async def thread1(websocket, path):
client_address = await websocket.recv()
CONNECTIONS[client_address] = websocket
async for message in websocket:
... do something with message
start_server = websockets.serve(thread1, host, port)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.ensure_future(thread2())
asyncio.get_event_loop().run_forever()
The second thread processes some user data and once it needs to send a command it accesses a global variable to get the websocket info:
thread2()
...some data processing
soc = CONNECTIONS[ipaddress]
await soc.send("some message")
My question: What's the best way to allow another thread to send messages?
I can keep the global variable safe using thread locking and a function made only to process that data, however global variables aren't ideal. I cannot send information between threads since thread1 is stuck waiting to receive messages.

The first thing I would like to say is the incorrect use of the term thread. You use asyncio and here the concept is used - coroutine (coroutine is wrapped into a asyncio task). How it differs from threads can be found, for example, here.
The websockets server spawns a new task for each incoming connection (there are the same number of connections and spawned tasks). I don't see anything wrong with the global object, at least in a small script. However, below I gave an example where I placed this in a separate class.
Also, in this case, special synchronization between coroutines is not required, since they are implemented through cooperative multitasking (in fact, all are executed in one thread, transferring control at certain points.)
Here is a simple example in which the server stores a dictionary of incoming connections and starts a task that every 2 seconds, notifies all clients and sends them the current time. The server also prints confirmation from clients to the console.
# ws_server.py
import asyncio
import websockets
import datetime
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.connections = {}
self.is_active = False
self.server = None
async def start(self):
self.is_active = True
self.server = await websockets.serve(self.handler, self.host, self.port)
asyncio.create_task(self.periodic_notifier())
async def stop(self):
self.is_active = False
self.server.close()
await self.wait_closed()
async def wait_closed(self):
await self.server.wait_closed()
async def handler(self, websocket, path):
self.connections[websocket.remote_address] = websocket
try:
async for message in websocket:
print(message)
except ConnectionClosedError as e:
pass
del self.connections[websocket.remote_address]
print(f"Connection {websocket.remote_address} is closed")
async def periodic_notifier(self):
while self.is_active:
await asyncio.gather(
*[ws.send(f"Hello time {datetime.datetime.now()}") for ws in self.connections.values()],
return_exceptions=True)
await asyncio.sleep(2)
async def main():
server = Server("localhost", 8080)
await server.start()
await server.wait_closed()
asyncio.run(main())
# ws_client.py
import asyncio
import websockets
async def client():
uri = "ws://localhost:8080"
async with websockets.connect(uri) as websocket:
async for message in websocket:
print(message)
await websocket.send(f"ACK {message}")
asyncio.run(client())

Issues listening incoming messages in websocket client on Python 3.6

I'm trying to build a websocket client on Python using websockets package from here: Websockets 4.0 API
I'm using this way instead of example code because I want to create a websocket client class object, and use it as gateway.
I'm having issues with my listener method (receiveMessage) on client side, which raises a ConnectionClose exception at execution. I think maybe there is any problem with the loop.
This is the simple webSocket client I've tried to build:
import websockets
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
# Enable listener
await self.receiveMessage()
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self):
'''
Receiving all server messages and handling them
'''
while True:
message = await self.connection.recv()
print('Received message from server: ' + str(message))
And this is the main:
'''
Main file
'''
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
loop.run_until_complete(client.connect())
loop.run_forever()
loop.close()
To test incoming messages listener, server sends two messages to client when it stablishes the connection.
Client connects correctly to server, and sends the greeting. However, when client receives both messages, it raises a ConnectionClosed exception with code 1000 (no reason).
If I remove the loop in the receiveMessage client method, client does not raise any exception, but it only receives one message, so I suppose I need a loop to keep listener alive, but I don't know exactly where or how.
Any solution?
Thanks in advance.
EDIT: I realize that client closes connection (and breaks loop) when it receives all pending messages from server. However, I want client keeps alive listening future messages.
In addition, I've tried to add another function whose task is to send a 'heartbeat' to server, but client closes connection anyway.

Finally, based on this post answer, I modified my client and main files this way:
WebSocket Client:
import websockets
import asyncio
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
return self.connection
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self, connection):
'''
Receiving all server messages and handling them
'''
while True:
try:
message = await connection.recv()
print('Received message from server: ' + str(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
async def heartbeat(self, connection):
'''
Sending heartbeat to server every 5 seconds
Ping - pong messages to verify connection is alive
'''
while True:
try:
await connection.send('ping')
await asyncio.sleep(5)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
Main:
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
# Start connection and get client connection protocol
connection = loop.run_until_complete(client.connect())
# Start listener and heartbeat
tasks = [
asyncio.ensure_future(client.heartbeat(connection)),
asyncio.ensure_future(client.receiveMessage(connection)),
]
loop.run_until_complete(asyncio.wait(tasks))
Now, client keeps alive listening all messages from server and sending 'ping' messages every 5 seconds to server.

Asyncio: start and stop a TCP server while loop is running

I'm creating a program to act as a proxy between a browser and a piece of hardware. The browser initiates the connection as a websocket. The program then starts a TCP server using the port specified in the initial request. Once the external device makes a connection to the TCP server, it can send data to the browser. The code example works up to this point, but I also need the program to shutdown the TCP server and all connections if any of the connections are closed. Another websocket request should start the process again.
I don't seem to be stopping the server/websocket correctly since the system will not allow anymore websocket connections from the browser (The connection is refused). However, all other handlers will still respond on the http server.
import aiohttp
from aiohttp import web
import asyncio
def TCPServerFactory(conns):
def _f(*args,**kwargs):
return TCPServer(conns,*args,**kwargs)
return _f
async def sendWS(conn,data):
await conn.send_str(data)
class TCPServer(asyncio.Protocol):
def __init__(self,conns,*args,**kwargs):
self.conns=conns
super().__init__(*args,**kwargs)
def connection_made(self,transport):
self.transport=transport
self.conns['tcp']=transport
def data_received(self,data):
asyncio.ensure_future(sendWS(self.conns['ws'],data.decode()))
def connection_lost(self,exc):
self.conns['ws'].close()
async def sendTCP(conn,data):
await conn.write(data)
async def wsHandler(request):
ws=web.WebSocketResponse()
await ws.prepare(request)
#Setup TCP server
loop=asyncio.get_event_loop()
conns={'ws':ws,'tcp':None}
try:
serv=await loop.create_server(TCPServerFactory(conns),'127.0.0.1',8000)
except:
await ws.close()
return ws
conns['tcpServ']=serv
async for msg in ws:
if msg.type==aiohttp.WSMsgType.TEXT:
if msg.data:
try:
conns['tcp'].write(bytes(msg.data,'utf-8'))
except:
break
elif msg.type==aiohttp.WSMsgType.CLOSING:
break
else:
break
try:
conns['tcp'].close()
except:
pass
serv.close()
await serv.wait_closed()
await ws.close()
return ws
async def handleTCP(request):
return web.Response(text='TCP worked',headers={'Access-Control-Allow-Origin':'*'})
async def handleHome(request):
return web.FileResponse(r'c:\workspace\lserver\lserver\test.html')
class HttpServer:
def start(self):
asyncio.set_event_loop(asyncio.ProactorEventLoop())
self.loop=asyncio.get_event_loop()
self.app=web.Application()
self.app.router.add_get('/tcp',wsHandler)
self.app.router.add_get('/',handleHome)
web.run_app(self.app,host='0.0.0.0',port=8080)
self.loop.close()
http=HttpServer()
http.start()