I trying to use Python's logging and asyncio libraries to send logs via WebSockets. For context Python's logging runs logging handlers in different threads.
In the code bellow the event loop and all WebSocket connections get passed to the log handler so it can broadcast the messages to everyone.
class ESSocketLoggerHandlerWS(logging.Handler, object):
def __init__(self, name, other_attr=None, **kwargs):
logging.Handler.__init__(self)
self.connections = []
self.loop = kwargs.get("loop")
def addConnection(self, connection):
self.connections.append(connection)
async def send(self, socketcon, msg):
print("Sending...")
await socketcon.send(msg)
def emit(self, record):
msg = (self.format(record) + "\n").encode("utf-8")
# Remove old clients no longer connected
self.connections[:] = [x for x in self.connections if not x.closed]
# Send logs to the available sockets
for socketcon in self.connections:
print("Sending message to all WS clients")
self.loop.call_soon_threadsafe(self.send(socketcon, msg))
eventLoop = asyncio.new_event_loop()
asyncio.set_event_loop(eventLoop);
esSocketHandlerWS = ESSocketLoggerHandlerWS(name='mainlogger', loop=eventLoop)
log.addHandler(esSocketHandlerWS)
async def handle_client_ws(websocket):
log.info('New websocket connection')
esSocketHandlerWS.addConnection(websocket)
async for message in websocket:
# do suff with incoming messages...
async def main():
serverWs = await websockets.serve(handle_client_ws, "127.0.0.1", 8765)
async with server, serverWs:
await asyncio.gather(
...,
serverWs.serve_forever()
)
asyncio.run(main())
Unfortunately it doesn't work nor gives me any errors. Clients can connect and send messages but no logs get to them. I get Sending message to all WS clients in stdout, however print("Sending...") is never called nor await socketcon.send(msg).
What's the proper way of doing this? Thank you.
Related
I have a python socket server using asyncio and websockets. When the websocket is active 100+ devices will connect and hold their connection waiting for commands/messages.
There are two threads the first thread accepts connections and adds their details to a global variable then waits for messages from the device:
async def thread1(websocket, path):
client_address = await websocket.recv()
CONNECTIONS[client_address] = websocket
async for message in websocket:
... do something with message
start_server = websockets.serve(thread1, host, port)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.ensure_future(thread2())
asyncio.get_event_loop().run_forever()
The second thread processes some user data and once it needs to send a command it accesses a global variable to get the websocket info:
thread2()
...some data processing
soc = CONNECTIONS[ipaddress]
await soc.send("some message")
My question: What's the best way to allow another thread to send messages?
I can keep the global variable safe using thread locking and a function made only to process that data, however global variables aren't ideal. I cannot send information between threads since thread1 is stuck waiting to receive messages.
The first thing I would like to say is the incorrect use of the term thread. You use asyncio and here the concept is used - coroutine (coroutine is wrapped into a asyncio task). How it differs from threads can be found, for example, here.
The websockets server spawns a new task for each incoming connection (there are the same number of connections and spawned tasks). I don't see anything wrong with the global object, at least in a small script. However, below I gave an example where I placed this in a separate class.
Also, in this case, special synchronization between coroutines is not required, since they are implemented through cooperative multitasking (in fact, all are executed in one thread, transferring control at certain points.)
Here is a simple example in which the server stores a dictionary of incoming connections and starts a task that every 2 seconds, notifies all clients and sends them the current time. The server also prints confirmation from clients to the console.
# ws_server.py
import asyncio
import websockets
import datetime
class Server:
def __init__(self, host, port):
self.host = host
self.port = port
self.connections = {}
self.is_active = False
self.server = None
async def start(self):
self.is_active = True
self.server = await websockets.serve(self.handler, self.host, self.port)
asyncio.create_task(self.periodic_notifier())
async def stop(self):
self.is_active = False
self.server.close()
await self.wait_closed()
async def wait_closed(self):
await self.server.wait_closed()
async def handler(self, websocket, path):
self.connections[websocket.remote_address] = websocket
try:
async for message in websocket:
print(message)
except ConnectionClosedError as e:
pass
del self.connections[websocket.remote_address]
print(f"Connection {websocket.remote_address} is closed")
async def periodic_notifier(self):
while self.is_active:
await asyncio.gather(
*[ws.send(f"Hello time {datetime.datetime.now()}") for ws in self.connections.values()],
return_exceptions=True)
await asyncio.sleep(2)
async def main():
server = Server("localhost", 8080)
await server.start()
await server.wait_closed()
asyncio.run(main())
# ws_client.py
import asyncio
import websockets
async def client():
uri = "ws://localhost:8080"
async with websockets.connect(uri) as websocket:
async for message in websocket:
print(message)
await websocket.send(f"ACK {message}")
asyncio.run(client())
I have a publisher/subscriber architecture running on my websocket server, where the publisher runs in one thread, and the websocket server in another. I connect to the server from the publisher over localhost, and the server distributes the published messages to any other connected clients on the /sub path. However, since the publisher thread not always has new data to publish, it has a tendency to disconnect after a timeout of 50 sec. To fix this, I implemented a heartbeat ping function:
async def ping(websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
This keeps the publisher from disconnecting. However, when I'm trying to run this concurrently with the coroutine that sends the actual data, I cannot get both ping() and send_data() to run in parallel. I've tried just awaiting both functions as well as asyncio.gather() (which according to documentation is supposed to run tasks concurrently) as well as flipping the order, but it seems like in all cases only the first function call is ran.
My thread class for reference:
class Publisher(threading.Thread):
"""
Thread acting as the websocket publisher
Pulls data from the data merger queue and publishes onto the websocket server
"""
def __init__(self, loop, q, addr, port):
threading.Thread.__init__(self)
self.loop = loop
self.queue = q
self.id = threading.get_ident()
self.addr = addr
self.port = port
self.name = 'publisher'
print("Publisher thread started (ID:%s)" % self.id)
def run(self):
self.loop.run_until_complete(self.publish())
asyncio.get_event_loop().run_forever()
async def ping(self, websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
async def send_data(self, websocket):
while True:
try:
msg = json.dumps(self.queue.get()) # Get the data from the queue
print(msg)
await asyncio.sleep(0.1)
if not msg:
print("No message")
break
await websocket.send(msg)
except websockets.exceptions.ConnectionClosedError:
print("Connection closed")
break
async def publish(self):
uri = 'ws://' + str(self.addr) + ':' + str(self.port) + '/pub'
async with websockets.connect(uri) as websocket:
await asyncio.gather(
self.ping(websocket),
self.send_data(websocket)
)
I'm working on a application. Where am using python websockets. Now I need UDP and WS asynchronously running and listening on different ports.
I'm unable to do it because WS recv() waits indefinitely untill a message is received. Message will be received and pushed into queue. I need UDP to receive and push to same queue. This below class implements only websockets. I need another class with UDP and both class instance run asynchronously.
import websockets
import json
from sinric.command.mainqueue import queue
from sinric.callback_handler.cbhandler
import CallBackHandler
from time import sleep
class SinricProSocket:
def __init__(self, apiKey, deviceId, callbacks):
self.apiKey = apiKey
self.deviceIds = deviceId
self.connection = None
self.callbacks = callbacks
self.callbackHandler = CallBackHandler(self.callbacks)
pass
async def connect(self): # Producer
self.connection = await websockets.client.connect('ws://2.5.2.2:301',
extra_headers={'Authorization': self.apiKey,
'deviceids': self.deviceIds},
ping_interval=30000, ping_timeout=10000)
if self.connection.open:
print('Client Connected')
return self.connection
async def sendMessage(self, message):
await self.connection.send(message)
async def receiveMessage(self, connection):
try:
message = await connection.recv()
queue.put(json.loads(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
async def handle(self):
# sleep(6)
while queue.qsize() > 0:
await self.callbackHandler.handleCallBacks(queue.get(), self.connection)
return
thanks for your time in the comments. I solved this issue by running instances of WS and UDP in 2 different daemon threads.
A good way to solve this issue would be to use threads. You could accept a message and put it into a queue, then handle the queue on a different thread.
I'm trying to build a websocket client on Python using websockets package from here: Websockets 4.0 API
I'm using this way instead of example code because I want to create a websocket client class object, and use it as gateway.
I'm having issues with my listener method (receiveMessage) on client side, which raises a ConnectionClose exception at execution. I think maybe there is any problem with the loop.
This is the simple webSocket client I've tried to build:
import websockets
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
# Enable listener
await self.receiveMessage()
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self):
'''
Receiving all server messages and handling them
'''
while True:
message = await self.connection.recv()
print('Received message from server: ' + str(message))
And this is the main:
'''
Main file
'''
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
loop.run_until_complete(client.connect())
loop.run_forever()
loop.close()
To test incoming messages listener, server sends two messages to client when it stablishes the connection.
Client connects correctly to server, and sends the greeting. However, when client receives both messages, it raises a ConnectionClosed exception with code 1000 (no reason).
If I remove the loop in the receiveMessage client method, client does not raise any exception, but it only receives one message, so I suppose I need a loop to keep listener alive, but I don't know exactly where or how.
Any solution?
Thanks in advance.
EDIT: I realize that client closes connection (and breaks loop) when it receives all pending messages from server. However, I want client keeps alive listening future messages.
In addition, I've tried to add another function whose task is to send a 'heartbeat' to server, but client closes connection anyway.
Finally, based on this post answer, I modified my client and main files this way:
WebSocket Client:
import websockets
import asyncio
class WebSocketClient():
def __init__(self):
pass
async def connect(self):
'''
Connecting to webSocket server
websockets.client.connect returns a WebSocketClientProtocol, which is used to send and receive messages
'''
self.connection = await websockets.client.connect('ws://127.0.0.1:8765')
if self.connection.open:
print('Connection stablished. Client correcly connected')
# Send greeting
await self.sendMessage('Hey server, this is webSocket client')
return self.connection
async def sendMessage(self, message):
'''
Sending message to webSocket server
'''
await self.connection.send(message)
async def receiveMessage(self, connection):
'''
Receiving all server messages and handling them
'''
while True:
try:
message = await connection.recv()
print('Received message from server: ' + str(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
async def heartbeat(self, connection):
'''
Sending heartbeat to server every 5 seconds
Ping - pong messages to verify connection is alive
'''
while True:
try:
await connection.send('ping')
await asyncio.sleep(5)
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
break
Main:
import asyncio
from webSocketClient import WebSocketClient
if __name__ == '__main__':
# Creating client object
client = WebSocketClient()
loop = asyncio.get_event_loop()
# Start connection and get client connection protocol
connection = loop.run_until_complete(client.connect())
# Start listener and heartbeat
tasks = [
asyncio.ensure_future(client.heartbeat(connection)),
asyncio.ensure_future(client.receiveMessage(connection)),
]
loop.run_until_complete(asyncio.wait(tasks))
Now, client keeps alive listening all messages from server and sending 'ping' messages every 5 seconds to server.
I am trying to add two coroutines to asyncio loop and getting an error:
RuntimeError: This event loop is already running
My objective is to communicate to a server (that I have no control of). This server expects an initial connection from the client. The server then provided a port to the client on this connection. The client has to use this port to create a second connection. This second connection is used by the server to send unsolicited messages to the client. The first connection remains up throughout for other two-way communications.
To recreate this scenario, I have some code that reproduces the error:
class Connection():
def __init__(self, ip, port, ioloop):
self.ip = ip
self.port = port
self.ioloop = ioloop
self.reader, self.writer = None, None
self.protocol = None
self.fileno = None
async def __aenter__(self):
# Applicable when doing 'with Connection(...'
log.info("Entering and Creating Connection")
self.reader, self.writer = (
await asyncio.open_connection(self.ip, self.port, loop=self.ioloop)
)
self.protocol = self.writer.transport.get_protocol()
self.fileno = self.writer.transport.get_extra_info('socket').fileno()
log.info(f"Created connection {self}")
return self
async def __aexit__(self, *args):
# Applicable when doing 'with Connection(...'
log.info(f"Exiting and Destroying Connection {self}")
if self.writer:
self.writer.close()
def __await__(self):
# Applicable when doing 'await Connection(...'
return self.__aenter__().__await__()
def __repr__(self):
return f"[Connection {self.ip}:{self.port}, {self.protocol}, fd={self.fileno}]"
async def send_recv_message(self, message):
log.debug(f"send: '{message}'")
self.writer.write(message.encode())
await self.writer.drain()
log.debug("awaiting data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv: '{data}'")
return data
class ServerConnection(Connection):
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
self.ioloop.run_until_complete(event_connection.recv_message())
class EventConnection(Connection):
async def recv_message(self):
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv only: '{data}'")
return data
async def main(loop):
client1 = await ServerConnection('127.0.0.1', 8888, loop)
await client1.setup_connection()
await client1.send_recv_message("Hello1")
await client1.send_recv_message("Hello2")
await asyncio.sleep(5)
if __name__ == '__main__':
#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
The error occurs in ServerConnection.setup_connection() method where run_until_complete is being called.
I am probably doing something wrong due to lack of understanding asyncio. Basically, how do I setup a secondary connection which will get event notifications (unsolicited) while setting up the first connection?
Thanks.
Followup
Since the code is very similar (a few changes to add more functionality to it), I hope it's not bad etiquette to followup to the original post as the resulting error is still the same.
The new issue is that when it receives the unsolicited message (which is received by EventConnection), the recv_message calls process_data method. I would like to make process_data be a future so that recv_message completes (ioloop should stop). The ensure_future would then pick it up and continue running again to use ServerConnection to do a request/response to the server. Before it does that though, it has to go to some user code (represented by external_command() and from whom I would prefer to hide the async stuff). This would make it synchronous again. Hence, once they've done what they need to, they should call execute_command on ServerConnection, which then kicks off the loop again.
The problem is, my expectation for using ensure_future didn't pan out as it seems the loop didn't stop from running. Hence, when the code execution reaches execute_command which does the run_until_complete, an exception with the error "This event loop is already running" occurs.
I have two questions:
How can I make it so that the ioloop can stop after process_data is
placed into ensure_future, and subsequently be able to run it again
in execute_command?
Once recv_message has received something, how can we make it so that
it can receive more unsolicited data? Is it enough/safe to just use
ensure_future to call itself again?
Here's the example code that simulates this issue.
client1 = None
class ServerConnection(Connection):
connection_type = 'Server Connection'
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
asyncio.ensure_future(event_connection.recv_message())
async def _execute_command(self, data):
return await self.send_recv_message(data)
def execute_command(self, data):
response_str = self.ioloop.run_until_complete(self._execute_command(data))
print(f"exec cmd response_str: {response_str}")
def external_command(self, data):
self.execute_command(data)
class EventConnection(Connection):
connection_type = 'Event Connection'
async def recv_message(self):
global client1
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv-only: '{data}'")
asyncio.ensure_future(self.process_data(data))
asyncio.ensure_future(self.recv_message())
async def process_data(self, data):
global client1
await client1.external_command(data)
async def main(ioloop):
global client1
client1 = await ServerConnection('127.0.0.1', 8888, ioloop)
await client1.setup_connection()
print(f"after connection setup loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello1")
print(f"after Hello1 loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello2")
print(f"after Hello2 loop running is {ioloop.is_running()}")
while True:
print(f"inside while loop running is {ioloop.is_running()}")
t = 10
print(f"asyncio sleep {t} sec")
await asyncio.sleep(t)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
Try replacing:
self.ioloop.run_until_complete
With
await