Develop Reference

Python loop.sock_accept only accepts one connection

My process is using asyncio and socket to communicate with other processes. It handles one client process connection perfectly, but when a second client tries to connect, it waits forever at the client's connect_ex method and the server's sock_accept method.
The flow is this: Process #1 spawns a worker process which creates a socket server. Process #1 connects and communicates with worker process. When/if process #1 dies, process #2 tries to connect with worker process. Process #2 can't connect.
# process #1
class Communicator:
def __init__(self, ..., port):
...
self.port = port
self.loop = asyncio.get_event_loop()
async def connect(self):
self.psocket = socket.socket(socket.AF_INET, (socket.SOCK_STREAM | socket.SOCK_NONBLOCK))
ex = 1
while ex:
ex = self.psocket.connect_ex(('localhost', self.port))
self.psocket.setblocking(False)
self.listen_task = self.loop.create_task(self.listen())
self.emit_task = self.loop.create_task(self.emit())
print('Connected on port', self.port)
async def listen(self):
...
async def emit(self):
...
# worker process
class Communicator(threading.Thread):
def __init__(self, ..., port):
...
self.port = port
super().__init__()
def set_event_loop(self):
try:
self.loop = asyncio.get_event_loop()
except RuntimeError:
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
def run(self):
self.set_event_loop()
self.psocket = socket.socket(socket.AF_INET, (socket.SOCK_STREAM | socket.SOCK_NONBLOCK))
self.psocket.bind(('localhost', self.port))
self.psocket.listen()
self.psocket.setblocking(False)
self.accept_task = self.loop.create_task(self.accept())
pending = asyncio.all_tasks(loop=self.loop)
self.loop.run_until_complete(asyncio.gather(*pending))
async def accept(self):
while True:
connection, addr = await self.loop.sock_accept(self.psocket)
self.tasks.append({
'listener': self.loop.create_task(self.listen(connection)),
'emitter': self.loop.create_task(self.emit(connection)),
})
async def listen(self, connection):
...
async def emit(self, connection):
...
I know how to do this with threading, I only want to use asynchronous methods for handling multiple client connections.
Also connect_ex blocks when the second process tries to connects. The first process runs through the connect_ex loop many times before connecting.
What's causing connect_ex to block and sock_accept to wait forever?

Interrupt python websocket server

I'm currently struggling with something "simple".
I'd like to have a python WebSocket Server, which is capable of closing down by outside events (e.g. a Ctrl+C from the command line).
Here is my code so far:
PORT = 8765
class Server(object):
def __init__(self):
self.online_players = dict()
self.online_players_lock = asyncio.Lock()
self.websocket_server = None
async def add_online_player(self, id, player):
async with self.online_players_lock:
self.online_players[id] = player
async def remove_online_player(self, id):
async with self.online_players_lock:
if id in self.online_players.keys():
del self.online_players[id]
def start(self):
end = False
loop = asyncio.new_event_loop()
thread = threading.Thread(target=listen, args=(loop, self))
thread.start()
while not end:
try:
time.sleep(500)
except KeyboardInterrupt:
end = True
loop.call_soon_threadsafe(stop_listening, loop, server)
async def on_connect(websocket, path, server):
print("New user...")
id = await websocket.recv()
player = WebSocketPlayer(id, websocket, server)
await server.add_online_player(id, player)
# from this point on WebSocketPlayer class handles communication
await player.listen()
def listen(loop, server:Server):
asyncio.set_event_loop(loop)
bound_handler = functools.partial(on_connect, server=server)
start_server_task = websockets.serve(bound_handler, "localhost", PORT, ping_timeout=None, loop=loop)
start_server = loop.run_until_complete(start_server_task)
server.websocket_server = start_server
print("Server running ...")
loop.run_forever()
async def stop_listening(loop, server:Server):
await server.websocket_server.wait_close()
loop.stop()
loop.close()
if __name__ == "__main__":
server = Server()
server.start()
Signal handlers from asyncio like loop.add_signal_handler(signum, callback, *args) are not an option for me, because they only work on Unix.
The error that I currently get is that the stop_listening method was never awaited, which kind of makes sense to me. So I am not that much interested in fixing my code example, but more in general how is it possible to achieve my goal, or how is it usually solved?
Thank you very much in advance

Nevermind, this question is related to this question: Why does the asyncio's event loop suppress the KeyboardInterrupt on Windows? which is actually bug of asyncio on Windows.

Simple Python multi threaded webserver with Asyncio and events called in main function

I have a simple Python program that I want to do three things:
Serve an HTTP document
Serve Websockets
Interact with the Websocket data
I am trying to use / grok asyncio. The issue is that I can't figure out how to access data acquired from a function in the main event loop.
For example in my code below I have two threads.
One thread is the HTTP server thread, one thread is the Websocket server thread and there is the main thread.
What I want to do is to print data captured in the websocket receiving thread in the main thread.
The only way I know how to do this is to use Queues to pass data between threads at which point I do not even know what the advantage of using asyncio is.
Similarly, it feels weird to pass the event loop to the serve_websocket function.
Can anyone please explain how to architect this to get data from the Websocket function into the main function?
It seems like / I want a way to do this without using the threading library at all, which seems possible. In an async project I would want to react to websocket events in different function than where they are called.
NOTE: I know there are other libraries for websockets and http serving with asyncio but this is an example to help me understarnd how to structure projects using this paradigm.
Thanks
#!/usr/bin/env python
import json
import socketserver
import threading
import http.server
import asyncio
import time
import websockets
SERVER_ADDRESS = '127.0.0.1'
HTTP_PORT = 8087
WEBSOCKET_PORT = 5678
def serve_http():
http_handler = http.server.SimpleHTTPRequestHandler
with socketserver.TCPServer(("", HTTP_PORT), http_handler) as httpd:
print(f'HTTP server listening on port {HTTP_PORT}')
httpd.serve_forever()
def serve_websocket(server, event_loop):
print(f'Websocket server listening on port {WEBSOCKET_PORT}')
event_loop.run_until_complete(server)
event_loop.run_forever()
async def ws_callback(websocket, path):
while True:
data = await websocket.recv()
# How do I access parsed_data in the main function below
parsed_data = json.loads(data)
await websocket.send(data)
def main():
event_loop = asyncio.get_event_loop()
ws_server = websockets.serve(ws_callback, SERVER_ADDRESS, WEBSOCKET_PORT)
threading.Thread(target=serve_http, daemon=True).start()
threading.Thread(target=serve_websocket, args=(ws_server, event_loop), daemon=True).start()
try:
while True:
# Keep alive - this is where I want to access the data from ws_callback
# i.e.
# print(data.values)
time.sleep(.01)
except KeyboardInterrupt:
print('Exit called')
if __name__ == '__main__':
main()

I believe that you should not mix asyncio and multithreading without special need. And in your case, use only asyncio tools.
In this case, you have no problem sharing data between coroutines, because they all run on the same thread using cooperative multitasking.
Your code can be rewtitten as:
#!/usr/bin/env python
import json
import socketserver
import threading
import http.server
import asyncio
import time
import websockets
SERVER_ADDRESS = '127.0.0.1'
HTTP_PORT = 8087
WEBSOCKET_PORT = 5678
parsed_data = {}
async def handle_http(reader, writer):
data = await reader.read(100)
message = data.decode()
writer.write(data)
await writer.drain()
writer.close()
async def ws_callback(websocket, path):
global parsed_data
while True:
data = await websocket.recv()
# How do I access parsed_data in the main function below
parsed_data = json.loads(data)
await websocket.send(data)
async def main():
ws_server = await websockets.serve(ws_callback, SERVER_ADDRESS, WEBSOCKET_PORT)
print(f'Websocket server listening on port {WEBSOCKET_PORT}')
http_server = await asyncio.start_server(
handle_http, SERVER_ADDRESS, HTTP_PORT)
print(f'HTTP server listening on port {HTTP_PORT}')
try:
while True:
if parsed_data:
print(parsed_data.values())
await asyncio.sleep(0.1)
except KeyboardInterrupt:
print('Exit called')
if __name__ == '__main__':
asyncio.run(main())

Python UDP and Websockets together

I'm working on a application. Where am using python websockets. Now I need UDP and WS asynchronously running and listening on different ports.
I'm unable to do it because WS recv() waits indefinitely untill a message is received. Message will be received and pushed into queue. I need UDP to receive and push to same queue. This below class implements only websockets. I need another class with UDP and both class instance run asynchronously.
import websockets
import json
from sinric.command.mainqueue import queue
from sinric.callback_handler.cbhandler
import CallBackHandler
from time import sleep
class SinricProSocket:
def __init__(self, apiKey, deviceId, callbacks):
self.apiKey = apiKey
self.deviceIds = deviceId
self.connection = None
self.callbacks = callbacks
self.callbackHandler = CallBackHandler(self.callbacks)
pass
async def connect(self): # Producer
self.connection = await websockets.client.connect('ws://2.5.2.2:301',
extra_headers={'Authorization': self.apiKey,
'deviceids': self.deviceIds},
ping_interval=30000, ping_timeout=10000)
if self.connection.open:
print('Client Connected')
return self.connection
async def sendMessage(self, message):
await self.connection.send(message)
async def receiveMessage(self, connection):
try:
message = await connection.recv()
queue.put(json.loads(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
async def handle(self):
# sleep(6)
while queue.qsize() > 0:
await self.callbackHandler.handleCallBacks(queue.get(), self.connection)
return

thanks for your time in the comments. I solved this issue by running instances of WS and UDP in 2 different daemon threads.

A good way to solve this issue would be to use threads. You could accept a message and put it into a queue, then handle the queue on a different thread.

Python/Socket: How to send multi messages without waiting the respond?

I want to send multi messages using socket module without waiting the respond from the client or server. However the codes that are below can not do this. What are your suggestions in order to do that? Thanks in advance.
Here are the codes:
server.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 12345))
s.listen(1)
print(s)
c, addr = s.accept()
print('{} connected.'.format(addr))
while True:
respond = input("Server: ").encode("utf-8")
if respond == b"q":
exit()
else:
c.sendall(bytes(respond))
data = str(c.recv(1024))[1:]
if data:
print("Client: {}".format(data))
client.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("", 12345))
while True:
respond = input("Client: ").encode("utf-8")
if respond == b"q":
exit()
else:
s.sendall(bytes(respond))
data = str(s.recv(1024))[1:]
if data:
print("Server: {}".format(data))

Ok, so for what you want to achieve, you need to change the approach completely and use asyncio equivalents of socket methods as well as replacement for bare standard input handling.
The following code works on Python >= 3.5 and requires aioconsole Python package which can be installed with pip install aioconsole.
server.py
import asyncio
import aioconsole
class StreamWrapper(object):
"""This class is used to make socket stream created by server available for send_messgaes() function"""
def __init__(self):
self.reader = None
self.writer = None
async def send_messages(stream_wrapper, stdin):
# Wait asynchronously until server part initializes socket stream
while stream_wrapper.writer is None:
await asyncio.sleep(0.1)
writer = stream_wrapper.writer
# Asynchronusly read from standard input
async for message in stdin:
if message.decode().strip() == "q":
writer.close()
exit()
else:
# Send message through the socket
writer.write(message)
def receive_messages_wrapper(stream_wrapper, stdout):
"""Wrapper function which adds stream_wrapper and stdout to the scope of receive_messages()"""
async def receive_messages(reader, writer):
# Copy socket stream reference to stream wrapper
stream_wrapper.reader = reader
stream_wrapper.writer = writer
# Asynchronusly read messages from the socket
async for message in reader:
stdout.write('\nClient: {}'.format(message.decode()))
stdout.write("Server: ")
# Wrapper returns receive_messages function with enhanced scope - receive_messages() can "see" stream_wrapper and stdout
return receive_messages
async def run_server(loop):
"""Initialize stdin and stdout asynchronous streams and start the server"""
stdin, stdout = await aioconsole.get_standard_streams()
stream_wrapper = StreamWrapper()
# Asynchronously execute send_messages and start_server()
await asyncio.gather(
send_messages(stream_wrapper, stdin),
asyncio.start_server(receive_messages_wrapper(stream_wrapper, stdout), '127.0.0.1', 8888, loop=loop)
)
# Run the server on the event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(run_server(loop))
loop.close()
client.py
import asyncio
import aioconsole
async def send_messages(writer, stdin):
# Asynchronusly read from standard input
async for message in stdin:
if message.decode().strip() == "q":
writer.close()
exit()
else:
# Send message through the socket
writer.write(message)
async def receive_messages(reader, stdout):
# Asynchronusly read messages from the socket
async for message in reader:
stdout.write('\nServer: {}'.format(message.decode()))
stdout.write("Client: ")
async def run_client(loop):
"""Initialize stdin and stdout asynchronous streams and open the client connection, then start exchanging messages"""
stdin, stdout = await aioconsole.get_standard_streams()
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
stdout.write("Client: ")
# Asynchronously execute send_messages and receive_messages()
await asyncio.gather(
send_messages(writer, stdin),
receive_messages(reader, stdout)
)
# Run the client on the event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(run_client(loop))
loop.close()
This might seem complicated if you never used asyncio before but it does exactly what you want. You can spam multiple messages from any end (client or server) and the other end will receive it and print, while waiting for user input. I've provided comments but if you want to fully understand it, you should get familiar with asyncio documentation.
The other possible approaches involve using threads or multiprocessing. I wouldn't say they are easier than asyncio.