Develop Reference

how do i switch between connected clients with asyncio sockets

I am trying to make a socket server that's able to have multiple clients connected using the asyncio sockets and is able to easily switch between which client it communicates while still having all the clients connected. I thought there would be some type of FD of the clients like there is in sockets, but I looked through the docs and did not find anything, or I missed it.
Here is my server code:
import socket
import asyncio
host = "localhost"
port = 9998
list_of_auths = ['desktop-llpeu0p\\tomiss', 'desktop-llpeu0p\\tomisss',
'desktop-llpeu0p\\tomissss', 'desktop-llpeu0p\\tomisssss']
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print('socket initiated.')
confirmed = 'CONFIRMED'
deny = 'denied'
#(so i dont forget) to get recv in async do: var = (await reader.read(4096)).decode('utf-8') if -1 then it will read all
#(so i dont forget) to write sendall do: writer.write(var.encode('utf-8')) should be used with await writer.drain()
async def handle_client(reader, writer):
idrecv = (await reader.read(255)).decode('utf-8')
if idrecv in list_of_auths:
writer.write(confirmed.encode('utf-8'))
else:
writer.write(deny.encode('utf-8'))
writer.close()
request = None
while request != 'quit':
print("second checkpoint")
writer.close()
async def run_server():
print("first checkpoint")
server = await asyncio.start_server(handle_client, host, port)
async with server:
await server.serve_forever()
asyncio.run(run_server())
This code allows multiple clients to connect at once; However, it only lets me communicate with the last one that connected.

I would suggest to implement it like so:
class SocketHandler(asyncio.Protocol):
def __init__(self):
asyncio.Protocol.__init__(self)
self.transport = None
self.peername = None
# your other code
def connection_made(self, transport):
""" incoming connection """
global ALL_CONNECTIONS
self.transport = transport
self.peername = transport.get_extra_info('peername')
ALL_CONNECTIONS.append(self)
# your other code
def connection_lost(self, exception):
self.close()
# your other code
def data_received(self, data):
# your code handling incoming data
def close(self):
try:
self.transport.close()
except AttributeError:
pass
# global list to store all connections
ALL_CONNECTIONS = []
def send_to_all(message):
""" sending a message to all connected clients """
global ALL_CONNECTIONS
for sh in ALL_CONNECTIONS:
# here you can also check sh.peername to know which client it is
if sh.transport is not None:
sh.transport.write(message)
port = 5060
loop = asyncio.get_event_loop()
coro = loop.create_server(SocketHandler, '', port)
server = loop.run_until_complete(coro)
loop.run_forever()
This way, each connection to the server is represented by an instance of SocketHandler. Whenever you process some data inside this instance, you know which client connection it is.

How can i send socket asynchronously? [duplicate]

I would like to re-implement my code using asyncio coroutines instead of multi-threading.
server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
try:
while True:
client, _ = server.accept()
threading.Thread(target=handle_client, args=(client,)).start()
except KeyboardInterrupt:
server.close()
client.py
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(('localhost', 15555))
request = None
try:
while request != 'quit':
request = input('>> ')
if request:
server.send(request.encode('utf8'))
response = server.recv(255).decode('utf8')
print(response)
except KeyboardInterrupt:
server.close()
I know there are some appropriate asynchronous network librairies to do that. But I just want to only use asyncio core library on this case in order to have a better understanding of it.
It would have been so nice to only add async keyword before handle client definition... Here a piece of code which seems to work, but I'm still confused about the implementation.
asyncio_server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
def run_server(server):
client, _ = server.accept()
handle_client(client)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
loop = asyncio.get_event_loop()
asyncio.async(run_server(server))
try:
loop.run_forever()
except KeyboardInterrupt:
server.close()
How adapt this in the best way and using async await keywords.

The closest literal translation of the threading code would create the socket as before, make it non-blocking, and use asyncio low-level socket operations to implement the server. Here is an example, sticking to the more relevant server part (the client is single-threaded and likely fine as-is):
import asyncio, socket
async def handle_client(client):
loop = asyncio.get_event_loop()
request = None
while request != 'quit':
request = (await loop.sock_recv(client, 255)).decode('utf8')
response = str(eval(request)) + '\n'
await loop.sock_sendall(client, response.encode('utf8'))
client.close()
async def run_server():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
server.setblocking(False)
loop = asyncio.get_event_loop()
while True:
client, _ = await loop.sock_accept(server)
loop.create_task(handle_client(client))
asyncio.run(run_server())
The above works, but is not the intended way to use asyncio. It is very low-level and therefore error-prone, requiring you to remember to set the appropriate flags on the socket. Also, there is no buffering, so something as simple as reading a line from the client becomes a tiresome chore. This API level is really only intended for implementors of alternative event loops, which would provide their implementation of sock_recv, sock_sendall, etc.
Asyncio's public API provides two abstraction layers intended for consumption: the older transport/protocol layer modeled after Twisted, and the newer streams layer. In new code, you almost certainly want to use the streams API, i.e. call asyncio.start_server and avoid raw sockets. That significantly reduces the line count:
import asyncio, socket
async def handle_client(reader, writer):
request = None
while request != 'quit':
request = (await reader.read(255)).decode('utf8')
response = str(eval(request)) + '\n'
writer.write(response.encode('utf8'))
await writer.drain()
writer.close()
async def run_server():
server = await asyncio.start_server(handle_client, 'localhost', 15555)
async with server:
await server.serve_forever()
asyncio.run(run_server())

I have read the answers and comments above, trying to figure out how to use the asyncio lib for sockets.
As it often happens with Python, the official documentation along with the examples is the best source of useful information.
I got understanding of Transports and Protocols (low-level API), and Streams (high-level API) from the examples presented in the end of the support article.
For example, TCP Echo Server:
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
and TCP Echo Client:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
Hope it help someone searching for simple explanation of asyncio.

Running coroutines concurrently not working properly in pythong/asyncio

I have a publisher/subscriber architecture running on my websocket server, where the publisher runs in one thread, and the websocket server in another. I connect to the server from the publisher over localhost, and the server distributes the published messages to any other connected clients on the /sub path. However, since the publisher thread not always has new data to publish, it has a tendency to disconnect after a timeout of 50 sec. To fix this, I implemented a heartbeat ping function:
async def ping(websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
This keeps the publisher from disconnecting. However, when I'm trying to run this concurrently with the coroutine that sends the actual data, I cannot get both ping() and send_data() to run in parallel. I've tried just awaiting both functions as well as asyncio.gather() (which according to documentation is supposed to run tasks concurrently) as well as flipping the order, but it seems like in all cases only the first function call is ran.
My thread class for reference:
class Publisher(threading.Thread):
"""
Thread acting as the websocket publisher
Pulls data from the data merger queue and publishes onto the websocket server
"""
def __init__(self, loop, q, addr, port):
threading.Thread.__init__(self)
self.loop = loop
self.queue = q
self.id = threading.get_ident()
self.addr = addr
self.port = port
self.name = 'publisher'
print("Publisher thread started (ID:%s)" % self.id)
def run(self):
self.loop.run_until_complete(self.publish())
asyncio.get_event_loop().run_forever()
async def ping(self, websocket):
while True:
await asyncio.sleep(30)
print("[%s] Pinging server..." % datetime.now())
await websocket.send('ping')
async def send_data(self, websocket):
while True:
try:
msg = json.dumps(self.queue.get()) # Get the data from the queue
print(msg)
await asyncio.sleep(0.1)
if not msg:
print("No message")
break
await websocket.send(msg)
except websockets.exceptions.ConnectionClosedError:
print("Connection closed")
break
async def publish(self):
uri = 'ws://' + str(self.addr) + ':' + str(self.port) + '/pub'
async with websockets.connect(uri) as websocket:
await asyncio.gather(
self.ping(websocket),
self.send_data(websocket)
)

Python simple socket client/server using asyncio

I would like to re-implement my code using asyncio coroutines instead of multi-threading.
server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
try:
while True:
client, _ = server.accept()
threading.Thread(target=handle_client, args=(client,)).start()
except KeyboardInterrupt:
server.close()
client.py
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(('localhost', 15555))
request = None
try:
while request != 'quit':
request = input('>> ')
if request:
server.send(request.encode('utf8'))
response = server.recv(255).decode('utf8')
print(response)
except KeyboardInterrupt:
server.close()
I know there are some appropriate asynchronous network librairies to do that. But I just want to only use asyncio core library on this case in order to have a better understanding of it.
It would have been so nice to only add async keyword before handle client definition... Here a piece of code which seems to work, but I'm still confused about the implementation.
asyncio_server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
def run_server(server):
client, _ = server.accept()
handle_client(client)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
loop = asyncio.get_event_loop()
asyncio.async(run_server(server))
try:
loop.run_forever()
except KeyboardInterrupt:
server.close()
How adapt this in the best way and using async await keywords.

The closest literal translation of the threading code would create the socket as before, make it non-blocking, and use asyncio low-level socket operations to implement the server. Here is an example, sticking to the more relevant server part (the client is single-threaded and likely fine as-is):
import asyncio, socket
async def handle_client(client):
loop = asyncio.get_event_loop()
request = None
while request != 'quit':
request = (await loop.sock_recv(client, 255)).decode('utf8')
response = str(eval(request)) + '\n'
await loop.sock_sendall(client, response.encode('utf8'))
client.close()
async def run_server():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
server.setblocking(False)
loop = asyncio.get_event_loop()
while True:
client, _ = await loop.sock_accept(server)
loop.create_task(handle_client(client))
asyncio.run(run_server())
The above works, but is not the intended way to use asyncio. It is very low-level and therefore error-prone, requiring you to remember to set the appropriate flags on the socket. Also, there is no buffering, so something as simple as reading a line from the client becomes a tiresome chore. This API level is really only intended for implementors of alternative event loops, which would provide their implementation of sock_recv, sock_sendall, etc.
Asyncio's public API provides two abstraction layers intended for consumption: the older transport/protocol layer modeled after Twisted, and the newer streams layer. In new code, you almost certainly want to use the streams API, i.e. call asyncio.start_server and avoid raw sockets. That significantly reduces the line count:
import asyncio, socket
async def handle_client(reader, writer):
request = None
while request != 'quit':
request = (await reader.read(255)).decode('utf8')
response = str(eval(request)) + '\n'
writer.write(response.encode('utf8'))
await writer.drain()
writer.close()
async def run_server():
server = await asyncio.start_server(handle_client, 'localhost', 15555)
async with server:
await server.serve_forever()
asyncio.run(run_server())

I have read the answers and comments above, trying to figure out how to use the asyncio lib for sockets.
As it often happens with Python, the official documentation along with the examples is the best source of useful information.
I got understanding of Transports and Protocols (low-level API), and Streams (high-level API) from the examples presented in the end of the support article.
For example, TCP Echo Server:
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
and TCP Echo Client:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
Hope it help someone searching for simple explanation of asyncio.

Asyncio persisent client protocol class using queue

I'm trying to get my head around the Python 3 asyncio module, in particular using the transport/protocol API. I want to create a publish/subscribe pattern, and use the asyncio.Protocol class to create my client and server.
At the moment I've got the server up and running, and listening for incoming client connections. The client is able to connect to the server, send a message and receive the reply.
I would like to be able to keep the TCP connection alive and maintain a queue that allows me to add messages. I've tried to find a way to do this using the low-level API (Transport/Protocols) but the limited asyncio docs/examples online all seem to go into the high level API - using streams, etc. Would someone be able to point me in the right direction on how to implement this?
Here's the server code:
#!/usr/bin/env python3
import asyncio
import json
class SubscriberServerProtocol(asyncio.Protocol):
""" A Server Protocol listening for subscriber messages """
def connection_made(self, transport):
""" Called when connection is initiated """
self.peername = transport.get_extra_info('peername')
print('connection from {}'.format(self.peername))
self.transport = transport
def data_received(self, data):
""" The protocol expects a json message containing
the following fields:
type: subscribe/unsubscribe
channel: the name of the channel
Upon receiving a valid message the protocol registers
the client with the pubsub hub. When succesfully registered
we return the following json message:
type: subscribe/unsubscribe/unknown
channel: The channel the subscriber registered to
channel_count: the amount of channels registered
"""
# Receive a message and decode the json output
recv_message = json.loads(data.decode())
# Check the message type and subscribe/unsubscribe
# to the channel. If the action was succesful inform
# the client.
if recv_message['type'] == 'subscribe':
print('Client {} subscribed to {}'.format(self.peername,
recv_message['channel']))
send_message = json.dumps({'type': 'subscribe',
'channel': recv_message['channel'],
'channel_count': 10},
separators=(',', ':'))
elif recv_message['type'] == 'unsubscribe':
print('Client {} unsubscribed from {}'
.format(self.peername, recv_message['channel']))
send_message = json.dumps({'type': 'unsubscribe',
'channel': recv_message['channel'],
'channel_count': 9},
separators=(',', ':'))
else:
print('Invalid message type {}'.format(recv_message['type']))
send_message = json.dumps({'type': 'unknown_type'},
separators=(',', ':'))
print('Sending {!r}'.format(send_message))
self.transport.write(send_message.encode())
def eof_received(self):
""" an EOF has been received from the client.
This indicates the client has gracefully exited
the connection. Inform the pubsub hub that the
subscriber is gone
"""
print('Client {} closed connection'.format(self.peername))
self.transport.close()
def connection_lost(self, exc):
""" A transport error or EOF is seen which
means the client is disconnected.
Inform the pubsub hub that the subscriber has
Disappeared
"""
if exc:
print('{} {}'.format(exc, self.peername))
loop = asyncio.get_event_loop()
# Each client will create a new protocol instance
coro = loop.create_server(SubscriberServerProtocol, '127.0.0.1', 10666)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
try:
server.close()
loop.until_complete(server.wait_closed())
loop.close()
except:
pass
And here's the client code:
#!/usr/bin/env python3
import asyncio
import json
class SubscriberClientProtocol(asyncio.Protocol):
def __init__(self, message, loop):
self.message = message
self.loop = loop
def connection_made(self, transport):
""" Upon connection send the message to the
server
A message has to have the following items:
type: subscribe/unsubscribe
channel: the name of the channel
"""
transport.write(self.message.encode())
print('Message sent: {!r}'.format(self.message))
def data_received(self, data):
""" After sending a message we expect a reply
back from the server
The return message consist of three fields:
type: subscribe/unsubscribe
channel: the name of the channel
channel_count: the amount of channels subscribed to
"""
print('Message received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
if __name__ == '__main__':
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: SubscriberClientProtocol(message,
loop),
'127.0.0.1', 10666)
loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print('Closing connection')
loop.close()

Your server is fine as-is for what you're trying to do; your code as written actually keeps the TCP connection alive, it's you just don't have the plumbing in place to continously feed it new messages. To do that, you need to tweak the client code so that you can feed new messages into it whenever you want, rather than only doing it when the connection_made callback fires.
This is easy enough; we'll add an internal asyncio.Queue to the ClientProtocol which can receive messages, and then run a coroutine in an infinite loop that consumes the messages from that Queue, and sends them on to the server. The final piece is to actually store the ClientProtocol instance you get back from the create_connection call, and then pass it to a coroutine that actually sends messages.
import asyncio
import json
class SubscriberClientProtocol(asyncio.Protocol):
def __init__(self, loop):
self.transport = None
self.loop = loop
self.queue = asyncio.Queue()
self._ready = asyncio.Event()
asyncio.async(self._send_messages()) # Or asyncio.ensure_future if using 3.4.3+
#asyncio.coroutine
def _send_messages(self):
""" Send messages to the server as they become available. """
yield from self._ready.wait()
print("Ready!")
while True:
data = yield from self.queue.get()
self.transport.write(data.encode('utf-8'))
print('Message sent: {!r}'.format(message))
def connection_made(self, transport):
""" Upon connection send the message to the
server
A message has to have the following items:
type: subscribe/unsubscribe
channel: the name of the channel
"""
self.transport = transport
print("Connection made.")
self._ready.set()
#asyncio.coroutine
def send_message(self, data):
""" Feed a message to the sender coroutine. """
yield from self.queue.put(data)
def data_received(self, data):
""" After sending a message we expect a reply
back from the server
The return message consist of three fields:
type: subscribe/unsubscribe
channel: the name of the channel
channel_count: the amount of channels subscribed to
"""
print('Message received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
#asyncio.coroutine
def feed_messages(protocol):
""" An example function that sends the same message repeatedly. """
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
while True:
yield from protocol.send_message(message)
yield from asyncio.sleep(1)
if __name__ == '__main__':
message = json.dumps({'type': 'subscribe', 'channel': 'sensor'},
separators=(',', ':'))
loop = asyncio.get_event_loop()
coro = loop.create_connection(lambda: SubscriberClientProtocol(loop),
'127.0.0.1', 10666)
_, proto = loop.run_until_complete(coro)
asyncio.async(feed_messages(proto)) # Or asyncio.ensure_future if using 3.4.3+
try:
loop.run_forever()
except KeyboardInterrupt:
print('Closing connection')
loop.close()