Develop Reference

How can i send socket asynchronously? [duplicate]

I would like to re-implement my code using asyncio coroutines instead of multi-threading.
server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
try:
while True:
client, _ = server.accept()
threading.Thread(target=handle_client, args=(client,)).start()
except KeyboardInterrupt:
server.close()
client.py
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(('localhost', 15555))
request = None
try:
while request != 'quit':
request = input('>> ')
if request:
server.send(request.encode('utf8'))
response = server.recv(255).decode('utf8')
print(response)
except KeyboardInterrupt:
server.close()
I know there are some appropriate asynchronous network librairies to do that. But I just want to only use asyncio core library on this case in order to have a better understanding of it.
It would have been so nice to only add async keyword before handle client definition... Here a piece of code which seems to work, but I'm still confused about the implementation.
asyncio_server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
def run_server(server):
client, _ = server.accept()
handle_client(client)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
loop = asyncio.get_event_loop()
asyncio.async(run_server(server))
try:
loop.run_forever()
except KeyboardInterrupt:
server.close()
How adapt this in the best way and using async await keywords.

The closest literal translation of the threading code would create the socket as before, make it non-blocking, and use asyncio low-level socket operations to implement the server. Here is an example, sticking to the more relevant server part (the client is single-threaded and likely fine as-is):
import asyncio, socket
async def handle_client(client):
loop = asyncio.get_event_loop()
request = None
while request != 'quit':
request = (await loop.sock_recv(client, 255)).decode('utf8')
response = str(eval(request)) + '\n'
await loop.sock_sendall(client, response.encode('utf8'))
client.close()
async def run_server():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
server.setblocking(False)
loop = asyncio.get_event_loop()
while True:
client, _ = await loop.sock_accept(server)
loop.create_task(handle_client(client))
asyncio.run(run_server())
The above works, but is not the intended way to use asyncio. It is very low-level and therefore error-prone, requiring you to remember to set the appropriate flags on the socket. Also, there is no buffering, so something as simple as reading a line from the client becomes a tiresome chore. This API level is really only intended for implementors of alternative event loops, which would provide their implementation of sock_recv, sock_sendall, etc.
Asyncio's public API provides two abstraction layers intended for consumption: the older transport/protocol layer modeled after Twisted, and the newer streams layer. In new code, you almost certainly want to use the streams API, i.e. call asyncio.start_server and avoid raw sockets. That significantly reduces the line count:
import asyncio, socket
async def handle_client(reader, writer):
request = None
while request != 'quit':
request = (await reader.read(255)).decode('utf8')
response = str(eval(request)) + '\n'
writer.write(response.encode('utf8'))
await writer.drain()
writer.close()
async def run_server():
server = await asyncio.start_server(handle_client, 'localhost', 15555)
async with server:
await server.serve_forever()
asyncio.run(run_server())

I have read the answers and comments above, trying to figure out how to use the asyncio lib for sockets.
As it often happens with Python, the official documentation along with the examples is the best source of useful information.
I got understanding of Transports and Protocols (low-level API), and Streams (high-level API) from the examples presented in the end of the support article.
For example, TCP Echo Server:
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
and TCP Echo Client:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
Hope it help someone searching for simple explanation of asyncio.

Python UDP and Websockets together

I'm working on a application. Where am using python websockets. Now I need UDP and WS asynchronously running and listening on different ports.
I'm unable to do it because WS recv() waits indefinitely untill a message is received. Message will be received and pushed into queue. I need UDP to receive and push to same queue. This below class implements only websockets. I need another class with UDP and both class instance run asynchronously.
import websockets
import json
from sinric.command.mainqueue import queue
from sinric.callback_handler.cbhandler
import CallBackHandler
from time import sleep
class SinricProSocket:
def __init__(self, apiKey, deviceId, callbacks):
self.apiKey = apiKey
self.deviceIds = deviceId
self.connection = None
self.callbacks = callbacks
self.callbackHandler = CallBackHandler(self.callbacks)
pass
async def connect(self): # Producer
self.connection = await websockets.client.connect('ws://2.5.2.2:301',
extra_headers={'Authorization': self.apiKey,
'deviceids': self.deviceIds},
ping_interval=30000, ping_timeout=10000)
if self.connection.open:
print('Client Connected')
return self.connection
async def sendMessage(self, message):
await self.connection.send(message)
async def receiveMessage(self, connection):
try:
message = await connection.recv()
queue.put(json.loads(message))
except websockets.exceptions.ConnectionClosed:
print('Connection with server closed')
async def handle(self):
# sleep(6)
while queue.qsize() > 0:
await self.callbackHandler.handleCallBacks(queue.get(), self.connection)
return

thanks for your time in the comments. I solved this issue by running instances of WS and UDP in 2 different daemon threads.

A good way to solve this issue would be to use threads. You could accept a message and put it into a queue, then handle the queue on a different thread.

How can I accept new connection requests in the background?

I have the server which accepts connection requests from clients. Clients send connection requests using this command: bash -i > /dev/tcp/ip/port 0<&1 1>&1. I want my server to instantly accept new connection requests and log them to console but I don't know how. In the code below there is while loop. As we can see command_accept() need to finish itself for client_accept() to start. That means I always need to pass some command to accept new client requests. I need client_accept() to be always running in the background.
I tried to set a time limit to my input but that's not a solution I need. Also I tried different libraries for asynchronous programming though I'm not sure I'm doing this correctly.
import socket
import time
import sys
host = '127.0.0.1'
port = 1344
id_counter = 0
server = socket.socket()
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.settimeout(0.1)
server.bind((host, port))
server.listen()
clients = {}
def client_accept(server):
while True:
try:
conn, addr = server.accept()
global id_counter
id_counter += 1
clients[id_counter] = (conn, addr)
print(f'{time.ctime()} New client [ID {id_counter}] with address {str(addr[0])}:{str(addr[1])}')
except socket.timeout:
break
def command_accept():
command = input('server > ')
#** don't pay attention **#
if command == 'exit':
sys.exit()
else:
print(f'command {command} accepted!')
while True:
command_accept()
client_accept(server)
Expected result: I don't pass anything to the input in command_accept and yet if new client sent request then the server will instantly accept it and print something like New client [ID 1] with address 127.0.0.1:45431.

Try to do that with socket.io and Threading, so if the socket got a ON_CONNECT event you can just push the information in a list and print it to the console.

as an excuse to experiment with the trio async library I ported your code to it
start by defining a simple class for client connections and the code to keep track of them:
from sys import stderr
from itertools import count
class Client:
def __init__(self, stream):
self.stream = stream
async def run(self):
lines = LineReader(self.stream)
while True:
line = (await lines.readline()).decode('ascii')
if not line or line.strip().casefold() in {'quit', 'exit'}:
await self.stream.send_all(b'bye!\r\n')
break
resp = f'got {line!r}'
await self.stream.send_all(resp.encode('ascii') + b'\r\n')
CLIENT_COUNTER = count()
CLIENTS = {}
async def handle_client(stream):
client_id = next(CLIENT_COUNTER)
client = Client(stream)
async with stream:
CLIENTS[client_id] = client
try:
await client.run()
except Exception as err:
print('client failed', err, file=stderr)
finally:
del CLIENTS[client_id]
LineReader comes from here: https://stackoverflow.com/a/53576829/1358308
next we can define the server stdin processing:
async def handle_local(nursery):
while True:
try:
command = await async_input('server > ')
except EOFError:
command = 'exit'
if command == 'exit':
nursery.cancel_scope.cancel()
elif command == 'list':
for id, client in CLIENTS.items():
print(id, client.stream.socket.getpeername())
else:
print(f'unknown command {command!r}')
check out the docs for info about nurseries
this uses a utility function to wrap input up into an async function.
import trio
async def async_input(prompt=None):
return await trio.run_sync_in_worker_thread(
input, prompt, cancellable=True)
then we define code to tie all the pieces together:
SERVE_HOST = 'localhost'
SERVE_PORT = 1344
async def async_main():
async with trio.open_nursery() as nursery:
nursery.start_soon(handle_local, nursery)
await trio.serve_tcp(
handle_client,
port=SERVE_PORT, host=SERVE_HOST,
handler_nursery=nursery)
trio.run(async_main)
some more links/references (by trio's author):
tutorial echo server
motivation behind the trio library

Python simple socket client/server using asyncio

I would like to re-implement my code using asyncio coroutines instead of multi-threading.
server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
try:
while True:
client, _ = server.accept()
threading.Thread(target=handle_client, args=(client,)).start()
except KeyboardInterrupt:
server.close()
client.py
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.connect(('localhost', 15555))
request = None
try:
while request != 'quit':
request = input('>> ')
if request:
server.send(request.encode('utf8'))
response = server.recv(255).decode('utf8')
print(response)
except KeyboardInterrupt:
server.close()
I know there are some appropriate asynchronous network librairies to do that. But I just want to only use asyncio core library on this case in order to have a better understanding of it.
It would have been so nice to only add async keyword before handle client definition... Here a piece of code which seems to work, but I'm still confused about the implementation.
asyncio_server.py
def handle_client(client):
request = None
while request != 'quit':
request = client.recv(255).decode('utf8')
response = cmd.run(request)
client.send(response.encode('utf8'))
client.close()
def run_server(server):
client, _ = server.accept()
handle_client(client)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
loop = asyncio.get_event_loop()
asyncio.async(run_server(server))
try:
loop.run_forever()
except KeyboardInterrupt:
server.close()
How adapt this in the best way and using async await keywords.

The closest literal translation of the threading code would create the socket as before, make it non-blocking, and use asyncio low-level socket operations to implement the server. Here is an example, sticking to the more relevant server part (the client is single-threaded and likely fine as-is):
import asyncio, socket
async def handle_client(client):
loop = asyncio.get_event_loop()
request = None
while request != 'quit':
request = (await loop.sock_recv(client, 255)).decode('utf8')
response = str(eval(request)) + '\n'
await loop.sock_sendall(client, response.encode('utf8'))
client.close()
async def run_server():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 15555))
server.listen(8)
server.setblocking(False)
loop = asyncio.get_event_loop()
while True:
client, _ = await loop.sock_accept(server)
loop.create_task(handle_client(client))
asyncio.run(run_server())
The above works, but is not the intended way to use asyncio. It is very low-level and therefore error-prone, requiring you to remember to set the appropriate flags on the socket. Also, there is no buffering, so something as simple as reading a line from the client becomes a tiresome chore. This API level is really only intended for implementors of alternative event loops, which would provide their implementation of sock_recv, sock_sendall, etc.
Asyncio's public API provides two abstraction layers intended for consumption: the older transport/protocol layer modeled after Twisted, and the newer streams layer. In new code, you almost certainly want to use the streams API, i.e. call asyncio.start_server and avoid raw sockets. That significantly reduces the line count:
import asyncio, socket
async def handle_client(reader, writer):
request = None
while request != 'quit':
request = (await reader.read(255)).decode('utf8')
response = str(eval(request)) + '\n'
writer.write(response.encode('utf8'))
await writer.drain()
writer.close()
async def run_server():
server = await asyncio.start_server(handle_client, 'localhost', 15555)
async with server:
await server.serve_forever()
asyncio.run(run_server())

I have read the answers and comments above, trying to figure out how to use the asyncio lib for sockets.
As it often happens with Python, the official documentation along with the examples is the best source of useful information.
I got understanding of Transports and Protocols (low-level API), and Streams (high-level API) from the examples presented in the end of the support article.
For example, TCP Echo Server:
import asyncio
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
print('Close the client socket')
self.transport.close()
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
server = await loop.create_server(
lambda: EchoServerProtocol(),
'127.0.0.1', 8888)
async with server:
await server.serve_forever()
asyncio.run(main())
and TCP Echo Client:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 8888)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
Hope it help someone searching for simple explanation of asyncio.

UDP server with asyncio

I am trying to receive UDP packets in a python asyncio loop. I am very new at asyncio so I'm probably doing something wrong, as the callbacks never get called:
import asyncio
class DiscoveryProtocol(asyncio.DatagramProtocol):
def __init__(self):
super().__init__()
def connection_made(self, transport):
self.transport = transport
def datagram_received(self, data, addr):
print(data)
def start_discovery():
loop = asyncio.get_event_loop()
t = loop.create_datagram_endpoint(DiscoveryProtocol,local_addr=('',5006))
loop.run_until_complete(t)
loop.run_forever()
I can receive packets with plain old sockets (without asyncio).
What am I doing wrong?

No accepted answer so this seems to have atrophied, but it comes up in searches. If someone gets here and wants a final solution, the following code snippet illustrates a fully functional UDP server. The write_messages() function is just a test method. It reads a log file with whatever you want in it, and publishes each line as a Syslog message to UDP port 514. Running this as a script illustrates the server listening and printing whatever it drains from syslog. update the SyslogProtocol with whatever formatting/processing needs you have.
import socket
import asyncio
import os, random
HOST, PORT = 'localhost', 514
def send_test_message(message: 'Message to send to UDP port 514') -> None:
sock = socket.socket(socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
sock.sendto(message.encode(), (HOST, PORT))
async def write_messages() -> "Continuously write messages to UDP port 514":
dir_path = os.path.dirname(os.path.realpath(__file__))
fp = open(os.path.join(dir_path, "tests/example.log"))
print("writing")
for line in fp.readlines():
await asyncio.sleep(random.uniform(0.1, 3.0))
send_test_message(line)
class SyslogProtocol(asyncio.DatagramProtocol):
def __init__(self):
super().__init__()
def connection_made(self, transport) -> "Used by asyncio":
self.transport = transport
def datagram_received(self, data, addr) -> "Main entrypoint for processing message":
# Here is where you would push message to whatever methods/classes you want.
print(f"Received Syslog message: {data}")
if __name__ == '__main__':
loop = asyncio.get_event_loop()
t = loop.create_datagram_endpoint(SyslogProtocol, local_addr=('0.0.0.0', PORT))
loop.run_until_complete(t) # Server starts listening
loop.run_until_complete(write_messages()) # Start writing messages (or running tests)
loop.run_forever()