Develop Reference

Client get message from server using asyncio in python

We are trying to use asyncio to run a straightforward client/server. The server is an echo server with two possible commands sent by the client, "quit" and "timer". The timer command starts a timer that will print a message in the console every second (at the server and client), and the quit command closes the connection.
The actual problem is the following:
When we run the server and the client, and we start the timer, the result of the timer is not sent to the client. It blocks the server and the client.
I believe that the problem is on the client's side. However, I was not able to detect it.
Server
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, writer: asyncio.StreamWriter, name = None, interval = 1) -> None:
self.name = name
self.interval = interval
self.writer = writer
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
delta = time.time() - self._init_time
self.writer.write(f"Timer {delta} ticked\n".encode())
self.writer.drain()
print("Delta time: ", delta)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def msg_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
'''Handle the echo protocol.'''
# timer task that the client can start:
timer_task = False
try:
while True:
data = await reader.read(1024) # Read 256 bytes from the reader. Size of the message
msg = data.decode() # Decode the message
addr, port = writer.get_extra_info("peername") # Get the address of the client
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = "Message received: " + msg
writer.write(send_message.encode()) # Echo the data back to the client
await writer.drain() # This will wait until everything is clear to move to the next thing.
if data == b"quit" and timer_task is True:
# cancel the timer_task (if any)
if timer_task:
timer_task.cancel()
await timer_task
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"quit" and timer_task is False:
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"start" and timer_task is False:
print("Starting timer")
t = Timer(writer)
timer_task = True
await t.start()
elif data == b"stop" and timer_task is True:
print("Stopping timer")
await t.stop()
timer_task = False
except ConnectionResetError:
print("Client disconnected")
async def run_server() -> None:
# Our awaitable callable.
# This callable is ran when the server recieves some data
server = await asyncio.start_server(msg_handler, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == "__main__":
loop = asyncio.new_event_loop() # new_event_loop() is for python 3.10. For older versions, use get_event_loop()
loop.run_until_complete(run_server())
Client
import asyncio
HOST = '127.0.0.1'
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input('Enter a message: ')
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception('Socket not communicating with the client')
print(f"Received {data.decode()!r}")
if (message == 'quit'):
writer.write(b"quit")
writer.close()
await writer.wait_closed()
exit(2)
# break # Don't know if this is necessary
if __name__ == '__main__':
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())

The client blocks on the input() function. This question is similar to server stop receiving msg after 1 msg receive

Finally, I found a possible solution, by separating the thread.
import asyncio
import websockets
import warnings
warnings.filterwarnings("ignore")
async def send_msg(websocket):
while True:
imp = await asyncio.get_event_loop().run_in_executor(None, lambda: input("Enter something: "))
print("MESSAGE: ", imp)
await websocket.send(imp)
#return imp
async def recv_msg(websocket):
while True:
msg = await websocket.recv()
print(f":> {msg}")
async def echo_loop():
uri = f"ws://localhost:8765"
async with websockets.connect(uri, ssl=None) as websocket:
while True:
await asyncio.gather(recv_msg(websocket),send_msg(websocket))
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(echo_loop())
asyncio.get_event_loop().run_forever()

It seems that there is no clear solution. In particular, there have been many changes in python since the early releases of asyncio, so many possible solutions are outdated.
I change the code to use WebSockets. However, the problem persists: input blocks the code, and none of the solutions above have solved my problem.
Below is the new version of the code (and the error remains):
Server
import asyncio
import websockets
import time
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, websocket, name=None, interval=1) -> None:
self.websocket = websocket
self.name = name
self.interval = interval
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
await self.websocket.send("tick")
print("Delta time: ", time.time() - self._init_time)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def handler(websocket):
print("[WS-SERVER] client connected")
while True:
try:
msg = await websocket.recv()
print(f"<: {msg}")
await websocket.send("Message received. {}".format(msg))
if(msg == "start"):
timer = Timer(websocket)
await timer.start()
except websockets.ConnectionClosed:
print("[WS-SERVER] client disconnected")
break
async def main():
async with websockets.serve(handler, "localhost", 8765):
print("[WS-SERVER] ready")
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())
Client
import asyncio
import websockets
'''async function that recieves and prints messages from the server'''
async def recieve_message(websocket):
msg1 = await websocket.recv()
print(f"<: {msg1}")
async def send_message(websocket):
msg = input("Put your message here: ")
await websocket.send(msg)
print(":> Sent message: ", msg)
async def handler():
uri = "ws://localhost:8765"
async with websockets.connect(uri) as websocket:
while True:
'''run input() in a separate thread'''
recv_msg, send_msg = await asyncio.gather(
recieve_message(websocket),
send_message(websocket),
return_exceptions=True)
if(send_msg == "test"):
print("Supertest")
async def main():
await handler()
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(handler())
print("[WS-CLIENT] bye")

TCP Server to communicate with Client on demand

Is there a way to memorize all the client instances (in a dictionary for example), and send a message/communicate on demand to whichever client needed?
import asyncio
class EchoServerClientProtocol(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
self.peername = transport.get_extra_info('peername')
print('Connection from {}'.format(self.peername))
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
def connection_lost(self, exc):
print('Lost connection of {}'.format(self.peername))
self.transport.close()
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerClientProtocol, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()

If I understand your question correctly, you want to be able to maintain a map of connections and then send data to the connections at a later time.
import asyncio
from functools import partial
from random import choice, randint
class Echo(asyncio.Protocol):
def __init__(self, client_map):
self.client_map = client_map
def connection_made(self, transport):
self.transport = transport
peername = transport.get_extra_info("peername")
print(f"[x] connected to {peername}")
self.client_map[transport.get_extra_info("peername")] = self
def connection_lost(self, exc):
peername = self.transport.get_extra_info("peername")
print(f"[!] disconnected from {peername}")
del self.client_map[peername]
def data_received(self, data):
print(f"<<< {data}")
self.transport.write(b">>> " + data)
def protocol_factory(client_map):
proto = Echo(client_map)
return proto
def random_hello(loop, client_map):
delay = randint(1, 10)
if len(client_map) > 0:
peername = choice(sorted(client_map))
client_map[peername].transport.write(b">>> random hello\n")
loop.call_later(
delay, random_hello, loop, client_map,
)
async def main():
loop = asyncio.get_running_loop()
client_map = {}
_protocol_factory = partial(protocol_factory, client_map=client_map)
server = await loop.create_server(_protocol_factory, "127.0.0.1", 8888,)
random_hello(loop, client_map)
async with server:
await server.serve_forever()
asyncio.run(main())
What makes this all work is protocol_factory(). You pass in a client_map: dict, which will then be a shared variable in all the protocol objects, and as clients connect, the protocol object will be stored under the peername key. The random_hello() function demonstrates sending data to a protocol. Pass in client_map anywhere you want to be able to access connected clients and send data over the network.

In python socketio RuntimeWarning: coroutine 'initial' was never awaited despite writing an await statement

python3 client.py
/home/aijax/.local/lib/python3.6/site-packages/socketio/client.py:592: RuntimeWarning: coroutine 'initial' was never awaited
self._handle_event(pkt.namespace, pkt.id, pkt.data)
connection established
despite of having the await I'm getting the error
PS: I have little to no knowledge of Async-io of python I kinda have finish this task overnight for a school proj
my client.py
import socketio
import time
sio = socketio.Client()
val={"num":35}
#sio.event
def connect():
print('connection established')
#sio.event
def disconnect():
print('disconnected from server')
#sio.event
async def initial(sid):
global val,sio
print("hey there!! ")
await sio.emit('clientUpdate',val)
#sio.event
async def serverAggregate(avg):
global val
val["num"] = (avg["num"] + val["num"])/2
# print(val)
print("kaada ",avg)
time.sleep(4)
await sio.emit('clientUpdate',val)
# await
sio.connect('http://0.0.0.0:8080')
# sio.wait()
# sio.my_message()
sio.wait()
My task is to create a server and many clients which send a number to server and server aggregates that and sends it back to the client and perform a local update and send back the number
my server.py
from aiohttp import web
import socketio
import time
import threading
# creates a new Async Socket IO Server
sio = socketio.AsyncServer()
users_count=0 #num of clients
temp_count=users_count
avg={"num":0}
temp_sum=0
initFlag=True
# Creates a new Aiohttp Web Application
# app = web.Application()
async def handle(request):
print(request)
name = request.match_info['name']
text = "Hello, " + name
return web.Response(text=text)
app = web.Application()
sio.attach(app)
async def resetter():
global avg,temp_sum,users_count,temp_count,sio
avg = temp_sum/users_count
temp_count = users_count
temp_sum = 0
print("broadcast requests to clients")
time.sleep(2)
await sio.emit('serverAggregate',avg,broadcast = True, include_self = False)
#sio.on('clientUpdate')
async def serverAggregate(sid, data):
global temp_count,users_count,initFlag,sio,resetter
if initFlag:
temp_count=users_count
initFlag=False
temp_count=temp_count -1
if(temp_count==0):
await resetter()
else:
print("message ", data)
global temp_sum
temp_sum = temp_sum + data["num"]
print("message ", data)
#sio.event
async def connect(sid, environ):
global users_count,sio
print("connect ", sid)
users_count=users_count+1
# time.sleep(3)
# print("say something")
time.sleep(2)
await sio.emit('initial')
# await sio.emit('intialize', broadcast = True, include_self = False)
#sio.event
def disconnect(sid):
global users_count
print('disconnect ', sid)
users_count=users_count-1
app.add_routes([web.get('/', handle),
web.get('/{name}', handle)])
# app.router.add_static('/static', 'static')
class BgProc(threading.Thread):
def __init__(self, sio):
# calling superclass init
threading.Thread.__init__(self)
# self.text = text
self.sio = sio
def run(self):
time.sleep(5)
print("sending broadcast to all clients ")
self.sio.emit("initial",broadcast = True, include_self = False)
if __name__ == '__main__':
web.run_app(app)

You are using the socketio.Client() class which is the standard Python client. If you want to write an asyncio application, you must use socketio.AsyncClient().

Autobahn Asyncio ReconnectingClientFactory

I would like to make a ReconnectingClientFactory with asyncio. In particular to handle the case that the server is not available when the client is started in which case the ReconnectingClientFactory will keep trying. That is something that the asyncio.events.create_connection does not do.
Concretely:
The EchoClient example would be fine.
The crux is how the connection is made.
factory = EchoClientFactory('ws://127.0.0.1:5678')
connectWS(factory)
in the case of the twisted version with ReconnectingClientFactory.
Vs
factory = EchoClientFactory(u"ws://127.0.0.1:5678")
factory.protocol = SecureServerClientProtocol
loop = asyncio.get_event_loop()
# coro = loop.create_connection(factory, 'ws_server', 5678)
coro = loop.create_connection(factory, '127.0.0.1', 5678)
loop.run_until_complete(asyncio.wait([
alive(), coro
]))
loop.run_forever()
loop.close()
Or similar with the asycnio version.
The problem is that in the asyncio version the connection is established by asyncio.events.create_connection which simply fails if the server is not available.
How can I reconcile the two?
Many thanks

I think I get what you want. Here's the code and example based on asyncio TCP echo client protocol example.
import asyncio
import random
class ReconnectingTCPClientProtocol(asyncio.Protocol):
max_delay = 3600
initial_delay = 1.0
factor = 2.7182818284590451
jitter = 0.119626565582
max_retries = None
def __init__(self, *args, loop=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
self._args = args
self._kwargs = kwargs
self._retries = 0
self._delay = self.initial_delay
self._continue_trying = True
self._call_handle = None
self._connector = None
def connection_lost(self, exc):
if self._continue_trying:
self.retry()
def connection_failed(self, exc):
if self._continue_trying:
self.retry()
def retry(self):
if not self._continue_trying:
return
self._retries += 1
if self.max_retries is not None and (self._retries > self.max_retries):
return
self._delay = min(self._delay * self.factor, self.max_delay)
if self.jitter:
self._delay = random.normalvariate(self._delay,
self._delay * self.jitter)
self._call_handle = self._loop.call_later(self._delay, self.connect)
def connect(self):
if self._connector is None:
self._connector = self._loop.create_task(self._connect())
async def _connect(self):
try:
await self._loop.create_connection(lambda: self,
*self._args, **self._kwargs)
except Exception as exc:
self._loop.call_soon(self.connection_failed, exc)
finally:
self._connector = None
def stop_trying(self):
if self._call_handle:
self._call_handle.cancel()
self._call_handle = None
self._continue_trying = False
if self._connector is not None:
self._connector.cancel()
self._connector = None
if __name__ == '__main__':
class EchoClientProtocol(ReconnectingTCPClientProtocol):
def __init__(self, message, *args, **kwargs):
super().__init__(*args, **kwargs)
self.message = message
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self._loop.stop()
loop = asyncio.get_event_loop()
client = EchoClientProtocol('Hello, world!', '127.0.0.1', 8888, loop=loop)
client.connect()
loop.run_forever()
loop.close()

Why isn't data_received() being called?

I'm on Python 3.4 making a practice app to learn Python's asyncio module.
I've got several modules in my application:
app.py
import asyncio
import logging
class Client(asyncio.Protocol):
def __init__(self, loop):
self.loop = loop
def connection_made(self, transport):
print('Connection with server established')
self.transport = transport
def data_received(self, data):
data = data.decode()
print('Received: ', data)
if not int(data):
print('Stopping loop')
self.loop.stop()
else:
message = 'remove one'.encode()
self.transport.write(message)
print('Sent:', message)
def connection_lost(self, exc):
print('Connection with server lost.')
self.loop.stop()
loop = asyncio.get_event_loop()
fn = loop.create_connection(
lambda: Client(loop), '127.0.0.1', 9999
)
logging.basicConfig(level=logging.DEBUG)
client = loop.run_until_complete(fn)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
print('Loop ended')
loop.close()
counter.py
import asyncio
class Counter:
def __init__(self, maxcount):
self.maxcount = maxcount
#asyncio.coroutine
def count(self):
yield from asyncio.sleep(self.maxcount)
print('Done counting')
serie.py
import asyncio
class Serie:
def __init__(self, loop, items=None):
self.loop = loop
self.items = items or []
#asyncio.coroutine
def remove_one(self, counter):
if len(self.items) is not 0:
yield from counter.count()
item = self.items.pop(0)
print('Removed', item)
else:
print('Serie is empty')
#asyncio.coroutine
def start_removing(self, counter):
while self.items:
yield from self.remove_one(counter)
print('Serie.start_removing() has finished')
mission_control.py
import asyncio
import logging
from counter import Counter
from serie import Serie
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
yield from self.serie.remove_one()
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
else:
print('Done')
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
counter = Counter(2)
planets = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
fn = loop.create_server(
lambda: MissionControl(loop, counter, planets), '127.0.0.1', 9999
)
server = loop.run_until_complete(fn)
print('Server started')
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
You can also find the source at this Github gist.
In mission_control.py, the method data_received() appears not to be called when the client (app.py) sends data via its self.transport property.
Where is the implementation error and how can I fix it?

The problem is that data_received is not (and cannot be) a coroutine, but you're using yield from inside it. Internally asyncio is just calling self.data_received(data) without any yield from call, which means the body of the method isn't being executed at all - instead a generator object is immediately returned. You need to refactor your implementation to not use yield from, which requires using callbacks instead:
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
fut = asyncio.async(self.serie.remove_one(self.counter))
fut.add_done_callback(self.on_removed_one)
else:
print('Done')
def on_removed_one(self, result):
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
Another option would be to use the asyncio Streams API instead of asyncio.Protocol, which will allow you to use coroutines:
import asyncio
import logging
from counter import Counter
from serie import Serie
#asyncio.coroutine
def mission_control(reader, writer):
counter = Counter(2)
serie = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
writer.write(str(len(serie.items)).encode())
while True:
data = (yield from reader.read(100)).decode()
print('Received:', data)
if data == 'remove one':
result = yield from serie.remove_one(counter)
else:
print('Done')
return
print('Removed one: {}'.format(serie.items))
writer.write(str(len(serie.items)).encode())
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
coro = asyncio.start_server(mission_control, '127.0.0.1', 9999, loop=loop)
server = loop.run_until_complete(coro)
# The rest is the same