Develop Reference

Python, server - client connection coroutine issues asyncio

Been trying to create a server and client script that allows clients to connect to the server and join different rooms to chat. As of now everything is terminal based. The issue at hand is the following.
When a new clients connects to a existing room i get the following error:
Task exception was never retrieved
future: <Task finished name='Task-9' coro=<ChatServer.handle_client() done, defined at C:\Users\~~~~\server.py:10> exception=TypeError('a coroutine was expected, got None')>
Traceback (most recent call last):
File "C:\Users\~~~~\server.py", line 41, in handle_client
await self.broadcast(f'{addr} has joined room {room}!\n', room, writer)
File "C:\Users\~~~~\server.py", line 75, in broadcast
task = asyncio.create_task(client.write(message.encode()))
File "C:\Users\~~~~\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 361, in create_task
task = loop.create_task(coro)
File "C:\Users\~~~~\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 438, in create_task
task = tasks.Task(coro, loop=self, name=name)
TypeError: a coroutine was expected, got None
But, if the clients join separete rooms there is no error. If a client join an exists room with another client it inside, it gets that error. I have been trying to understand the error at hand but cant wrap my head around it.
Here is the code for the server:
import asyncio
#available_rooms = {"Room_1"}
class ChatServer:
def __init__(self):
self.clients = {}
self.rooms = {}
async def handle_client(self, reader, writer):
addr = writer.get_extra_info('peername')
print(f'Connected by {addr}')
# Send the list of available rooms to the client
available_rooms = '\n'.join(self.rooms.keys())
print("---->" + available_rooms)
writer.write(f'Available Rooms: {available_rooms}\n'.encode())
await writer.drain()
while True:
message = await reader.readline()
if not message:
break
message = message.decode().strip()
command = message.split()[0]
if command == "LIST":
available_rooms = '\n'.join(self.rooms.keys())
writer.write(f'Available Rooms: {available_rooms}\n'.encode())
await writer.drain()
elif command == "JOIN":
room = message.split()[1]
if room not in self.rooms:
writer.write(f"{room} room not found. creating it\n".encode())
self.rooms[room] = []
await writer.drain()
self.clients[writer] = room
self.rooms[room].append(writer)
writer.write(f'You joined room: {room}!\n'.encode())
await writer.drain()
await self.broadcast(f'{addr} has joined room {room}!\n', room, writer)
elif command == "LEAVE":
if writer not in self.clients:
writer.write(f'You are not currently in a room!\n'.encode())
await writer.drain()
else:
room = self.clients[writer]
writer.write(f'You left room {room}!\n'.encode())
await writer.drain()
self.clients.pop(writer)
self.rooms[room].remove(writer)
await self.broadcast(f'{addr} has left room {room}!\n', room, writer)
# remove the room if it is empty
if not self.rooms[room]:
self.rooms.pop(room)
else:
try:
await self.broadcast(f'{addr}: {message}\n', self.clients[writer], writer)
except KeyError:
writer.write(f'You are not currently in a room!\n'.encode())
await writer.drain()
continue
print(f'Disconnected {addr}')
writer.close()
async def broadcast(self, message, room, sender):
tasks = []
for client in self.rooms[room]:
# exclude sender from broadcast
if client != sender:
task = asyncio.create_task(client.write(message.encode()))
tasks.append(task)
if tasks:
await asyncio.gather(*tasks)
def start(self):
loop = asyncio.get_event_loop()
coro = asyncio.start_server(self.handle_client, '127.0.0.1', 5323, loop=loop)
#coro = asyncio.start_server(self.handle_client, '10.44.33.158', 5000, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
server = ChatServer()
server.start()
And here is the code for the client:
import asyncio
class ChatClient:
def __init__(self):
self.room = None
async def start(self, host, port):
self.reader, self.writer = await asyncio.open_connection(host, port)
# Get the list of available rooms
available_rooms = await self.reader.readline()
print(available_rooms.decode())
self.writer.write('JOIN myroom_2\n'.encode())
await self.writer.drain()
resp = await self.reader.readline()
if resp:
print(resp.decode())
self.room = "myroom_2"
while True:
message = await self.reader.readline()
print(message.decode())
message_to_send = input()
if message_to_send == "LEAVE":
self.writer.write("LEAVE\n".encode())
await self.writer.drain()
print("left the room")
break
elif message_to_send.startswith("JOIN"):
_, new_room = message_to_send.split()
self.writer.write(f"JOIN {new_room}\n".encode())
await self.writer.drain()
resp = await self.reader.readline()
if resp:
print(resp.decode())
self.room = new_room
continue
elif message_to_send == "LIST":
self.writer.write("LIST\n".encode())
await self.writer.drain()
response = await self.reader.readline()
print(response.decode())
continue
self.writer.write(f"{message_to_send}\n".encode())
await self.writer.drain()
async def list_rooms(self):
self.writer.write("LIST\n".encode())
await self.writer.drain()
try:
response = await asyncio.wait_for(self.reader.readline(), timeout=5)
print(response.decode())
except asyncio.TimeoutError:
print("timeout reached while waiting for a response")
async def change_room(self):
self.writer.write("Enter new room name:\n".encode())
await self.writer.drain()
new_room = input()
self.writer.write(f"JOIN {new_room}\n".encode())
await self.writer.drain()
response = await self.reader.readline()
print(response.decode())
self.room = new_room
async def write(self, message):
self.writer.write(message.encode())
await self.writer.drain()
client = ChatClient()
asyncio.run(client.start('127.0.0.1',5323 ))
I have tried different write methods but nothing works, unable to wrap my head around this issue.

The problem is in how you are using asyncio.create_task.
In the Server.broadcast function you call:
task = asyncio.create_task(client.write(message.encode()))
asyncio.create_task expects a coroutine but you are giving it a normal funciton. client in this context will be a asyncio.StreamWriter, and the write method is a normal function. you should probably create a task from the client.drain coroutine:
client.write(message.encode())
task = asyncio.create_task(client.drain())

How to detect closed websocket in asyncio.gather(*tasks)

I have a list of asyncio tasks which contains with connecting,handshaking and receiving data from a websocket. This process is running correctly but sometimes the connection of one of the websocket (or maybe all of them) is closed.
How can I detect and make a new conncetion to the closed one?
Here is the code which I use:
async def main(_id):
try:
async with websockets.connect("wss://ws.bitpin.ir/", extra_headers = request_header, timeout=10, ping_interval=None) as websocket:
await websocket.send('{"method":"sub_to_price_info"}')
recv_msg = await websocket.recv()
if recv_msg == '{"message": "sub to price info"}':
await websocket.send(json.dumps({"method":"sub_to_market","id":_id}))
recv_msg = await websocket.recv()
print(recv_msg)
counter = 1
task = asyncio.create_task(ping_func(websocket))
while True:
msg = await websocket.recv()
return_func(msg, counter, asyncio.current_task().get_name()) ## Return recieved message
counter+=1
except Exception as e:
err_log(name='Error in main function', text=str(e))
async def ping_func(websocket):
try:
while True:
await websocket.send('{"message":"PING"}')
# print('------ ping')
await asyncio.sleep(5)
except Exception as e:
err_log(name='Error in ping function', text=str(e))
def return_func(msg, counter, task_name):
if msg != '{"message": "PONG"}' and len(json.loads(msg)) <20:
print(task_name, counter, msg[:100])
else:
print(task_name, counter)
async def handler():
try:
tasks = []
for _id in symbols_id_dict.values():
tasks.append(asyncio.create_task(main(_id), name='task{}'.format(_id)))
responses = await asyncio.gather(*tasks)
except Exception as e:
err_log(name='Error in handler function', text=str(e))
try:
if __name__ == '__main__':
asyncio.run(handler())
else:
os._exit(0)
except Exception as e:
err_log(name='Error in running asyncio handler', text=str(e))
finally:
os._exit(0)
According to the below line, each task is specified with a name:
tasks.append(asyncio.create_task(main(_id), name='task{}'.format(_id)))
So each task can be detected. How can I use this feature to detect closed websocket.

try:
data = await ws.recv()
except (ConnectionClosed):
print("Connection is Closed")
data = None
print('Reconnecting')
websocket = await websockets.connect(params)

None block 'while True' using asyncio

Using below code I'm attempting to start 2 infinite loops using asyncio:
async def do_job_1():
while True :
print('do_job_1')
await asyncio.sleep(5)
async def do_job_2():
while True :
print('do_job_2')
await asyncio.sleep(5)
if __name__ == '__main__':
asyncio.run(do_job_1())
asyncio.run(do_job_2())
do_job_1 blocks do_job_2, as do_job_2 never prints do_job_1. What mistake have I made ?
Ultimately I'm trying to convert kafka consumer code:
from confluent_kafka import Consumer, KafkaError
settings = {
'bootstrap.servers': 'localhost:9092',
'group.id': 'mygroup',
'client.id': 'client-1',
'enable.auto.commit': True,
'session.timeout.ms': 6000,
'default.topic.config': {'auto.offset.reset': 'smallest'}
}
c = Consumer(settings)
c.subscribe(['mytopic'])
try:
while True:
msg = c.poll(0.1)
if msg is None:
continue
elif not msg.error():
print('Received message: {0}'.format(msg.value()))
elif msg.error().code() == KafkaError._PARTITION_EOF:
print('End of partition reached {0}/{1}'
.format(msg.topic(), msg.partition()))
else:
print('Error occured: {0}'.format(msg.error().str()))
except KeyboardInterrupt:
pass
finally:
c.close()
taken from https://www.confluent.io/blog/introduction-to-apache-kafka-for-python-programmers to be concurrent so I can parallelize processing of Kafka messages.

From help(asyncio.run):
It should be used as a main entry point for asyncio programs, and should ideally only be called once.
But you can use asyncio.gather to join the tasks:
import asyncio
async def do_job_1():
while True :
print('do_job_1')
await asyncio.sleep(5)
async def do_job_2():
while True :
print('do_job_2')
await asyncio.sleep(5)
async def main():
await asyncio.gather(do_job_1(), do_job_2())
if __name__ == '__main__':
asyncio.run(main())

Sending and receiving messages in parallel does not work

I have a simple asyncio tcp chat server/client.
But there is an issue with, it seems to me, sending messages. Instead of receiving and sending messages in parallel, if I have 2 or more clients, they receive other messages only after they send theirs.
How to solve this problem?
import asyncio
list_of_users = {}
async def handle_echo(reader, writer):
name = await reader.read(1024)
name.decode()
print(name)
addr = writer
list_of_users[addr]
while True:
data = await reader.read(1024)
message = data.decode()
if not message:
del list_of_users[addr]
break
msg(message)
def msg(message):
for user in list_of_users:
print('send message - ', message)
user.write(message.encode())
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
asyncio.run(main())
import asyncio
from aioconsole import ainput
async def tcp_echo_client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
name = input('Enter your name: ')
writer.write(name.encode())
while True:
await output_messages(writer)
await incoming_messages(reader)
async def output_messages(writer):
message = await ainput()
writer.write(message.encode())
async def incoming_messages(reader):
input_message = await reader.read(1024)
print('print incoming message', input_message)
async def main():
await tcp_echo_client()
asyncio.run(main())

The main issue was that the coroutine incoming_messages did not run in the background. Instead it was repeatedly called after output_messages finished.
I also had to make other adjustments to get the code running on Python 3.6
run incoming_messages in the background
list_of_users is now a list
no context handler for the server
no serve_forever
Try this code:
Server
import asyncio
list_of_users = []
async def handle_echo(reader, writer):
name = await reader.read(1024)
name.decode()
print(name)
addr = writer
list_of_users.append(addr)
while True:
data = await reader.read(1024)
message = data.decode()
if not message:
list_of_users.remove(addr)
break
msg(message)
def msg(message):
for user in list_of_users:
print('send message - ', message)
user.write(message.encode())
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
await asyncio.sleep(3600)
asyncio.ensure_future(main())
loop = asyncio.get_event_loop()
loop.run_forever()
Client
import asyncio
from aioconsole import ainput
async def tcp_echo_client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
name = input('Enter your name: ')
writer.write(name.encode())
asyncio.ensure_future(incoming_messages(reader))
while True:
await output_messages(writer)
async def output_messages(writer):
message = await ainput()
writer.write(message.encode())
async def incoming_messages(reader):
while True:
input_message = await reader.read(1024)
print('print incoming message', input_message)
async def main():
await tcp_echo_client()
asyncio.get_event_loop().run_until_complete(main())

Listening with socket from async method Python

I want my script to do two things:
Start some function with infinite loop and
Listen some port using socket.
I tried to run them both as async methods like in sample code below, but foo() is blocked after listen() was started.
async def foo(name):
while True:
print(name, datetime.datetime.now())
await asyncio.sleep(1)
return await True
async def listen(port):
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('localhost', port))
serversocket.listen(1)
while True:
connection, address = serversocket.accept()
buf = connection.recv(512)
if len(buf) > 0:
print(buf)
break
return True
async def go_foo(name='foo'):
b = await foo(name)
async def go_server(port=8089):
print('server on port', port)
b = await listen(port)
asyncio.ensure_future(go_server())
asyncio.ensure_future(go_foo())
loop = asyncio.get_event_loop()
loop.run_forever()
How to realize such task correctly?