I'm on Python 3.4 making a practice app to learn Python's asyncio module.
I've got several modules in my application:
app.py
import asyncio
import logging
class Client(asyncio.Protocol):
def __init__(self, loop):
self.loop = loop
def connection_made(self, transport):
print('Connection with server established')
self.transport = transport
def data_received(self, data):
data = data.decode()
print('Received: ', data)
if not int(data):
print('Stopping loop')
self.loop.stop()
else:
message = 'remove one'.encode()
self.transport.write(message)
print('Sent:', message)
def connection_lost(self, exc):
print('Connection with server lost.')
self.loop.stop()
loop = asyncio.get_event_loop()
fn = loop.create_connection(
lambda: Client(loop), '127.0.0.1', 9999
)
logging.basicConfig(level=logging.DEBUG)
client = loop.run_until_complete(fn)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
print('Loop ended')
loop.close()
counter.py
import asyncio
class Counter:
def __init__(self, maxcount):
self.maxcount = maxcount
#asyncio.coroutine
def count(self):
yield from asyncio.sleep(self.maxcount)
print('Done counting')
serie.py
import asyncio
class Serie:
def __init__(self, loop, items=None):
self.loop = loop
self.items = items or []
#asyncio.coroutine
def remove_one(self, counter):
if len(self.items) is not 0:
yield from counter.count()
item = self.items.pop(0)
print('Removed', item)
else:
print('Serie is empty')
#asyncio.coroutine
def start_removing(self, counter):
while self.items:
yield from self.remove_one(counter)
print('Serie.start_removing() has finished')
mission_control.py
import asyncio
import logging
from counter import Counter
from serie import Serie
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
yield from self.serie.remove_one()
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
else:
print('Done')
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
counter = Counter(2)
planets = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
fn = loop.create_server(
lambda: MissionControl(loop, counter, planets), '127.0.0.1', 9999
)
server = loop.run_until_complete(fn)
print('Server started')
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
You can also find the source at this Github gist.
In mission_control.py, the method data_received() appears not to be called when the client (app.py) sends data via its self.transport property.
Where is the implementation error and how can I fix it?
The problem is that data_received is not (and cannot be) a coroutine, but you're using yield from inside it. Internally asyncio is just calling self.data_received(data) without any yield from call, which means the body of the method isn't being executed at all - instead a generator object is immediately returned. You need to refactor your implementation to not use yield from, which requires using callbacks instead:
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
fut = asyncio.async(self.serie.remove_one(self.counter))
fut.add_done_callback(self.on_removed_one)
else:
print('Done')
def on_removed_one(self, result):
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
Another option would be to use the asyncio Streams API instead of asyncio.Protocol, which will allow you to use coroutines:
import asyncio
import logging
from counter import Counter
from serie import Serie
#asyncio.coroutine
def mission_control(reader, writer):
counter = Counter(2)
serie = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
writer.write(str(len(serie.items)).encode())
while True:
data = (yield from reader.read(100)).decode()
print('Received:', data)
if data == 'remove one':
result = yield from serie.remove_one(counter)
else:
print('Done')
return
print('Removed one: {}'.format(serie.items))
writer.write(str(len(serie.items)).encode())
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
coro = asyncio.start_server(mission_control, '127.0.0.1', 9999, loop=loop)
server = loop.run_until_complete(coro)
# The rest is the same
Related
I am trying to convert the following echo server IRC alike chat terminal into asyncio , but I don't think too much information about trio since it's new, but what is the translation of this to asyncio?
import trio
from itertools import count
from datetime import datetime
PORT = 9999
BUFSIZE = 16384
CONNECTION_COUNTER = count()
class ServerProtocol:
def __init__(self, server_stream):
self.ident = next(CONNECTION_COUNTER)
self.stream = server_stream
async def listen(self):
while True:
data = await self.stream.receive_some(BUFSIZE)
print("echo_server {}: received data {!r}".format(self.ident, data))
if not data:
print("echo_server {}: connection closed".format(self.ident))
return
print("echo_server {}: sending data {!r}".format(self.ident, data))
await self.stream.send_all('success'.encode())
class Server:
def __init__(self):
self.protocols = []
async def receive_connection(self, server_stream):
print('{} - {} CONNECTED.'.format(datetime.now(), dir(server_stream)))
sp: ServerProtocol = ServerProtocol(server_stream)
self.protocols.append(sp)
await sp.listen()
async def main():
await trio.serve_tcp(Server().receive_connection, PORT)
trio.run(main)
Thanks to anyio you don't have to change much of your code to make it work with asyncio ;)
import anyio
from itertools import count
from datetime import datetime
PORT = 9999
BUFSIZE = 16384
CONNECTION_COUNTER = count()
class ServerProtocol:
def __init__(self, server_stream):
self.ident = next(CONNECTION_COUNTER)
self.stream = server_stream
async def listen(self):
async with self.stream:
data = await self.stream.receive(BUFSIZE)
print("echo_server {}: received data {!r}".format(self.ident, data))
if not data:
print("echo_server {}: connection closed".format(self.ident))
return
print("echo_server {}: sending data {!r}".format(self.ident, data))
await self.stream.send('success'.encode())
class Server:
def __init__(self):
self.protocols = []
async def receive_connection(self, server_stream):
print('{} - {} CONNECTED.'.format(datetime.now(), dir(server_stream)))
sp: ServerProtocol = ServerProtocol(server_stream)
self.protocols.append(sp)
await sp.listen()
async def main():
listener = await anyio.create_tcp_listener(local_port=PORT)
await listener.serve(Server().receive_connection)
anyio.run(main)
You should definitely take a look at anyio :)
I'd go with something like:
import asyncio
from itertools import count
PORT = 9999
BUFSIZE = 16384
CONNECTION_COUNTER = count()
class ServerProtocol:
def __init__(self, read, write):
self.ident = next(CONNECTION_COUNTER)
self.read = read
self.write = write
async def listen(self):
while True:
data = await self.read.read(BUFSIZE)
print("echo_server {}: received data {!r}".format(self.ident, data))
if not data:
print("echo_server {}: connection closed".format(self.ident))
return
print("echo_server {}: sending data {!r}".format(self.ident, data))
self.write.write('success'.encode())
await self.write.drain()
class Server:
def __init__(self):
self.protocols = []
async def receive_connection(self, read, write):
sp: ServerProtocol = ServerProtocol(read, write)
self.protocols.append(sp)
await sp.listen()
async def main():
server = await asyncio.start_server(Server().receive_connection, '127.0.0.1', PORT)
async with server:
await server.serve_forever()
asyncio.run(main())
Note that I've kept your name ServerProtocol, but keep in mind that the "protocol" classes have a different meaning in asyncio. The code uses the asyncio "streams" layer (much like the trio original) and not the lower-level transport/protocol layer as might be understood from the "protocol" suffix in the name of the class.
Is there a way to memorize all the client instances (in a dictionary for example), and send a message/communicate on demand to whichever client needed?
import asyncio
class EchoServerClientProtocol(asyncio.Protocol):
def connection_made(self, transport):
self.transport = transport
self.peername = transport.get_extra_info('peername')
print('Connection from {}'.format(self.peername))
def data_received(self, data):
message = data.decode()
print('Data received: {!r}'.format(message))
print('Send: {!r}'.format(message))
self.transport.write(data)
def connection_lost(self, exc):
print('Lost connection of {}'.format(self.peername))
self.transport.close()
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerClientProtocol, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
If I understand your question correctly, you want to be able to maintain a map of connections and then send data to the connections at a later time.
import asyncio
from functools import partial
from random import choice, randint
class Echo(asyncio.Protocol):
def __init__(self, client_map):
self.client_map = client_map
def connection_made(self, transport):
self.transport = transport
peername = transport.get_extra_info("peername")
print(f"[x] connected to {peername}")
self.client_map[transport.get_extra_info("peername")] = self
def connection_lost(self, exc):
peername = self.transport.get_extra_info("peername")
print(f"[!] disconnected from {peername}")
del self.client_map[peername]
def data_received(self, data):
print(f"<<< {data}")
self.transport.write(b">>> " + data)
def protocol_factory(client_map):
proto = Echo(client_map)
return proto
def random_hello(loop, client_map):
delay = randint(1, 10)
if len(client_map) > 0:
peername = choice(sorted(client_map))
client_map[peername].transport.write(b">>> random hello\n")
loop.call_later(
delay, random_hello, loop, client_map,
)
async def main():
loop = asyncio.get_running_loop()
client_map = {}
_protocol_factory = partial(protocol_factory, client_map=client_map)
server = await loop.create_server(_protocol_factory, "127.0.0.1", 8888,)
random_hello(loop, client_map)
async with server:
await server.serve_forever()
asyncio.run(main())
What makes this all work is protocol_factory(). You pass in a client_map: dict, which will then be a shared variable in all the protocol objects, and as clients connect, the protocol object will be stored under the peername key. The random_hello() function demonstrates sending data to a protocol. Pass in client_map anywhere you want to be able to access connected clients and send data over the network.
I have two servers, created with asyncio.start_server:
asyncio.start_server(self.handle_connection, host = host, port = port) and running in one loop:
loop.run_until_complete(asyncio.gather(server1, server2))
loop.run_forever()
I'm using asyncio.Queue to communicate between servers. Messages from Server2, added via queue.put(msg) successfully receives by queue.get() in Server1. I'm running queue.get() by asyncio.ensure_future and using as callback for
add_done_callback method from Server1:
def callback(self, future):
msg = future.result()
self.msg = msg
But this callback not working as expected - self.msg do not updates. What am I doing wrong?
UPDATED
with additional code to show max full example:
class Queue(object):
def __init__(self, loop, maxsize: int):
self.instance = asyncio.Queue(loop = loop, maxsize = maxsize)
async def put(self, data):
await self.instance.put(data)
async def get(self):
data = await self.instance.get()
self.instance.task_done()
return data
#staticmethod
def get_instance():
return Queue(loop = asyncio.get_event_loop(), maxsize = 10)
Server class:
class BaseServer(object):
def __init__(self, host, port):
self.instance = asyncio.start_server(self.handle_connection, host = host, port = port)
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
pass
def get_instance(self):
return self.instance
#staticmethod
def create():
return BaseServer(None, None)
Next I'm running the servers:
loop.run_until_complete(asyncio.gather(server1.get_instance(), server2.get_instance()))
loop.run_forever()
In the handle_connection of server2 I'm calling queue.put(msg), in the handle_connection of server1 I'm registered queue.get() as task:
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
The process_queue method of server1:
def process_queue(self, future):
msg = future.result()
self.msg = msg
The handle_connection method of server1:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
while self.msg != SPECIAL_VALUE:
# doing something
Although task_queue is done, self.process_queue called, self.msg never updates.
Basically as you are using asynchronous structure, I think you can directly await the result:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
msg = await queue.get()
process_queue(msg) # change it to accept real value instead of a future.
# do something
I would like to make a ReconnectingClientFactory with asyncio. In particular to handle the case that the server is not available when the client is started in which case the ReconnectingClientFactory will keep trying. That is something that the asyncio.events.create_connection does not do.
Concretely:
The EchoClient example would be fine.
The crux is how the connection is made.
factory = EchoClientFactory('ws://127.0.0.1:5678')
connectWS(factory)
in the case of the twisted version with ReconnectingClientFactory.
Vs
factory = EchoClientFactory(u"ws://127.0.0.1:5678")
factory.protocol = SecureServerClientProtocol
loop = asyncio.get_event_loop()
# coro = loop.create_connection(factory, 'ws_server', 5678)
coro = loop.create_connection(factory, '127.0.0.1', 5678)
loop.run_until_complete(asyncio.wait([
alive(), coro
]))
loop.run_forever()
loop.close()
Or similar with the asycnio version.
The problem is that in the asyncio version the connection is established by asyncio.events.create_connection which simply fails if the server is not available.
How can I reconcile the two?
Many thanks
I think I get what you want. Here's the code and example based on asyncio TCP echo client protocol example.
import asyncio
import random
class ReconnectingTCPClientProtocol(asyncio.Protocol):
max_delay = 3600
initial_delay = 1.0
factor = 2.7182818284590451
jitter = 0.119626565582
max_retries = None
def __init__(self, *args, loop=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
self._args = args
self._kwargs = kwargs
self._retries = 0
self._delay = self.initial_delay
self._continue_trying = True
self._call_handle = None
self._connector = None
def connection_lost(self, exc):
if self._continue_trying:
self.retry()
def connection_failed(self, exc):
if self._continue_trying:
self.retry()
def retry(self):
if not self._continue_trying:
return
self._retries += 1
if self.max_retries is not None and (self._retries > self.max_retries):
return
self._delay = min(self._delay * self.factor, self.max_delay)
if self.jitter:
self._delay = random.normalvariate(self._delay,
self._delay * self.jitter)
self._call_handle = self._loop.call_later(self._delay, self.connect)
def connect(self):
if self._connector is None:
self._connector = self._loop.create_task(self._connect())
async def _connect(self):
try:
await self._loop.create_connection(lambda: self,
*self._args, **self._kwargs)
except Exception as exc:
self._loop.call_soon(self.connection_failed, exc)
finally:
self._connector = None
def stop_trying(self):
if self._call_handle:
self._call_handle.cancel()
self._call_handle = None
self._continue_trying = False
if self._connector is not None:
self._connector.cancel()
self._connector = None
if __name__ == '__main__':
class EchoClientProtocol(ReconnectingTCPClientProtocol):
def __init__(self, message, *args, **kwargs):
super().__init__(*args, **kwargs)
self.message = message
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self._loop.stop()
loop = asyncio.get_event_loop()
client = EchoClientProtocol('Hello, world!', '127.0.0.1', 8888, loop=loop)
client.connect()
loop.run_forever()
loop.close()
I am trying to make a program to make a lot of web-socket connections to the server I've created:
class WebSocketClient():
#asyncio.coroutine
def run(self):
print(self.client_id, 'Connecting')
ws = yield from aiohttp.ws_connect(self.url)
print(self.client_id, 'Connected')
print(self.client_id, 'Sending the message')
ws.send_str(self.make_new_message())
while not ws.closed:
msg = yield from ws.receive()
if msg.tp == aiohttp.MsgType.text:
print(self.client_id, 'Received the echo')
yield from ws.close()
break
print(self.client_id, 'Closed')
#asyncio.coroutine
def make_clients():
for client_id in range(args.clients):
yield from WebSocketClient(client_id, WS_CHANNEL_URL.format(client_id=client_id)).run()
event_loop.run_until_complete(make_clients())
The problem is that all the clients do their jobs one after another:
0 Connecting
0 Connected
0 Sending the message
0 Received the echo
0 Closed
1 Connecting
1 Connected
1 Sending the message
1 Received the echo
1 Closed
...
I've tried to use asyncio.wait, but all the clients start together. I want them to be created gradually and connected to the server immediately once each of them is created. At the same time continuing creating new clients.
What approach should I apply to accomplish this?
Using asyncio.wait is a good approach. You can combine it with asyncio.ensure_future and asyncio.sleep to create tasks gradually:
#asyncio.coroutine
def make_clients(nb_clients, delay):
futures = []
for client_id in range(nb_clients):
url = WS_CHANNEL_URL.format(client_id=client_id)
coro = WebSocketClient(client_id, url).run()
futures.append(asyncio.ensure_future(coro))
yield from asyncio.sleep(delay)
yield from asyncio.wait(futures)
EDIT: I implemented a FutureSet class that should do what you want. This set can be filled with futures and removes them automatically when they're done. It is also possible to wait for all the futures to complete.
class FutureSet:
def __init__(self, maxsize, *, loop=None):
self._set = set()
self._loop = loop
self._maxsize = maxsize
self._waiters = []
#asyncio.coroutine
def add(self, item):
if not asyncio.iscoroutine(item) and \
not isinstance(item, asyncio.Future):
raise ValueError('Expecting a coroutine or a Future')
if item in self._set:
return
while len(self._set) >= self._maxsize:
waiter = asyncio.Future(loop=self._loop)
self._waiters.append(waiter)
yield from waiter
item = asyncio.async(item, loop=self._loop)
self._set.add(item)
item.add_done_callback(self._remove)
def _remove(self, item):
if not item.done():
raise ValueError('Cannot remove a pending Future')
self._set.remove(item)
if self._waiters:
waiter = self._waiters.pop(0)
waiter.set_result(None)
#asyncio.coroutine
def wait(self):
return asyncio.wait(self._set)
Example:
#asyncio.coroutine
def make_clients(nb_clients, limit=0):
futures = FutureSet(maxsize=limit)
for client_id in range(nb_clients):
url = WS_CHANNEL_URL.format(client_id=client_id)
client = WebSocketClient(client_id, url)
yield from futures.add(client.run())
yield from futures.wait()