I would like to make a ReconnectingClientFactory with asyncio. In particular to handle the case that the server is not available when the client is started in which case the ReconnectingClientFactory will keep trying. That is something that the asyncio.events.create_connection does not do.
Concretely:
The EchoClient example would be fine.
The crux is how the connection is made.
factory = EchoClientFactory('ws://127.0.0.1:5678')
connectWS(factory)
in the case of the twisted version with ReconnectingClientFactory.
Vs
factory = EchoClientFactory(u"ws://127.0.0.1:5678")
factory.protocol = SecureServerClientProtocol
loop = asyncio.get_event_loop()
# coro = loop.create_connection(factory, 'ws_server', 5678)
coro = loop.create_connection(factory, '127.0.0.1', 5678)
loop.run_until_complete(asyncio.wait([
alive(), coro
]))
loop.run_forever()
loop.close()
Or similar with the asycnio version.
The problem is that in the asyncio version the connection is established by asyncio.events.create_connection which simply fails if the server is not available.
How can I reconcile the two?
Many thanks
I think I get what you want. Here's the code and example based on asyncio TCP echo client protocol example.
import asyncio
import random
class ReconnectingTCPClientProtocol(asyncio.Protocol):
max_delay = 3600
initial_delay = 1.0
factor = 2.7182818284590451
jitter = 0.119626565582
max_retries = None
def __init__(self, *args, loop=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop()
self._loop = loop
self._args = args
self._kwargs = kwargs
self._retries = 0
self._delay = self.initial_delay
self._continue_trying = True
self._call_handle = None
self._connector = None
def connection_lost(self, exc):
if self._continue_trying:
self.retry()
def connection_failed(self, exc):
if self._continue_trying:
self.retry()
def retry(self):
if not self._continue_trying:
return
self._retries += 1
if self.max_retries is not None and (self._retries > self.max_retries):
return
self._delay = min(self._delay * self.factor, self.max_delay)
if self.jitter:
self._delay = random.normalvariate(self._delay,
self._delay * self.jitter)
self._call_handle = self._loop.call_later(self._delay, self.connect)
def connect(self):
if self._connector is None:
self._connector = self._loop.create_task(self._connect())
async def _connect(self):
try:
await self._loop.create_connection(lambda: self,
*self._args, **self._kwargs)
except Exception as exc:
self._loop.call_soon(self.connection_failed, exc)
finally:
self._connector = None
def stop_trying(self):
if self._call_handle:
self._call_handle.cancel()
self._call_handle = None
self._continue_trying = False
if self._connector is not None:
self._connector.cancel()
self._connector = None
if __name__ == '__main__':
class EchoClientProtocol(ReconnectingTCPClientProtocol):
def __init__(self, message, *args, **kwargs):
super().__init__(*args, **kwargs)
self.message = message
def connection_made(self, transport):
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self._loop.stop()
loop = asyncio.get_event_loop()
client = EchoClientProtocol('Hello, world!', '127.0.0.1', 8888, loop=loop)
client.connect()
loop.run_forever()
loop.close()
Related
I'm trying to write code that enables using asyncpg from mostly sync code (to avoid duplication).
For some very strange reason, the coroutine Database.test() will execute and return in my worker eventloop/thread. The future seems to work correctly. But connecting to a database with asyncpg will just hang. Any clue as to why?
Also, maybe I should use asyncio.run() instead.
from threading import Thread
import asyncio
import asyncpg
class AsyncioWorkerThread(Thread):
def __init__(self, *args, daemon=True, loop=None, **kwargs):
super().__init__(*args, daemon=daemon, **kwargs)
self.loop = loop or asyncio.new_event_loop()
self.running = False
def run(self):
self.running = True
self.loop.run_forever()
def submit(self, coro):
fut = asyncio.run_coroutine_threadsafe(coro, loop=self.loop)
return fut.result()
def stop(self):
self.loop.call_soon_threadsafe(self.loop.stop)
self.join()
self.running = False
class Database:
async def test(self):
print('In test')
await asyncio.sleep(5)
async def connect(self):
# Put in your db credentials here
# pg_user = ''
# pg _password = ''
# pg_host = ""
# pg_port = 20
# pg_db
connection_uri = f'postgres://{pg_user}:{pg_password}#{pg_host}:{pg_port}/{pg_db}'
self.connection_pool = await asyncpg.create_pool(
connection_uri, min_size=5, max_size=10)
if __name__ == "__main__":
db = Database()
worker = AsyncioWorkerThread()
worker.start()
worker.submit(db.test()) # Works future returns correctly
worker.submit(db.connect()) # Hangs, thread never manages to acquire
I have an echo client based on the python documentation as follow:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
self.loop = asyncio.get_running_loop()
def connection_made(self, transport):
self.transport = transport
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def write_to_device(self):
self.transport.write(self.message.encode())
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
self.loop.call_later(1.0 , self.write_to_device)
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def main():
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 8888)
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
Everytime it receives a message from server, it sends again the message after 1 second. My question is: How can I change the message to send once the connection is made?
Oki, just simply modifying the protocol instance as suggested works... here is a simple proof:
import asyncio
class EchoClientProtocol(asyncio.Protocol):
def __init__(self, message, on_con_lost):
self.message = message
self.on_con_lost = on_con_lost
self.loop = asyncio.get_running_loop()
def connection_made(self, transport):
self.transport = transport
transport.write(self.message.encode())
print('Data sent: {!r}'.format(self.message))
def write_to_device(self):
print('Data sent: {!r}'.format(self.message))
self.transport.write(self.message.encode())
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
self.loop.call_later(1, self.write_to_device)
def connection_lost(self, exc):
print('The server closed the connection')
self.on_con_lost.set_result(True)
async def changeMessage(message, protocol):
await asyncio.sleep(10)
print('Changing message')
protocol.message = message
async def main():
# Get a reference to the event loop as we plan to use
# low-level APIs.
loop = asyncio.get_running_loop()
on_con_lost = loop.create_future()
message = 'Hello World!'
transport, protocol = await loop.create_connection(
lambda: EchoClientProtocol(message, on_con_lost),
'127.0.0.1', 65432)
await changeMessage('Message is now changed', protocol)
# Wait until the protocol signals that the connection
# is lost and close the transport.
try:
await on_con_lost
finally:
transport.close()
asyncio.run(main())
I have two servers, created with asyncio.start_server:
asyncio.start_server(self.handle_connection, host = host, port = port) and running in one loop:
loop.run_until_complete(asyncio.gather(server1, server2))
loop.run_forever()
I'm using asyncio.Queue to communicate between servers. Messages from Server2, added via queue.put(msg) successfully receives by queue.get() in Server1. I'm running queue.get() by asyncio.ensure_future and using as callback for
add_done_callback method from Server1:
def callback(self, future):
msg = future.result()
self.msg = msg
But this callback not working as expected - self.msg do not updates. What am I doing wrong?
UPDATED
with additional code to show max full example:
class Queue(object):
def __init__(self, loop, maxsize: int):
self.instance = asyncio.Queue(loop = loop, maxsize = maxsize)
async def put(self, data):
await self.instance.put(data)
async def get(self):
data = await self.instance.get()
self.instance.task_done()
return data
#staticmethod
def get_instance():
return Queue(loop = asyncio.get_event_loop(), maxsize = 10)
Server class:
class BaseServer(object):
def __init__(self, host, port):
self.instance = asyncio.start_server(self.handle_connection, host = host, port = port)
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
pass
def get_instance(self):
return self.instance
#staticmethod
def create():
return BaseServer(None, None)
Next I'm running the servers:
loop.run_until_complete(asyncio.gather(server1.get_instance(), server2.get_instance()))
loop.run_forever()
In the handle_connection of server2 I'm calling queue.put(msg), in the handle_connection of server1 I'm registered queue.get() as task:
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
The process_queue method of server1:
def process_queue(self, future):
msg = future.result()
self.msg = msg
The handle_connection method of server1:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
while self.msg != SPECIAL_VALUE:
# doing something
Although task_queue is done, self.process_queue called, self.msg never updates.
Basically as you are using asynchronous structure, I think you can directly await the result:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
msg = await queue.get()
process_queue(msg) # change it to accept real value instead of a future.
# do something
I'm on Python 3.4 making a practice app to learn Python's asyncio module.
I've got several modules in my application:
app.py
import asyncio
import logging
class Client(asyncio.Protocol):
def __init__(self, loop):
self.loop = loop
def connection_made(self, transport):
print('Connection with server established')
self.transport = transport
def data_received(self, data):
data = data.decode()
print('Received: ', data)
if not int(data):
print('Stopping loop')
self.loop.stop()
else:
message = 'remove one'.encode()
self.transport.write(message)
print('Sent:', message)
def connection_lost(self, exc):
print('Connection with server lost.')
self.loop.stop()
loop = asyncio.get_event_loop()
fn = loop.create_connection(
lambda: Client(loop), '127.0.0.1', 9999
)
logging.basicConfig(level=logging.DEBUG)
client = loop.run_until_complete(fn)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
print('Loop ended')
loop.close()
counter.py
import asyncio
class Counter:
def __init__(self, maxcount):
self.maxcount = maxcount
#asyncio.coroutine
def count(self):
yield from asyncio.sleep(self.maxcount)
print('Done counting')
serie.py
import asyncio
class Serie:
def __init__(self, loop, items=None):
self.loop = loop
self.items = items or []
#asyncio.coroutine
def remove_one(self, counter):
if len(self.items) is not 0:
yield from counter.count()
item = self.items.pop(0)
print('Removed', item)
else:
print('Serie is empty')
#asyncio.coroutine
def start_removing(self, counter):
while self.items:
yield from self.remove_one(counter)
print('Serie.start_removing() has finished')
mission_control.py
import asyncio
import logging
from counter import Counter
from serie import Serie
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
yield from self.serie.remove_one()
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
else:
print('Done')
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
counter = Counter(2)
planets = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
fn = loop.create_server(
lambda: MissionControl(loop, counter, planets), '127.0.0.1', 9999
)
server = loop.run_until_complete(fn)
print('Server started')
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.stop()
loop.close()
You can also find the source at this Github gist.
In mission_control.py, the method data_received() appears not to be called when the client (app.py) sends data via its self.transport property.
Where is the implementation error and how can I fix it?
The problem is that data_received is not (and cannot be) a coroutine, but you're using yield from inside it. Internally asyncio is just calling self.data_received(data) without any yield from call, which means the body of the method isn't being executed at all - instead a generator object is immediately returned. You need to refactor your implementation to not use yield from, which requires using callbacks instead:
class MissionControl(asyncio.Protocol):
def __init__(self, loop, counter, serie):
self.loop = loop
self.counter = counter
self.serie = serie
def connection_made(self, transport):
print('Connection established with', transport.get_extra_info('peername'))
self.transport = transport
self.transport.write(str(len(self.serie.items)).encode())
def data_received(self, data):
data = data.decode()
print('Received:', data)
if data == 'remove one':
fut = asyncio.async(self.serie.remove_one(self.counter))
fut.add_done_callback(self.on_removed_one)
else:
print('Done')
def on_removed_one(self, result):
print('Removed one: {}'.format(self.serie.items))
self.transport.write(str(len(self.serie.items)).encode())
def connection_lost(self, exc):
print('Connection with {} ended'.format(self.transport.get_extra_info('peername')))
Another option would be to use the asyncio Streams API instead of asyncio.Protocol, which will allow you to use coroutines:
import asyncio
import logging
from counter import Counter
from serie import Serie
#asyncio.coroutine
def mission_control(reader, writer):
counter = Counter(2)
serie = Serie(loop, ['Mercúrio', 'Vênus', 'Terra', 'Marte',
'Júpiter', 'Saturno', 'Urano', 'Netuno'])
writer.write(str(len(serie.items)).encode())
while True:
data = (yield from reader.read(100)).decode()
print('Received:', data)
if data == 'remove one':
result = yield from serie.remove_one(counter)
else:
print('Done')
return
print('Removed one: {}'.format(serie.items))
writer.write(str(len(serie.items)).encode())
logging.basicConfig(level=logging.DEBUG)
loop = asyncio.get_event_loop()
coro = asyncio.start_server(mission_control, '127.0.0.1', 9999, loop=loop)
server = loop.run_until_complete(coro)
# The rest is the same
I have problem figuring out how to use zmq with ioloops and multithreading. Whatever I'm doing I have some exceptions.
__author__ = 'michael'
class ZmqLoopRunner(Thread):
def __init__(self, callback):
super(ZmqLoopRunner, self).__init__()
self.loop = IOLoop.current()
self.callback = callback
def run(self):
self.loop.start()
print('loop have been stopped')
self.callback()
def stop(self):
self.loop.stop()
class BaseZmqNode():
__metaclass__ = ABCMeta
def __init__(self, host, port, bind, hwm):
self.node = self.create_node()
self.node.host = host
self.port = port
self.context = zmq.Context().instance()
self.socket = self.create_socket()
if bind:
self.socket.bind(self.build_address(host, port))
else:
self.socket.connect(self.build_address(host, port))
self.set_hwm(hwm)
def set_hwm(self, hwm):
self.socket.set_hwm(hwm)
def send_multipart(self, message):
self.socket.send_multipart(message)
def send_json(self, json):
self.socket.send_json(json)
def create_node(self):
return BaseMessagingNode
def close(self):
self.socket.close()
#staticmethod
def build_address(host, port):
strings = [host, ':', str(port)]
return ''.join(strings)
#abstractmethod
def create_socket(self):
pass
class BaseZmqReceiver(BaseZmqNode):
__metaclass__ = ABCMeta
def __init__(self, host, port, hwm, bind, on_receive_callback):
super(BaseZmqReceiver, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
self.node.on_message_callback = on_receive_callback
self.stream = ZMQStream(self.socket)
self.stream.on_recv(self.on_message_received)
self.runner = ZmqLoopRunner(self.stream.close)
self.runner.start()
def on_message_received(self, message):
return self.node.on_message_callback(message)
def create_node(self):
return ReceivingNode(None, None)
def close(self):
# super(BaseZmqReceiver, self).close()
self.runner.stop()
# self.socket.close()
Here is how my code looks right now. I have exceptions in test 'Address already in use' when running my tests.
Here is stacktrace
Traceback (most recent call last):
File "/opt/leos/code/messaging_system/tests/ZmqTest.py", line 51, in test_send_json
publisher = ZmqPublisher('tcp://*', 6000)
File "/opt/leos/code/messaging_system/zeromq/ZmqPublisher.py", line 11, in __init__
super(ZmqPublisher, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
File "/opt/leos/code/messaging_system/zeromq/BaseZmqNode.py", line 18, in __init__
self.socket.bind(self.build_address(host, port))
File "socket.pyx", line 434, in zmq.backend.cython.socket.Socket.bind (zmq/backend/cython/socket.c:3928)
File "checkrc.pxd", line 21, in zmq.backend.cython.checkrc._check_rc (zmq/backend/cython/socket.c:6058)
ZMQError: Address already in use
class ZmqTest(AbstractMessagingTest):
def setUp(self):
super(ZmqTest, self).setUp()
self.multipart_messages = self.create_multipart_messages(10)
def tearDown(self):
super(ZmqTest, self).tearDown()
def test_request_reply(self):
requester = ZmqReq(host='tcp://localhost', port=6000)
self.request = 'Hello'
self.reply = 'World!'
replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
self.assertEqual(self.reply, requester.execute(request=self.request))
# requester.close()
replier.close()
requester.close()
def test_address_creation(self):
full_address = "tcp://localhost:5559"
self.assertEqual(full_address, ZmqSubscriber.build_address("tcp://localhost", 5559))
self.assertEqual('tcp://*:6000', ZmqPublisher.build_address("tcp://*", 6000))
def test_publisher_subscriber(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_message)
self.send_messages(publisher, wait=False)
sleep(0.5)
self.assertSequenceEqual(self.test_messages, self.received_messages)
publisher.close()
subscriber.close()
def handle_message(self, message):
self.base_handle_message(message[0])
def test_send_json(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_json_message)
md = {'1' : 1}
publisher.send_json(md)
publisher.close()
subscriber.close()
def create_multipart_messages(self, size):
messages = []
for i in range(size):
messages.append(['Multipart test message', str(i)])
return messages
def send_multipart_messages(self, sender):
for message in self.multipart_messages:
sender.send_multipart(message)
def test_multipart_messages(self):
publisher = ZmqPublisher('tcp://*', 6000)
subscriber = ZmqSubscriber('tcp://localhost', 6000, self.base_handle_message)
self.send_multipart_messages(publisher)
sleep(0.5)
self.assertSequenceEqual(self.multipart_messages, self.received_messages)
publisher.close()
subscriber.close()
def test_push_pull_multipart(self):
ventilator = ZmqPush('tcp://*', 6000)
worker = ZmqPull('tcp://localhost', 6000, self.base_handle_message)
self.send_multipart_messages(ventilator)
sleep(0.5)
self.assertSequenceEqual(self.multipart_messages, self.received_messages)
ventilator.close()
worker.close()
def handle_json_message(self, json):
print(str(json))
def test_push_pull(self):
ventilator = ZmqPush('tcp://*', 6000)
worker = ZmqPull('tcp://localhost', 6000, self.handle_message)
self.send_messages(ventilator, wait=False)
sleep(0.5)
self.assertSequenceEqual(self.test_messages, self.received_messages)
ventilator.close()
worker.close()
def on_request_received(self, message):
if message[0] == self.request:
return self.reply
else:
return 'ERROR'
And I've tried many variants of this code. Like as you can see right now I'm trying to close the stream after the loop.start() method returned. I've tried to close the stream after the stop method and it just don't work.
So you're getting the error that the socket address is already open. This is probably because you have run up the program, an exception was thrown and you haven't closed down the socket.
I'd suggest some try, except and finally blocks:
try:
requester = ZmqReq(host='tcp://localhost', port=6000)
self.request = 'Hello'
self.reply = 'World!'
replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
self.assertEqual(self.reply, requester.execute(request=self.request))
except Exception as e:
# You can catch exceptions here
pass
finally:
# Once the code completes or exceptions are thrown, clean up
replier.close()
requester.close()