So here's the basic code (sorry it's long)
import argparse
import asyncio
from contextvars import ContextVar
import sys
# This thing is the offender
message_var = ContextVar("message")
class ServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info("peername")
print("Server: Connection from {}".format(peername))
self.transport = transport
def data_received(self, data):
message = data.decode()
print("Server: Data received: {!r}".format(message))
print("Server: Send: {!r}".format(message))
self.transport.write(data)
print("Server: Close the client socket")
self.transport.close()
class ClientProtocol(asyncio.Protocol):
def __init__(self, on_conn_lost):
self.on_conn_lost = on_conn_lost
self.transport = None
self.is_connected: bool = False
def connection_made(self, transport):
self.transport = transport
self.is_connected = True
def data_received(self, data):
# reading back supposed contextvar
message = message_var.get()
print(f"{message} : {data.decode()}")
def connection_lost(self, exc):
print("The server closed the connection")
self.is_connected = False
self.on_conn_lost.set_result(True)
def send(self, message: str):
# Setting context var
message_var.set(message)
if self.transport:
self.transport.write(message.encode())
def close(self):
self.transport.close()
self.is_connected = False
if not self.on_conn_lost.done():
self.on_conn_lost.set_result(True)
async def get_input(client: ClientProtocol):
loop = asyncio.get_running_loop()
while client.is_connected:
message = await loop.run_in_executor(None, input, ">>>")
if message == "q":
client.close()
return
client.send(message)
async def main(args):
host = "127.0.0.1"
port = 5001
loop = asyncio.get_running_loop()
if args.server:
server = await loop.create_server(lambda: ServerProtocol(), host, port)
async with server:
await server.serve_forever()
return
on_conn_lost = loop.create_future()
client = ClientProtocol(on_conn_lost)
await loop.create_connection(lambda: client, host, port)
await get_input(client)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"--server", "-s", default=False, action="store_true", help="Start server"
)
arguments = parser.parse_args(sys.argv[1:])
asyncio.run(main(args=arguments))
This crashes with the following exception:
Exception in callback _ProactorReadPipeTransport._loop_reading(<_OverlappedF...shed result=4>)
handle: <Handle _ProactorReadPipeTransport._loop_reading(<_OverlappedF...shed result=4>)>
Traceback (most recent call last):
File "C:\Users\brent\AppData\Local\Programs\Python\Python310\lib\asyncio\events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "C:\Users\brent\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 320, in _loop_reading
self._data_received(data, length)
File "C:\Users\brent\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 270, in _data_received
self._protocol.data_received(data)
File "E:\Development\Python\ibcs2023\_prep\experimental\asyncio_context.py", line 40, in data_received
message = message_var.get()
LookupError: <ContextVar name='message' at 0x0000023F30A54FE0>
The server closed the connection
Why does calling message = message_var.get() cause a crash? Why can't Python find the context var? Why is data_received not in the same context as send? How can I keep them in the same context?
I'm working on a larger project with the main branch of Textual and it uses a contextvar that loses context every time a message is received using a modified version of the code above.
Keeping a separated "context" for each task is exactly what contextvars are about. You could only assert that the send and data_received methods were called within the same context if you had control over the "uperlying" (as opposed to 'underlying') driver of your Protocol class - that is not the case, and both are called in different contexts. I mean, the answer to "How can I keep them in the same context?" is: you can't unless you write your own implementation of the code which makes this work inside asyncio.
There is no way you can keep track of metadata from a message, and retrieve this metadata on getting the reply, unless there is a marker on the message itself, that will survive the round-trip. That is: your networking/communication protocol itself have to spec a way to identify messages It might be as simple as a sequential integer number prefixing every string, for example - or, in this case where you simply echo the message back, it could be the message itself. Once you have that, a simple dictionary having these message IDs as keys, will work for what you seem to intend in this example.
Related
I'm looking to have many agents connect to different sockets of my local server so they can independently read and send to the server. So far I have this:
import json
import asyncio
class Agent():
def __init__(self, user, pw):
self.user = user
self.pw = pw
# Store host and port
self.host, self.port = "localhost", 12300
self.connect()
async def connect(self):
# Create asynchronous socket reader and writer
self.reader, self.writer = await asyncio.open_connection(self.host,
self.port)
request = <some_json>
# Create and send authentication request
self.writer.write(request)
await self.writer.drain()
response = await self.reader.read(1000)
response = json.loads(response.decode().split("\0")[0])
if response["content"]["result"] == "ok":
print("Connection Succesful")
else:
print("Connection Failed")
await self.listen()
async def receive_msg(self):
"""
Waits for a message from the server and returns it.
"""
msg = await self.reader.readuntil(b"\0")
msg = msg.decode().split("\0")[0]
# In case a message is received, parse it into a dictionary.
if len(msg) > 1:
return json.loads(msg)
else:
print("Retry receiving message...")
return self.receive_msg()
async def listen(self):
"""
Listens for output from server and writes if anything is received
"""
while True:
msg = await self.receive_msg()
print("Message received", self.user)
a_list = []
loop = asyncio.get_event_loop()
for i in range(15):
a_list.append(Agent(f"agentA{i}", "1").connect())
loop.run_until_complete(asyncio.gather(*a_list))
Because of the way asyncio works I think this is the only way to run this asynchrously. But I would like to be able to make the __init__ run asynchrously somehow instead of having to throw the connect function into the loop if that is possible. What I essentially would like to do is this:
a_list = []
loop = asyncio.get_event_loop()
for i in range(15):
a_list.append(Agent(f"agentA{i}", "1"))
loop.run_until_complete(asyncio.gather(*a_list))
I think that that makes more sense, but I can't figure out how to do it. Am I thinking wrongly, or is there a better way to do this?
You can't make __init__ async, but you can make Agent instances awaitable. To do so, define the __await__ magic method:
class Agent:
def __init__(self, user, pw):
self.user = user
self.pw = pw
self.host, self.port = "localhost", 12300
def __await__(self):
yield from self.connect().__await__()
This has the best of both worlds: your __init__ function remains sync, and yet Agent instances are valid arguments to functions like asyncio.gather().
I'm using the Autobahn asyncio system (to talk the Websocket WAMP protocol), which works fine and I can handle incoming RPC calls and pubsub.
My problem is I now have to connect TCP sockets and send information over these sockets as soon as an RPC call comes in through the Autobahn part.
The autobahn part works fine like this :
from autobahn.asyncio.component import Component, run
from asyncio import sleep
from autobahn.asyncio.wamp import ApplicationSession, ApplicationRunner
#comp.on_join
async def joined(session, details):
print("Connected to websocket")
def on_message(msg):
msg = json.loads(msg)
print(msg)
def some_rpc(with_data):
print("Doing something with the data")
return json.dumps({'status': 'OK'})
try:
session.subscribe(on_message, u'some_pubsub_topic')
session.register(some_rpc, u'some_rpc_call')
print("RPC and Pubsub initialized")
except Exception as e:
print("could not subscribe to topic: {0}".format(e))
if __name__ == "__main__":
run([comp])
However now I need to be able to connect to multiple regular TCP sockets :
class SocketClient(asyncio.Protocol):
def __init__(self, loop):
self.data = b''
self.loop = loop
def connection_made(self, transport):
self.transport = transport
print('connected')
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def send(self, data):
self.transport.write(data)
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
loop = asyncio.get_event_loop()
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
loop.run_until_complete(c)
loop.run_forever()
loop.close()
The problem is that, when I combine both and do this :
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
It barks at me and tells me :
StopIteration
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/usr/lib/python3.5/site-packages/autobahn/wamp/websocket.py", line
95, in onMessage
self._session.onMessage(msg) File "/usr/lib/python3.5/site-packages/autobahn/wamp/protocol.py", line
894, in onMessage
on_reply = txaio.as_future(endpoint.fn, *invoke_args, **invoke_kwargs) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 400, in
as_future
return create_future_error(create_failure()) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 393, in
create_future_error
reject(f, error) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 462, in reject
future.set_exception(error.value) File "/usr/lib64/python3.5/asyncio/futures.py", line 365, in set_exception
raise TypeError("StopIteration interacts badly with generators " TypeError: StopIteration interacts badly with generators and cannot be
raised into a Future
Does anyone have any idea on how to call the send function from within the RPC call function ?
In this code:
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
# [...]
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
create_connection is a coroutine function, so c contains a coroutine object. Such object does have a send method, but one that is entirely unrelated to sending things over the network. After calling create_connection, you probably want to get the resulting transport with something like:
transport, ignore = loop.run_until_complete(c)
and then use transport.write(), not c.send().
I am trying to add two coroutines to asyncio loop and getting an error:
RuntimeError: This event loop is already running
My objective is to communicate to a server (that I have no control of). This server expects an initial connection from the client. The server then provided a port to the client on this connection. The client has to use this port to create a second connection. This second connection is used by the server to send unsolicited messages to the client. The first connection remains up throughout for other two-way communications.
To recreate this scenario, I have some code that reproduces the error:
class Connection():
def __init__(self, ip, port, ioloop):
self.ip = ip
self.port = port
self.ioloop = ioloop
self.reader, self.writer = None, None
self.protocol = None
self.fileno = None
async def __aenter__(self):
# Applicable when doing 'with Connection(...'
log.info("Entering and Creating Connection")
self.reader, self.writer = (
await asyncio.open_connection(self.ip, self.port, loop=self.ioloop)
)
self.protocol = self.writer.transport.get_protocol()
self.fileno = self.writer.transport.get_extra_info('socket').fileno()
log.info(f"Created connection {self}")
return self
async def __aexit__(self, *args):
# Applicable when doing 'with Connection(...'
log.info(f"Exiting and Destroying Connection {self}")
if self.writer:
self.writer.close()
def __await__(self):
# Applicable when doing 'await Connection(...'
return self.__aenter__().__await__()
def __repr__(self):
return f"[Connection {self.ip}:{self.port}, {self.protocol}, fd={self.fileno}]"
async def send_recv_message(self, message):
log.debug(f"send: '{message}'")
self.writer.write(message.encode())
await self.writer.drain()
log.debug("awaiting data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv: '{data}'")
return data
class ServerConnection(Connection):
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
self.ioloop.run_until_complete(event_connection.recv_message())
class EventConnection(Connection):
async def recv_message(self):
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv only: '{data}'")
return data
async def main(loop):
client1 = await ServerConnection('127.0.0.1', 8888, loop)
await client1.setup_connection()
await client1.send_recv_message("Hello1")
await client1.send_recv_message("Hello2")
await asyncio.sleep(5)
if __name__ == '__main__':
#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
The error occurs in ServerConnection.setup_connection() method where run_until_complete is being called.
I am probably doing something wrong due to lack of understanding asyncio. Basically, how do I setup a secondary connection which will get event notifications (unsolicited) while setting up the first connection?
Thanks.
Followup
Since the code is very similar (a few changes to add more functionality to it), I hope it's not bad etiquette to followup to the original post as the resulting error is still the same.
The new issue is that when it receives the unsolicited message (which is received by EventConnection), the recv_message calls process_data method. I would like to make process_data be a future so that recv_message completes (ioloop should stop). The ensure_future would then pick it up and continue running again to use ServerConnection to do a request/response to the server. Before it does that though, it has to go to some user code (represented by external_command() and from whom I would prefer to hide the async stuff). This would make it synchronous again. Hence, once they've done what they need to, they should call execute_command on ServerConnection, which then kicks off the loop again.
The problem is, my expectation for using ensure_future didn't pan out as it seems the loop didn't stop from running. Hence, when the code execution reaches execute_command which does the run_until_complete, an exception with the error "This event loop is already running" occurs.
I have two questions:
How can I make it so that the ioloop can stop after process_data is
placed into ensure_future, and subsequently be able to run it again
in execute_command?
Once recv_message has received something, how can we make it so that
it can receive more unsolicited data? Is it enough/safe to just use
ensure_future to call itself again?
Here's the example code that simulates this issue.
client1 = None
class ServerConnection(Connection):
connection_type = 'Server Connection'
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
asyncio.ensure_future(event_connection.recv_message())
async def _execute_command(self, data):
return await self.send_recv_message(data)
def execute_command(self, data):
response_str = self.ioloop.run_until_complete(self._execute_command(data))
print(f"exec cmd response_str: {response_str}")
def external_command(self, data):
self.execute_command(data)
class EventConnection(Connection):
connection_type = 'Event Connection'
async def recv_message(self):
global client1
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv-only: '{data}'")
asyncio.ensure_future(self.process_data(data))
asyncio.ensure_future(self.recv_message())
async def process_data(self, data):
global client1
await client1.external_command(data)
async def main(ioloop):
global client1
client1 = await ServerConnection('127.0.0.1', 8888, ioloop)
await client1.setup_connection()
print(f"after connection setup loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello1")
print(f"after Hello1 loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello2")
print(f"after Hello2 loop running is {ioloop.is_running()}")
while True:
print(f"inside while loop running is {ioloop.is_running()}")
t = 10
print(f"asyncio sleep {t} sec")
await asyncio.sleep(t)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
Try replacing:
self.ioloop.run_until_complete
With
await
I'm working on a project which uses python asyncio socket server. The problem is that the implementation of the server doesn't call .close() on the transport when the server stops. This seems to leave clients connected and causes crashes in other parts of the code.
Python documents say that transports need to be closed explicitly, but in this project I don't know where I can close them because there is no reference to the transports that are created for each client.
https://docs.python.org/3/library/asyncio-dev.html#close-transports-and-event-loops
Here is the code:
"""
Socket server forwarding request to internal server
"""
import logging
try:
# we prefer to use bundles asyncio version, otherwise fallback to trollius
import asyncio
except ImportError:
import trollius as asyncio
from opcua import ua
from opcua.server.uaprocessor import UaProcessor
logger = logging.getLogger(__name__)
class BinaryServer(object):
def __init__(self, internal_server, hostname, port):
self.logger = logging.getLogger(__name__)
self.hostname = hostname
self.port = port
self.iserver = internal_server
self.loop = internal_server.loop
self._server = None
self._policies = []
def set_policies(self, policies):
self._policies = policies
def start(self):
class OPCUAProtocol(asyncio.Protocol):
"""
instanciated for every connection
defined as internal class since it needs access
to the internal server object
FIXME: find another solution
"""
iserver = self.iserver
loop = self.loop
logger = self.logger
policies = self._policies
def connection_made(self, transport):
self.peername = transport.get_extra_info('peername')
self.logger.info('New connection from %s', self.peername)
self.transport = transport
self.processor = UaProcessor(self.iserver, self.transport)
self.processor.set_policies(self.policies)
self.data = b""
def connection_lost(self, ex):
self.logger.info('Lost connection from %s, %s', self.peername, ex)
self.transport.close()
self.processor.close()
def data_received(self, data):
logger.debug("received %s bytes from socket", len(data))
if self.data:
data = self.data + data
self.data = b""
self._process_data(data)
def _process_data(self, data):
buf = ua.utils.Buffer(data)
while True:
try:
backup_buf = buf.copy()
try:
hdr = ua.Header.from_string(buf)
except ua.utils.NotEnoughData:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
if len(buf) < hdr.body_size:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
ret = self.processor.process(hdr, buf)
if not ret:
logger.info("processor returned False, we close connection from %s", self.peername)
self.transport.close()
return
if len(buf) == 0:
return
except Exception:
logger.exception("Exception raised while parsing message from client, closing")
self.transport.close()
break
coro = self.loop.create_server(OPCUAProtocol, self.hostname, self.port)
self._server = self.loop.run_coro_and_wait(coro)
print('Listening on {}:{}'.format(self.hostname, self.port))
def stop(self):
self.logger.info("Closing asyncio socket server")
self.loop.call_soon(self._server.close)
self.loop.run_coro_and_wait(self._server.wait_closed())
As you can see when we call stop() on this server class the asyncio server calls it's close method. However if clients are connected the created transports never get closed.
The project repository is here https://github.com/FreeOpcUa/python-opcua/ , you can take a look at Issue 137.
What is the correct way to close the transport object?
I solve this by applying this approach:
#self.OPCUAServer - this is my opcua server
nodes = []
nodes.append(self.OPCUAServer.get_node("ns=0; s=Measurements")) #Adding two root nodes
nodes.append(self.OPCUAServer.get_node("ns=1; s=Calibrations")) #to the list
self.OPCUAServer.delete_nodes(nodes, True) # Recursively call delete_nodes with this list
self.OPCUAServer.stop()
I'm trying to close the transport right after sending the UDP packet and I'm getting an Exception in callback _SelectorDatagramTransport._read_ready()
import asyncio
class MyProtocol:
def __init__(self, message, loop):
self.message = message
self.loop = loop
self.transport = None
def connection_made(self, transport):
self.transport = transport
print("Send:", self.message)
self.transport.sendto(self.message.encode())
self.transport.close() # <----------
def error_received(self, exc):
print('Error received', exc)
def connection_lost(self, exc):
print("Socket closed, stop the event loop")
self.loop.stop()
loop = asyncio.get_event_loop()
message = "hello"
connect = loop.create_datagram_endpoint(lambda: MyProtocol(message, loop), remote_addr=('127.0.0.1', 2222))
transport, protocol = loop.run_until_complete(connect)
loop.run_forever()
The full stack trace that I get is while running the snippet above in CPython 3.5.1 is:
Socket closed, stop the event loop
Exception in callback _SelectorDatagramTransport._read_ready()
handle: <Handle _SelectorDatagramTransport._read_ready()>
Traceback (most recent call last):
File "/home/ecerulm/.pyenv/versions/3.5.1/lib/python3.5/asyncio/selector_events.py", line 1002, in _read_ready
data, addr = self._sock.recvfrom(self.max_size)
AttributeError: 'NoneType' object has no attribute 'recvfrom'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ecerulm/.pyenv/versions/3.5.1/lib/python3.5/asyncio/events.py", line 125, in _run
self._callback(*self._args)
File "/home/ecerulm/.pyenv/versions/3.5.1/lib/python3.5/asyncio/selector_events.py", line 1008, in _read_ready
self._fatal_error(exc, 'Fatal read error on datagram transport')
File "/home/ecerulm/.pyenv/versions/3.5.1/lib/python3.5/asyncio/selector_events.py", line 587, in _fatal_error
self._loop.call_exception_handler({
AttributeError: 'NoneType' object has no attribute 'call_exception_handler'
I believe the exception is only generated if the UDP packet is actively refused, with an ICMP Destination Unreachable (which I'm not interested in).
So the question is what is the right way of doing this. I'm not interested in this connection anymore after sending so I want to get rid of the transport as soon as possible. The documentation for DatagramTransport.sendto() just says that the methods doesn't block. But how do I know when the sending is completed? (And by complete I mean when is handed over to the OS, not delivered to the remote).
Is there any other asyncio coroutine to send an UDP packet asynchronously and simple await (maybe even skipping the whole create_datagram_endpoint) ?
Is there any other asyncio coroutine to send an UDP packet asynchronously and simple await?
I would, base on DatagramTransport source, wrap it in Future to be yieldable/awaitable. It will raise exception on error and return True on success. The example PoC code:
import asyncio
import socket
class UDPClient():
def __init__(self, host, port, loop=None):
self._loop = asyncio.get_event_loop() if loop is None else loop
self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self._sock.setblocking(False)
self._addr = (host, port)
self._future = None
self._data = None
def sendto(self, data):
self._future = asyncio.Future(loop=self._loop)
self.data = data if isinstance(data, bytes) else str(data).encode('utf-8')
loop.add_writer(self._sock.fileno(), self._sendto)
return self._future
def _sendto(self):
try:
self._sock.sendto(self.data, self._addr)
except (BlockingIOError, InterruptedError):
return
except OSError as exc:
self.abort(exc)
except Exception as exc:
self.abort(exc)
else:
self.close()
self._future.set_result(True)
def abort(self, exc):
self.close()
self._future.set_exception(exc)
def close(self):
self._loop.remove_writer(self._sock.fileno())
self._sock.close()
Than simple example would look like:
#asyncio.coroutine
def test():
yield from UDPClient('127.0.0.1', 1234).sendto('ok')
# or 3.5+ syntax
# async def test():
# await UDPClient('127.0.0.1', 1234).sendto('ok')
loop = asyncio.get_event_loop()
loop.run_until_complete(test())