First of all, just want to clarify that I already tried what this thread answers how to add a coroutine to a running asyncio loop? no success.
Context: I have a coroutine listening to a websocket that gives orders to my script to run and keep running other coroutines.
Problem: When I run my code the first coroutine inside SomeClass run() method (sa.run() line) is never awaited.
Main file:
import websockets, asyncio
async def main():
async with websockets.connect(XXX) as serv:
print('Connected to server')
sa = SomeClass(XXX, XXX, serv)
sa.run() # Run in loop handler
asyncio.run(main())
SomeClass definition:
class AsyncLoopThread(Thread):
def __init__(self):
super().__init__(daemon=True)
self.loop = asyncio.new_event_loop()
def run(self):
asyncio.set_event_loop(self.loop)
self.loop.run_forever()
class SomeClass:
def __init__(self, params) -> None:
#Some initiation simple code
self.loop_handler = AsyncLoopThread()
self.loop_handler.start()
async def get_obj_generator(self) -> Generator:
async with websockets.connect(self.some_server_uri) as ws_some_server:
while True:
some_bytes = await ws_some_server.recv()
some_object = pickle.loads(some_bytes)
print('something received.')
yield some_object
async def on_obj_update(self):
obj_generator = self.get_obj_generator()
async for obj in obj_generator:
opportunities = await self.search_opportunities(obj)
for opportunity in opportunities:
#opportunity.execute() is a async def corroutine with some await lines
asyncio.run_coroutine_threadsafe(opportunity.execute(), self.loop_handler.loop)
def run(self):
import time
print(time.time())
asyncio.run_coroutine_threadsafe(self.on_obj_update(), self.loop_handler.loop)
print(time.time())
Output console:
Connected to valuation server...
1658947843.412255
1658947843.413252
Related
Am trying to send motion sensor data in json string using websocket client in another thread to avoid execution blocking for the rest of the code down by an infinite loop in MotionSensor class. but apparently the ws.send() needs await keyword. And if i add it throught i get an error
RuntimeWarning: coroutine 'MotionSensors.run' was never awaited
self.run()
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
and it doesn't send anything to the server
# motionSensor.py
import threading
import time
from client.ClientRequest import Request
class MotionSensors(threading.Thread):
def __init__(self, ws):
threading.Thread.__init__(self)
self.ws = ws
self.open = True
async def run(self):
await self.SendData()
async def SendData(self):
while self.open:
print("Sending motion state....")
state = 1 # Motion state demo value
request = Request()
request.push("mcu/sensors/motion")
request.addBody({
"state_type": "single",
"devices": {"state": state, "device_no": "DVC-876435"}
})
await self.ws.send(request.getAsJsonString())
print("sleeping now for 2 seconds....")
time.sleep(2)
here is my main code
client.py
# client.py
import settings
import asyncio
import websockets
from client.ClientHandler import Devices
from client.Rounte import Route
from ClientRequest import Request
from client.dbHandler import mcuConfig
from client.devices.motionSensor import MotionSensors
def ResponseMSG(request):
print(request)
route = Route()
route.addRoute("/response", ResponseMSG)
def onMessage(request):
route.fireRequest(request)
async def WsClient():
uri = settings.WS_URL
async with websockets.connect(uri) as websocket:
#####################################
###INITIALIZE DEVICES
motion = MotionSensors(websocket)
motion.start()
while True:
print("waiting to recieve......")
message = await websocket.recv()
onMessage(message)
loop = asyncio.get_event_loop()
loop.run_until_complete(WsClient())
loop.run_forever()
Guys i need your help to send data in another thread with a while loop without blocking the execution of code down and without errors. Thank you so much in advance
Actually i changed the code of motionSensor.py
i created a new event loop and i set it to the new thread
and it worked for the case of even those using python 3.7 and below. it works. All thanks to #user4815162342
# motionSensor.py
import threading
import time
import asyncio
from client.ClientRequest import Request
class MotionSensor(threading.Thread):
def __init__(self, ws):
threading.Thread.__init__(self)
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self.ws = ws
self.open = True
def run(self):
self.loop.run_until_complete(self.SendData())
# #asyncio.coroutine
async def SendData(self):
while True:
print("Sending motion state....")
state = 0
request = Request()
request.push("mcu/sensors/motion")
request.addBody({
"state_type": "single",
"devices": {"state": state, "device_no": "DVC-876435"}
})
await self.ws.send(request.getAsJsonString())
print("sleeping now for 5 seconds....")
time.sleep(5)
I'm currently struggling with something "simple".
I'd like to have a python WebSocket Server, which is capable of closing down by outside events (e.g. a Ctrl+C from the command line).
Here is my code so far:
PORT = 8765
class Server(object):
def __init__(self):
self.online_players = dict()
self.online_players_lock = asyncio.Lock()
self.websocket_server = None
async def add_online_player(self, id, player):
async with self.online_players_lock:
self.online_players[id] = player
async def remove_online_player(self, id):
async with self.online_players_lock:
if id in self.online_players.keys():
del self.online_players[id]
def start(self):
end = False
loop = asyncio.new_event_loop()
thread = threading.Thread(target=listen, args=(loop, self))
thread.start()
while not end:
try:
time.sleep(500)
except KeyboardInterrupt:
end = True
loop.call_soon_threadsafe(stop_listening, loop, server)
async def on_connect(websocket, path, server):
print("New user...")
id = await websocket.recv()
player = WebSocketPlayer(id, websocket, server)
await server.add_online_player(id, player)
# from this point on WebSocketPlayer class handles communication
await player.listen()
def listen(loop, server:Server):
asyncio.set_event_loop(loop)
bound_handler = functools.partial(on_connect, server=server)
start_server_task = websockets.serve(bound_handler, "localhost", PORT, ping_timeout=None, loop=loop)
start_server = loop.run_until_complete(start_server_task)
server.websocket_server = start_server
print("Server running ...")
loop.run_forever()
async def stop_listening(loop, server:Server):
await server.websocket_server.wait_close()
loop.stop()
loop.close()
if __name__ == "__main__":
server = Server()
server.start()
Signal handlers from asyncio like loop.add_signal_handler(signum, callback, *args) are not an option for me, because they only work on Unix.
The error that I currently get is that the stop_listening method was never awaited, which kind of makes sense to me. So I am not that much interested in fixing my code example, but more in general how is it possible to achieve my goal, or how is it usually solved?
Thank you very much in advance
Nevermind, this question is related to this question: Why does the asyncio's event loop suppress the KeyboardInterrupt on Windows? which is actually bug of asyncio on Windows.
so this is my code:
import asyncio
import logging
from asyncio import AbstractEventLoop
from aio_pika import connect, IncomingMessage
def test_one(a, b):
print("test_one", a, b)
class Consumer:
def __init__(self, url):
self.url = url
async def run(self, loop: AbstractEventLoop):
while True:
try:
connection = await connect(self.url, loop=loop)
connection.add_close_callback(test_one)
connection.add_close_callback(self.test_two)
# Creating a channel
channel = await connection.channel()
# Declaring queue
queue = await channel.declare_queue("snapshots")
logging.info("Started listening")
# Start listening the queue with name 'hello'
await queue.consume(self.on_message, no_ack=True)
break
except:
logging.error("Could not connect")
finally:
await asyncio.sleep(1)
def on_message(self, message: IncomingMessage):
print(message.body)
def test_two(self, a, b):
print("closed", a, b)
My problem is when I disconnect it only calls test_one function, but it doesn't call test_two function inside the class. I don't understand. I tried only adding the test_two function but that didn't work either. Tried removing the parameters. Same issue. I'm out of ideas. Do you know what I do wrong?
btw the self.on_message does work.
It's possible the API is creating a weak reference to the callback function that it is being passed. Try creating a strong reference to the callback function before passing it.
self._cb_func = self.test_two
connection.add_close_callback(self._cb_func)
The full code:
import asyncio
import logging
from asyncio import AbstractEventLoop
from aio_pika import connect, IncomingMessage
def test_one(a, b):
print("test_one", a, b)
class Consumer:
def __init__(self, url):
self.url = url
async def run(self, loop: AbstractEventLoop):
while True:
try:
connection = await connect(self.url, loop=loop)
connection.add_close_callback(test_one)
self._cb_func = self.test_two
connection.add_close_callback(self._cb_func)
# Creating a channel
channel = await connection.channel()
# Declaring queue
queue = await channel.declare_queue("snapshots")
logging.info("Started listening")
# Start listening the queue with name 'hello'
await queue.consume(self.on_message, no_ack=True)
break
except:
logging.error("Could not connect")
finally:
await asyncio.sleep(1)
def on_message(self, message: IncomingMessage):
print(message.body)
def test_two(self, a, b):
print("closed", a, b)
If you have a lot of callback functions, then there is an answer in this question for storing them as an array: using python WeakSet to enable a callback functionality
Problem
I have a library which currently has no async support and needs to be called from async code. The async code calls into the library through a handler (handler function in the code below). While the handler executed, the library periodically calls a callback (callback_wrapper) to report progress.
The synchronous handler is executed in a ThreadPoolExecutor in order for the main event loop to be able to process further events while the handler is running.
What happens is that the synchronous callback is executed immediately, but the async callback is only executed after the main handler has executed. The desired result is the async callbacks to be executed immediately.
I guess the event loop is blocked at the run_in_executor call, but I am not sure how to resolve this.
Code
import asyncio
import time
from concurrent.futures.thread import ThreadPoolExecutor
loop = asyncio.get_event_loop()
def handler():
print('handler started')
callback_wrapper()
time.sleep(1)
print('handler stopped')
async def callback():
print('callback')
def callback_wrapper():
print('callback wrapper started')
asyncio.ensure_future(callback(), loop=loop)
print('callback wrapper stopped')
async def main():
handler()
with ThreadPoolExecutor() as pool:
async def thread_handler():
await loop.run_in_executor(pool, handler)
loop.run_until_complete(main())
Output
handler started
callback wrapper started
callback wrapper stopped
handler stopped
callback
Desired Output
handler started
callback wrapper started
callback
callback wrapper stopped
handler stopped
thanks to #user4815162342's input, I came up the the following solution:
import asyncio
import time
from concurrent.futures.thread import ThreadPoolExecutor
loop = asyncio.get_event_loop()
def handler():
print('handler started')
callback_wrapper()
time.sleep(1)
print('handler stopped')
async def callback():
print('callback')
def callback_wrapper():
print('callback wrapper started')
asyncio.run_coroutine_threadsafe(callback(), loop).result()
print('callback wrapper stopped')
async def main():
await thread_handler()
with ThreadPoolExecutor() as pool:
async def thread_handler():
await loop.run_in_executor(pool, handler)
loop.run_until_complete(main())
which produces the desired result:
handler started
callback wrapper started
callback
callback wrapper stopped
handler stopped
I am trying to add two coroutines to asyncio loop and getting an error:
RuntimeError: This event loop is already running
My objective is to communicate to a server (that I have no control of). This server expects an initial connection from the client. The server then provided a port to the client on this connection. The client has to use this port to create a second connection. This second connection is used by the server to send unsolicited messages to the client. The first connection remains up throughout for other two-way communications.
To recreate this scenario, I have some code that reproduces the error:
class Connection():
def __init__(self, ip, port, ioloop):
self.ip = ip
self.port = port
self.ioloop = ioloop
self.reader, self.writer = None, None
self.protocol = None
self.fileno = None
async def __aenter__(self):
# Applicable when doing 'with Connection(...'
log.info("Entering and Creating Connection")
self.reader, self.writer = (
await asyncio.open_connection(self.ip, self.port, loop=self.ioloop)
)
self.protocol = self.writer.transport.get_protocol()
self.fileno = self.writer.transport.get_extra_info('socket').fileno()
log.info(f"Created connection {self}")
return self
async def __aexit__(self, *args):
# Applicable when doing 'with Connection(...'
log.info(f"Exiting and Destroying Connection {self}")
if self.writer:
self.writer.close()
def __await__(self):
# Applicable when doing 'await Connection(...'
return self.__aenter__().__await__()
def __repr__(self):
return f"[Connection {self.ip}:{self.port}, {self.protocol}, fd={self.fileno}]"
async def send_recv_message(self, message):
log.debug(f"send: '{message}'")
self.writer.write(message.encode())
await self.writer.drain()
log.debug("awaiting data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv: '{data}'")
return data
class ServerConnection(Connection):
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
self.ioloop.run_until_complete(event_connection.recv_message())
class EventConnection(Connection):
async def recv_message(self):
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv only: '{data}'")
return data
async def main(loop):
client1 = await ServerConnection('127.0.0.1', 8888, loop)
await client1.setup_connection()
await client1.send_recv_message("Hello1")
await client1.send_recv_message("Hello2")
await asyncio.sleep(5)
if __name__ == '__main__':
#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
The error occurs in ServerConnection.setup_connection() method where run_until_complete is being called.
I am probably doing something wrong due to lack of understanding asyncio. Basically, how do I setup a secondary connection which will get event notifications (unsolicited) while setting up the first connection?
Thanks.
Followup
Since the code is very similar (a few changes to add more functionality to it), I hope it's not bad etiquette to followup to the original post as the resulting error is still the same.
The new issue is that when it receives the unsolicited message (which is received by EventConnection), the recv_message calls process_data method. I would like to make process_data be a future so that recv_message completes (ioloop should stop). The ensure_future would then pick it up and continue running again to use ServerConnection to do a request/response to the server. Before it does that though, it has to go to some user code (represented by external_command() and from whom I would prefer to hide the async stuff). This would make it synchronous again. Hence, once they've done what they need to, they should call execute_command on ServerConnection, which then kicks off the loop again.
The problem is, my expectation for using ensure_future didn't pan out as it seems the loop didn't stop from running. Hence, when the code execution reaches execute_command which does the run_until_complete, an exception with the error "This event loop is already running" occurs.
I have two questions:
How can I make it so that the ioloop can stop after process_data is
placed into ensure_future, and subsequently be able to run it again
in execute_command?
Once recv_message has received something, how can we make it so that
it can receive more unsolicited data? Is it enough/safe to just use
ensure_future to call itself again?
Here's the example code that simulates this issue.
client1 = None
class ServerConnection(Connection):
connection_type = 'Server Connection'
async def setup_connection(self):
event_port = 8889 # Assume this came from the server
print("In setup connection")
event_connection = await EventConnection('127.0.0.1', event_port, self.ioloop)
asyncio.ensure_future(event_connection.recv_message())
async def _execute_command(self, data):
return await self.send_recv_message(data)
def execute_command(self, data):
response_str = self.ioloop.run_until_complete(self._execute_command(data))
print(f"exec cmd response_str: {response_str}")
def external_command(self, data):
self.execute_command(data)
class EventConnection(Connection):
connection_type = 'Event Connection'
async def recv_message(self):
global client1
log.debug("awaiting recv-only data...")
data = await self.reader.read(9999)
data = data.decode()
log.debug(f"recv-only: '{data}'")
asyncio.ensure_future(self.process_data(data))
asyncio.ensure_future(self.recv_message())
async def process_data(self, data):
global client1
await client1.external_command(data)
async def main(ioloop):
global client1
client1 = await ServerConnection('127.0.0.1', 8888, ioloop)
await client1.setup_connection()
print(f"after connection setup loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello1")
print(f"after Hello1 loop running is {ioloop.is_running()}")
await client1.send_recv_message("Hello2")
print(f"after Hello2 loop running is {ioloop.is_running()}")
while True:
print(f"inside while loop running is {ioloop.is_running()}")
t = 10
print(f"asyncio sleep {t} sec")
await asyncio.sleep(t)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger()
ioloop = asyncio.get_event_loop()
print('starting loop')
ioloop.run_until_complete(main(ioloop))
print('completed loop')
ioloop.close()
Try replacing:
self.ioloop.run_until_complete
With
await