I have one DB connection and many сoroutines to request data.
I make the minimal concept, and need help with correct understanding the way of realization.
import asyncio
db_queeu = asyncio.Queue()
async def db_pipe():
while True:
data = await db_queeu.get()
print("DB got", data)
# here process data and return result to requested exec_in_db
async def exec_in_db(query, timeout):
await asyncio.sleep(timeout)
await db_queeu.put(query)
# here I want got result from db_pipe
async def main():
asyncio.create_task(db_pipe())
await asyncio.gather(exec_in_db("Loong query", 4), exec_in_db("Fast query", 1))
print("Listener starts")
if __name__ == "__main__":
asyncio.run(main())
Related
I need to listen tasks on 2 queues, so I wrote the code below, but it has a problem. Currently it behaves like this: if the code started when 2 queues were full, it works great. But if queues were empty one of them was, the code reads messages, but does not proccess them (does not send ack, does not do the logic). But the messages became unacked, until I stop the code. I do not see any reason to be them unacked and unprocessed.
I can't understand what is wrong with the code? May be there is another way to aggregate 2 or more queues like this?
# task_processor.py
from aio_pika import IncomingMessage
class TaskProcessor:
MAX_TASKS_PER_INSTANCE = 1
def __init__(self):
self._tasks = []
def can_accept_new_task(self) -> bool:
return len(self._tasks) < self.MAX_TASKS_PER_INSTANCE
async def process(self, message: IncomingMessage):
self._tasks.append(message)
print(message.body)
await message.ack()
self._tasks.pop()
# main.py
import asyncio
from asyncio import QueueEmpty
from typing import Callable
import aio_pika
from aio_pika import RobustQueue
from dotenv import load_dotenv
load_dotenv()
from core.logger.logger import logger
from core.services.rabbitmq.task_processor.task_processor import TaskProcessor
async def get_single_task(queue: RobustQueue):
while True:
try:
msg = await queue.get(timeout=3600)
return msg
except QueueEmpty:
await asyncio.sleep(3)
except asyncio.exceptions.TimeoutError:
logger.warning('queue timeout error')
pass
except Exception as ex:
logger.error(f"{queue} errored", exc_info=ex)
async def task_aggregator(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
while True:
if should_take_new_task_cb():
queue2, queue1 = queue1, queue2
gen1 = get_single_task(queue1)
gen2 = get_single_task(queue2)
done, _ = await asyncio.wait([gen1, gen2], return_when=asyncio.FIRST_COMPLETED)
for item in done:
result = item.result()
yield result
else:
await asyncio.sleep(1)
async def tasks(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
async for task in task_aggregator(queue1, queue2, should_take_new_task_cb):
yield task
async def main():
connection = await aio_pika.connect_robust(
f"amqp://user:password#host:port/vhost?heartbeat={180}"
)
channel1 = connection.channel()
channel2 = connection.channel()
await channel1.initialize()
await channel2.initialize()
queue1 = await channel1.get_queue('queue1')
queue2 = await channel2.get_queue('queue2')
task_processor = TaskProcessor()
task_generator = tasks(queue1, queue2, task_processor.can_accept_new_task)
while True:
if task_processor.can_accept_new_task():
task = await anext(task_generator)
await task_processor.process(task)
else:
await asyncio.sleep(1)
if __name__ == '__main__':
asyncio.run(main())
We are trying to use asyncio to run a straightforward client/server. The server is an echo server with two possible commands sent by the client, "quit" and "timer". The timer command starts a timer that will print a message in the console every second (at the server and client), and the quit command closes the connection.
The actual problem is the following:
When we run the server and the client, and we start the timer, the result of the timer is not sent to the client. It blocks the server and the client.
I believe that the problem is on the client's side. However, I was not able to detect it.
Server
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, writer: asyncio.StreamWriter, name = None, interval = 1) -> None:
self.name = name
self.interval = interval
self.writer = writer
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
delta = time.time() - self._init_time
self.writer.write(f"Timer {delta} ticked\n".encode())
self.writer.drain()
print("Delta time: ", delta)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def msg_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
'''Handle the echo protocol.'''
# timer task that the client can start:
timer_task = False
try:
while True:
data = await reader.read(1024) # Read 256 bytes from the reader. Size of the message
msg = data.decode() # Decode the message
addr, port = writer.get_extra_info("peername") # Get the address of the client
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = "Message received: " + msg
writer.write(send_message.encode()) # Echo the data back to the client
await writer.drain() # This will wait until everything is clear to move to the next thing.
if data == b"quit" and timer_task is True:
# cancel the timer_task (if any)
if timer_task:
timer_task.cancel()
await timer_task
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"quit" and timer_task is False:
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"start" and timer_task is False:
print("Starting timer")
t = Timer(writer)
timer_task = True
await t.start()
elif data == b"stop" and timer_task is True:
print("Stopping timer")
await t.stop()
timer_task = False
except ConnectionResetError:
print("Client disconnected")
async def run_server() -> None:
# Our awaitable callable.
# This callable is ran when the server recieves some data
server = await asyncio.start_server(msg_handler, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == "__main__":
loop = asyncio.new_event_loop() # new_event_loop() is for python 3.10. For older versions, use get_event_loop()
loop.run_until_complete(run_server())
Client
import asyncio
HOST = '127.0.0.1'
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input('Enter a message: ')
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception('Socket not communicating with the client')
print(f"Received {data.decode()!r}")
if (message == 'quit'):
writer.write(b"quit")
writer.close()
await writer.wait_closed()
exit(2)
# break # Don't know if this is necessary
if __name__ == '__main__':
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())
The client blocks on the input() function. This question is similar to server stop receiving msg after 1 msg receive
Finally, I found a possible solution, by separating the thread.
import asyncio
import websockets
import warnings
warnings.filterwarnings("ignore")
async def send_msg(websocket):
while True:
imp = await asyncio.get_event_loop().run_in_executor(None, lambda: input("Enter something: "))
print("MESSAGE: ", imp)
await websocket.send(imp)
#return imp
async def recv_msg(websocket):
while True:
msg = await websocket.recv()
print(f":> {msg}")
async def echo_loop():
uri = f"ws://localhost:8765"
async with websockets.connect(uri, ssl=None) as websocket:
while True:
await asyncio.gather(recv_msg(websocket),send_msg(websocket))
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(echo_loop())
asyncio.get_event_loop().run_forever()
It seems that there is no clear solution. In particular, there have been many changes in python since the early releases of asyncio, so many possible solutions are outdated.
I change the code to use WebSockets. However, the problem persists: input blocks the code, and none of the solutions above have solved my problem.
Below is the new version of the code (and the error remains):
Server
import asyncio
import websockets
import time
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, websocket, name=None, interval=1) -> None:
self.websocket = websocket
self.name = name
self.interval = interval
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
await self.websocket.send("tick")
print("Delta time: ", time.time() - self._init_time)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def handler(websocket):
print("[WS-SERVER] client connected")
while True:
try:
msg = await websocket.recv()
print(f"<: {msg}")
await websocket.send("Message received. {}".format(msg))
if(msg == "start"):
timer = Timer(websocket)
await timer.start()
except websockets.ConnectionClosed:
print("[WS-SERVER] client disconnected")
break
async def main():
async with websockets.serve(handler, "localhost", 8765):
print("[WS-SERVER] ready")
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())
Client
import asyncio
import websockets
'''async function that recieves and prints messages from the server'''
async def recieve_message(websocket):
msg1 = await websocket.recv()
print(f"<: {msg1}")
async def send_message(websocket):
msg = input("Put your message here: ")
await websocket.send(msg)
print(":> Sent message: ", msg)
async def handler():
uri = "ws://localhost:8765"
async with websockets.connect(uri) as websocket:
while True:
'''run input() in a separate thread'''
recv_msg, send_msg = await asyncio.gather(
recieve_message(websocket),
send_message(websocket),
return_exceptions=True)
if(send_msg == "test"):
print("Supertest")
async def main():
await handler()
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(handler())
print("[WS-CLIENT] bye")
I'm trying to use asyncio with both a sync (would be the rest of the python program) and an async bloc and have the sync bloc send data through asyncio.queues.
Without the queueing everythig works fine.
but when I'm sending data in the queue it seems to block.
I'm trying different ways with get_nowait, etc... but with no success so far.
import asyncio
import time
queue = asyncio.Queue()
async def processor() -> None:
print("Started proc")
while True:
print("waiting for quee")
msg = await queue.get()
print(f"Got command from queue: {msg}")
# do something
await asyncio.sleep(5)
def run_sync(url: str) -> int:
while 1:
print("Sending HTTP request")
input("enter to send message to queue\n")
queue.put_nowait(url)
#do other work
time.sleep(10)
async def run_sync_threaded( url: str) -> int:
return await asyncio.to_thread(run_sync, url)
async def main() -> None:
await asyncio.gather(
processor(),
run_sync_threaded("https://www.example.com"),
)
asyncio.run(main())
EDIT:
Got this working, but looks like a work around instead of a proper solution. I don't know feels not very stable
import asyncio
import time
queue = asyncio.Queue()
async def processor() -> None:
print("Started proc")
while True:
print("waiting for quee")
msg = await queue.get()
print(f"Got command from queue: {msg}")
# do something
await asyncio.sleep(5)
async def async_send(url):
print(f'Adding {url} to queue')
queue.put_nowait(url)
def send(url, loop):
asyncio.run_coroutine_threadsafe(async_send(url), loop)
def run_sync(url: str, loop) -> int:
while 1:
input("enter to send message to queue\n")
send(url, loop)
#do other work
time.sleep(3)
async def run_sync_threaded( url: str, loop) -> int:
return await asyncio.to_thread(run_sync, url, loop)
async def main() -> None:
loop = asyncio.get_event_loop()
t = asyncio.create_task( processor())
t2 = asyncio.create_task(run_sync_threaded("https://www.example.com", loop))
asyncio.gather(
await t,
await t2
)
# This does not work
# asyncio.gather(
# await processor(),
# await run_sync_threaded("https://www.example.com", loop)
# )
asyncio.run(main())
I am trying to have two-way communication between two clients but as soon as the dumps I am no longer able to send any messages from the client file
The send message is what I believe is blocking the get part. Anyone know how to fix this? Thanks in advance
(i am able to send message one time from client to client_led but not after that)
#sio.event
async def get_message(message):
if clientName == message['from']:
pass
else:
if "Client1" == message['from']:
usb_serial.print_on_display(message['message'])
#print(message['from'])
async def send_message():
while True:
await asyncio.sleep(5)
messageToSend = dumps(GPIO_read.get_control_code())
await sio.emit('send_chat_room', {'message': messageToSend, 'name': clientName, 'room': roomName})
Full Scripts
CLIENT.PY
from socketio import AsyncClient
import asyncio
from json import dumps
from aioconsole import ainput
# if __name__ == '__main__':
IpAddress = '0.0.0.0'
PORT = '8080'
clientName = 'Client1'
roomName = 'room'
messageToSend = ''
sio = AsyncClient()
FullIp = 'http://'+IpAddress+':'+PORT
#sio.event
async def connect():
print('Connected to sever')
await sio.emit('join_chat', {'room': roomName,'name': clientName})
#sio.event
async def get_message(message):
if clientName == message['from']:
pass
else:
print(message['message'])
async def send_message(msg): #Pass param in this function
# while True:
await asyncio.sleep(0.1)
messageToSend = await ainput() # Instead of await ainput(), assign the param
await sio.emit('send_chat_room', {'message': messageToSend,'name': clientName, 'room': roomName})
async def connectToServer():
await sio.connect(FullIp)
await sio.wait()
async def main(IpAddress):
await asyncio.gather(
connectToServer(),
send_message("Hey")
)
loop = asyncio.get_event_loop()
loop.run_until_complete(main(FullIp))
Client_led_disp.py
from socketio import AsyncClient
import asyncio
from json import dumps
from aioconsole import ainput
import GPIO_read
import usb_serial
if __name__ == '__main__':
IpAddress = '0.0.0.0'
PORT = '8080'
clientName = 'Electronics'
roomName = 'room'
messageToSend = ''
sio = AsyncClient()
FullIp = 'http://'+IpAddress+':'+PORT
#sio.event
async def connect():
print('Connected to sever')
await sio.emit('join_chat', {'room': roomName, 'name': clientName})
#sio.event
async def get_message(message):
if clientName == message['from']:
pass
else:
if "Client1" == message['from']:
usb_serial.print_on_display(message['message'])
#print(message['from'])
async def send_message():
while True:
await asyncio.sleep(5)
messageToSend = dumps(GPIO_read.get_control_code())
await sio.emit('send_chat_room', {'message': messageToSend, 'name': clientName, 'room': roomName})
async def connectToServer():
await sio.connect(FullIp)
await sio.wait()
async def main(IpAddress):
await asyncio.gather(
connectToServer(),
send_message()
)
loop = asyncio.get_event_loop()
loop.run_until_complete(main(FullIp))
I think the GPIO_read.get_control_code() blocks the event loop.
Which makes you are not able to receive new message anymore.
You can try to put this function into thread pool executor to avoid this problem. You can use asyncio.loop.run_in_executor in your condition.
Using Python 3.6 and asyncio and aiohttp I wrote a simple async program:
from aiohttp import ClientSession
import asyncio, ssl, time
base_url = 'https://my-base-url.com/api'
async def fetch(session, id):
query_params = {'qp1':'v1','qp2':'v2', 'id': id}
async with session.get(base_url, params=query_params, ssl=ssl.SSLContext()) as response:
res_json = await response.json()
if response.status == 200:
time.sleep(2)
min_rating = res_json.get('minRating')
max_rating = res_json.get('maxRating')
print("id = %s, min = %s, max = %s" % (id, min_rating, max_rating))
async def run(ids):
tasks = []
async with ClientSession() as session:
for id in ids:
task = asyncio.ensure_future(fetch(session, id))
tasks.append(task)
responses = await asyncio.gather(*tasks)
return responses
if __name__ == '__main__':
ids = [123, 456, 789]
future = asyncio.ensure_future(run(ids))
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(future)
print("\n\ndone")
The time.sleep(2) inside fetch(session, id) makes it seem like this program is not asynchronous because it gets one response, sleeps, gets another, sleeps, so on and so forth. When I remove the sleep call, it does seem to be async/concurrent because the responses come back in a random order. What is sleep doing in this case? Is it locking all threads? Why does it appear to be sequential instead of parallel?
time.sleep(2) is a synchronous (blocking) call hence you are stopping the asynchronous call with it, you should use await asyncio.sleep(2) which it will "liberate" the resource.