I'm wrting a python program to listen to NFT mints event by using websocket, but after running some time, the websocket will lose its connection, hence, I need to stop loop and restart it, but after searching many answers there still no solution for it, here is my code:
async def main():
await asyncio.gather(erc1155_listener(),erc721_listener())
async def erc721_listener():
task_set = set()
async with connect(alchemy_ws_url) as ws:
await ws.send(erc721_log_filter)
while True:
try:
message = await asyncio.wait_for(ws.recv(), timeout=60)
event = json.loads(message)
topics = event['params']['result']['topics']
transaction_hash = event['params']['result']['transactionHash']
contract_address = event['params']['result']['address']
if len(topics) == 4 and re.match(re.compile(r'0x[0]{0,}$'),topics[1]):
task = asyncio.create_task(free_mint_filter(transaction_hash,contract_address))
task_set.add(task)
task.add_done_callback(task_set.discard)
except Exception:
pass
async def erc1155_listener():
task_set = set()
async with connect(alchemy_ws_url) as ws:
await ws.send(erc1155_log_filter)
while True:
try:
message = await asyncio.wait_for(ws.recv(), timeout=60)
event = json.loads(message)
topics = event['params']['result']['topics']
transaction_hash = event['params']['result']['transactionHash']
contract_address = event['params']['result']['address']
if re.match( re.compile(r'0x[0]{0,}$'),topics[2]):
task = asyncio.create_task(free_mint_filter(transaction_hash,contract_address))
task_set.add(task)
task.add_done_callback(task_set.discard)
except Exception as e:
pass
async def free_mint_filter(transaction_hash,contract_address):
loop = asyncio.get_running_loop()
try:
transaction = await loop.run_in_executor(None, web3.eth.getTransaction, transaction_hash)
value = transaction['value']
if value == 0:
print(transaction['hash'].hex(),contract_address)
else:
print(transaction['hash'].hex(),'is not a free mint')
except Exception as e:
print(e)
asyncio.run(main())
if you have another better solution than restarting asyncio loop please tell me.
Related
I need to listen tasks on 2 queues, so I wrote the code below, but it has a problem. Currently it behaves like this: if the code started when 2 queues were full, it works great. But if queues were empty one of them was, the code reads messages, but does not proccess them (does not send ack, does not do the logic). But the messages became unacked, until I stop the code. I do not see any reason to be them unacked and unprocessed.
I can't understand what is wrong with the code? May be there is another way to aggregate 2 or more queues like this?
# task_processor.py
from aio_pika import IncomingMessage
class TaskProcessor:
MAX_TASKS_PER_INSTANCE = 1
def __init__(self):
self._tasks = []
def can_accept_new_task(self) -> bool:
return len(self._tasks) < self.MAX_TASKS_PER_INSTANCE
async def process(self, message: IncomingMessage):
self._tasks.append(message)
print(message.body)
await message.ack()
self._tasks.pop()
# main.py
import asyncio
from asyncio import QueueEmpty
from typing import Callable
import aio_pika
from aio_pika import RobustQueue
from dotenv import load_dotenv
load_dotenv()
from core.logger.logger import logger
from core.services.rabbitmq.task_processor.task_processor import TaskProcessor
async def get_single_task(queue: RobustQueue):
while True:
try:
msg = await queue.get(timeout=3600)
return msg
except QueueEmpty:
await asyncio.sleep(3)
except asyncio.exceptions.TimeoutError:
logger.warning('queue timeout error')
pass
except Exception as ex:
logger.error(f"{queue} errored", exc_info=ex)
async def task_aggregator(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
while True:
if should_take_new_task_cb():
queue2, queue1 = queue1, queue2
gen1 = get_single_task(queue1)
gen2 = get_single_task(queue2)
done, _ = await asyncio.wait([gen1, gen2], return_when=asyncio.FIRST_COMPLETED)
for item in done:
result = item.result()
yield result
else:
await asyncio.sleep(1)
async def tasks(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
async for task in task_aggregator(queue1, queue2, should_take_new_task_cb):
yield task
async def main():
connection = await aio_pika.connect_robust(
f"amqp://user:password#host:port/vhost?heartbeat={180}"
)
channel1 = connection.channel()
channel2 = connection.channel()
await channel1.initialize()
await channel2.initialize()
queue1 = await channel1.get_queue('queue1')
queue2 = await channel2.get_queue('queue2')
task_processor = TaskProcessor()
task_generator = tasks(queue1, queue2, task_processor.can_accept_new_task)
while True:
if task_processor.can_accept_new_task():
task = await anext(task_generator)
await task_processor.process(task)
else:
await asyncio.sleep(1)
if __name__ == '__main__':
asyncio.run(main())
We are trying to use asyncio to run a straightforward client/server. The server is an echo server with two possible commands sent by the client, "quit" and "timer". The timer command starts a timer that will print a message in the console every second (at the server and client), and the quit command closes the connection.
The actual problem is the following:
When we run the server and the client, and we start the timer, the result of the timer is not sent to the client. It blocks the server and the client.
I believe that the problem is on the client's side. However, I was not able to detect it.
Server
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, writer: asyncio.StreamWriter, name = None, interval = 1) -> None:
self.name = name
self.interval = interval
self.writer = writer
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
delta = time.time() - self._init_time
self.writer.write(f"Timer {delta} ticked\n".encode())
self.writer.drain()
print("Delta time: ", delta)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def msg_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
'''Handle the echo protocol.'''
# timer task that the client can start:
timer_task = False
try:
while True:
data = await reader.read(1024) # Read 256 bytes from the reader. Size of the message
msg = data.decode() # Decode the message
addr, port = writer.get_extra_info("peername") # Get the address of the client
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = "Message received: " + msg
writer.write(send_message.encode()) # Echo the data back to the client
await writer.drain() # This will wait until everything is clear to move to the next thing.
if data == b"quit" and timer_task is True:
# cancel the timer_task (if any)
if timer_task:
timer_task.cancel()
await timer_task
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"quit" and timer_task is False:
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
elif data == b"start" and timer_task is False:
print("Starting timer")
t = Timer(writer)
timer_task = True
await t.start()
elif data == b"stop" and timer_task is True:
print("Stopping timer")
await t.stop()
timer_task = False
except ConnectionResetError:
print("Client disconnected")
async def run_server() -> None:
# Our awaitable callable.
# This callable is ran when the server recieves some data
server = await asyncio.start_server(msg_handler, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == "__main__":
loop = asyncio.new_event_loop() # new_event_loop() is for python 3.10. For older versions, use get_event_loop()
loop.run_until_complete(run_server())
Client
import asyncio
HOST = '127.0.0.1'
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input('Enter a message: ')
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception('Socket not communicating with the client')
print(f"Received {data.decode()!r}")
if (message == 'quit'):
writer.write(b"quit")
writer.close()
await writer.wait_closed()
exit(2)
# break # Don't know if this is necessary
if __name__ == '__main__':
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())
The client blocks on the input() function. This question is similar to server stop receiving msg after 1 msg receive
Finally, I found a possible solution, by separating the thread.
import asyncio
import websockets
import warnings
warnings.filterwarnings("ignore")
async def send_msg(websocket):
while True:
imp = await asyncio.get_event_loop().run_in_executor(None, lambda: input("Enter something: "))
print("MESSAGE: ", imp)
await websocket.send(imp)
#return imp
async def recv_msg(websocket):
while True:
msg = await websocket.recv()
print(f":> {msg}")
async def echo_loop():
uri = f"ws://localhost:8765"
async with websockets.connect(uri, ssl=None) as websocket:
while True:
await asyncio.gather(recv_msg(websocket),send_msg(websocket))
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(echo_loop())
asyncio.get_event_loop().run_forever()
It seems that there is no clear solution. In particular, there have been many changes in python since the early releases of asyncio, so many possible solutions are outdated.
I change the code to use WebSockets. However, the problem persists: input blocks the code, and none of the solutions above have solved my problem.
Below is the new version of the code (and the error remains):
Server
import asyncio
import websockets
import time
class Timer(object):
'''Simple timer class that can be started and stopped.'''
def __init__(self, websocket, name=None, interval=1) -> None:
self.websocket = websocket
self.name = name
self.interval = interval
async def _tick(self) -> None:
while True:
await asyncio.sleep(self.interval)
await self.websocket.send("tick")
print("Delta time: ", time.time() - self._init_time)
async def start(self) -> None:
self._init_time = time.time()
self.task = asyncio.create_task(self._tick())
async def stop(self) -> None:
self.task.cancel()
print("Delta time: ", time.time() - self._init_time)
async def handler(websocket):
print("[WS-SERVER] client connected")
while True:
try:
msg = await websocket.recv()
print(f"<: {msg}")
await websocket.send("Message received. {}".format(msg))
if(msg == "start"):
timer = Timer(websocket)
await timer.start()
except websockets.ConnectionClosed:
print("[WS-SERVER] client disconnected")
break
async def main():
async with websockets.serve(handler, "localhost", 8765):
print("[WS-SERVER] ready")
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(main())
Client
import asyncio
import websockets
'''async function that recieves and prints messages from the server'''
async def recieve_message(websocket):
msg1 = await websocket.recv()
print(f"<: {msg1}")
async def send_message(websocket):
msg = input("Put your message here: ")
await websocket.send(msg)
print(":> Sent message: ", msg)
async def handler():
uri = "ws://localhost:8765"
async with websockets.connect(uri) as websocket:
while True:
'''run input() in a separate thread'''
recv_msg, send_msg = await asyncio.gather(
recieve_message(websocket),
send_message(websocket),
return_exceptions=True)
if(send_msg == "test"):
print("Supertest")
async def main():
await handler()
await asyncio.Future() # run forever
if __name__ == "__main__":
asyncio.run(handler())
print("[WS-CLIENT] bye")
The following code works, but fails to exit upon CTRL+c
Can anyone explain why, and how to fix?
It's asyncio code, but I'm using trio-asyncio (forward planning) to allow me to also use trio code.
#!.venv/bin/python
import asyncio
import trio
import trio_asyncio
from binance import AsyncClient, BinanceSocketManager
as_trio = trio_asyncio.aio_as_trio
as_aio = trio_asyncio.trio_as_aio
from contextlib import asynccontextmanager
#asynccontextmanager
async def AClient():
client = await AsyncClient.create()
try:
yield client
except KeyboardInterrupt:
print('ctrl+c')
exit(0)
finally:
await client.close_connection()
async def kline_listener(client):
print(1)
bm = BinanceSocketManager(client)
async with bm.kline_socket(symbol='BNBBTC') as stream:
while True:
try:
res = await stream.recv()
print(res)
except KeyboardInterrupt:
break
#as_trio
async def aio_main():
async with AClient() as client:
exchange_info = await client.get_exchange_info()
tickers = await client.get_all_tickers()
print(client, exchange_info.keys(), tickers[:2])
kline = asyncio.create_task(kline_listener(client))
await kline
if __name__ == "__main__":
trio_asyncio.run(aio_main)
I've placed except KeyboardInterrupt at 2 places, though it appears to do nothing.
Simple example
import asyncio
import logging
from aiogram import Bot, Dispatcher, types
logging.basicConfig(level=logging.INFO)
token = 'token'
bot = Bot(token=token)
dp = Dispatcher(bot=bot)
#dp.callback_query_handler(text='stoploop')
async def stop_loop(query: types.CallbackQuery):
# TODO how to stop test loop?
await query.message.edit_text('stop')
#dp.callback_query_handler(text='test')
async def start_loop(query: types.CallbackQuery):
a = 100
while True:
a -= 1
markup = types.InlineKeyboardMarkup()
markup.add(types.InlineKeyboardButton('<<<Stop And Back To Home', callback_data='stoploop'))
await query.message.edit_text(str(a),reply_markup=markup)
await asyncio.sleep(1)
#dp.message_handler(commands='start')
async def start_cmd_handler(message: types.Message):
markup = types.InlineKeyboardMarkup()
markup.add(
types.InlineKeyboardButton('start loop', callback_data='test')
)
await message.reply('test', reply_markup=markup)
async def main():
try:
await dp.start_polling()
finally:
await bot.close()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I click start_loop, the tg message box on my page starts to display a countdown. When I click stop, how can I stop the previous countdown?
I use id(query) to confirm that the query instance sent twice is not the same. After I execute the stop_loop function, start_loop will still execute and change the content of the message.
Can someone tell me how to stop it?
I used redis to solve it, but I don't know if this is the most appropriate way. If there is a more suitable way, please let me know
To manage your loop you should take it outside the handlers and just get in from any storage (dict is used for example).
Basic example of the loop
loops = {}
class Loop:
def __init__(self, user_id):
self.user_id = user_id
self._active = False
self._stopped = True
loops[self.user_id] = self
#classmethod
def get_loop(cls, user_id):
return loops.get(user_id, cls(user_id))
#property
def is_running(self):
return not self._stopped
async def start(self):
self._active = True
asyncio.create_task(self._run_loop())
async def _run_loop(self):
while self._active:
await bot.send_message(self.user_id, 'loop is running')
await asyncio.sleep(5)
self._stopped = True
async def stop(self):
self._active = False
while not self._stopped:
await asyncio.sleep(1)
So then:
#dp.callback_query_handler(text='start')
async def start_loop(query: CallbackQuery):
user = query.from_user
loop = Loop.get_loop(user.id)
if loop.is_running:
return await query.answer('Loop is already running')
loop.start()
await query.answer('Started!')
#dp.callback_query_handler(text='stop')
async def stop_loop(query: CallbackQuery):
user = query.from_user
loop = Loop.get_loop(user.id)
await query.answer('Stopping...')
await loop.stop()
await bot.send_message(user.id, 'Loop successfully stopped.')
Currently, I am using asyncio.wait_for to poll websocket.recv so that I can also call websocket.send when I need to:
async def client_reader():
websocket = await websockets.connect(f"ws://localhost:{PORT}")
result = ''
while True:
try:
# Is there a better way to recv from a websocket?
result = result + await asyncio.wait_for(websocket.recv(), timeout=1.0)
if result.endswith('\n'):
result = result.rstrip()
print(f"result = {result}")
result = ''
# I need to do other asyncio here
await some_other_work()
except asyncio.TimeoutError:
pass
All of the documentation for asyncio and websockets that I could find only used toy examples. Is there a better way to do this?
This is a stand-alone program that simulates what I am doing:
import asyncio
import websockets
import random
import multiprocessing
import time
PORT = 49152
text = """\
This is a bunch of text that will be used to
simulate a server sending multiple lines of
text to a client with a random amount of delay
between each line.
"""
def get_next_line():
text_list = text.split('\n')
while True:
for line in text_list:
yield line + '\n'
line_generator = get_next_line()
async def delay_server(websocket, path):
while True:
await asyncio.sleep(random.random() * 5.0)
line = next(line_generator)
await websocket.send(line)
def server_func():
try:
start_server = websockets.serve(delay_server, "localhost", PORT)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
except KeyboardInterrupt:
pass
async def some_other_work():
print("I occasionally need to call websocket.send() here")
async def client_reader():
websocket = await websockets.connect(f"ws://localhost:{PORT}")
result = ''
while True:
try:
result = result + await asyncio.wait_for(websocket.recv(), timeout=1.0)
if result.endswith('\n'):
result = result.rstrip()
print(f"result = {result}")
result = ''
await some_other_work()
except asyncio.TimeoutError:
pass
def client_func():
try:
asyncio.run(client_reader())
except KeyboardInterrupt:
pass
server_proc = multiprocessing.Process(target=server_func)
server_proc.daemon = True
server_proc.start()
client_proc = multiprocessing.Process(target=client_func)
client_proc.daemon = True
client_proc.start()
try:
while True:
time.sleep(1.0)
except KeyboardInterrupt:
pass
server_proc.join()
client_proc.join()
I changed the code to look like this:
async def client_reader(websocket, result):
try:
result = result + await asyncio.wait_for(websocket.recv(), timeout=1.0)
if result.endswith('\n'):
result = result.rstrip()
print(f"result = {result}")
result = ''
except asyncio.TimeoutError:
pass
return result
async def if_flag_send_message():
print("if a flag is set, I will call websocket.send here")
async def client_writer(websocket):
await if_flag_send_message()
await asyncio.sleep(1.0)
async def client_handler():
websocket = await websockets.connect(f"ws://localhost:{PORT}")
result = ''
while True:
reader_task = asyncio.create_task(client_reader(websocket, result))
writer_task = asyncio.create_task(client_writer(websocket))
await asyncio.gather(reader_task, writer_task)
result = reader_task.result()
I am still not sure this is the right way to do things, but it works.