Develop Reference

python aiogram how to stop an asynchronous loop

Simple example
import asyncio
import logging
from aiogram import Bot, Dispatcher, types
logging.basicConfig(level=logging.INFO)
token = 'token'
bot = Bot(token=token)
dp = Dispatcher(bot=bot)
#dp.callback_query_handler(text='stoploop')
async def stop_loop(query: types.CallbackQuery):
# TODO how to stop test loop?
await query.message.edit_text('stop')
#dp.callback_query_handler(text='test')
async def start_loop(query: types.CallbackQuery):
a = 100
while True:
a -= 1
markup = types.InlineKeyboardMarkup()
markup.add(types.InlineKeyboardButton('<<<Stop And Back To Home', callback_data='stoploop'))
await query.message.edit_text(str(a),reply_markup=markup)
await asyncio.sleep(1)
#dp.message_handler(commands='start')
async def start_cmd_handler(message: types.Message):
markup = types.InlineKeyboardMarkup()
markup.add(
types.InlineKeyboardButton('start loop', callback_data='test')
)
await message.reply('test', reply_markup=markup)
async def main():
try:
await dp.start_polling()
finally:
await bot.close()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I click start_loop, the tg message box on my page starts to display a countdown. When I click stop, how can I stop the previous countdown?
I use id(query) to confirm that the query instance sent twice is not the same. After I execute the stop_loop function, start_loop will still execute and change the content of the message.
Can someone tell me how to stop it?

I used redis to solve it, but I don't know if this is the most appropriate way. If there is a more suitable way, please let me know

To manage your loop you should take it outside the handlers and just get in from any storage (dict is used for example).
Basic example of the loop
loops = {}
class Loop:
def __init__(self, user_id):
self.user_id = user_id
self._active = False
self._stopped = True
loops[self.user_id] = self
#classmethod
def get_loop(cls, user_id):
return loops.get(user_id, cls(user_id))
#property
def is_running(self):
return not self._stopped
async def start(self):
self._active = True
asyncio.create_task(self._run_loop())
async def _run_loop(self):
while self._active:
await bot.send_message(self.user_id, 'loop is running')
await asyncio.sleep(5)
self._stopped = True
async def stop(self):
self._active = False
while not self._stopped:
await asyncio.sleep(1)
So then:
#dp.callback_query_handler(text='start')
async def start_loop(query: CallbackQuery):
user = query.from_user
loop = Loop.get_loop(user.id)
if loop.is_running:
return await query.answer('Loop is already running')
loop.start()
await query.answer('Started!')
#dp.callback_query_handler(text='stop')
async def stop_loop(query: CallbackQuery):
user = query.from_user
loop = Loop.get_loop(user.id)
await query.answer('Stopping...')
await loop.stop()
await bot.send_message(user.id, 'Loop successfully stopped.')

set_exception_handler ignored in python3.6 with asyncio

I'm basically creating an object that needs to perform a number of tasks in async mode (+ other things but I've tried to simplify here). Here is a snippet of code for the object itself. It's successful (thanks to Lynn Root's example) at handling signals - but not at handling exceptions. Or at least not in the way I am hoping to be able to handle them.
class myobj(object):
def __init__(self, loop: asyncio.AbstractEventLoop):
self.shutdown = False
self.loop = loop
async def graceful_shutdown(self, s=None):
if s is not None:
logging.warning(f'Receiving signal {s.name}.')
else:
logging.warning(f'Shutting NOT via signal')
logging.warning(f'Initiating cancellation of {len(self.tasks)} tasks...')
[task.cancel() for task in self.tasks]
logging.warning(f'Gaterhing out put of cancellation of {len(self.tasks)} tasks...')
await asyncio.gather(*self.tasks, loop=self.loop, return_exceptions=True)
logging.warning('Done graceful shutdown of subtasks')
# Mark main task to shutdown
self.shutdown = True
async def run(self):
i = 0
taskx = self.loop.create_task(self.task_x())
self.tasks = [taskx]
while not self.shutdown:
await asyncio.sleep(1)
print(f'Main runner... {i}')
i += 1
logging.warning('Main runner is over.')
async def task_x(self):
logging.warning('Starting task X')
i = 0
while True:
await asyncio.sleep(2.25)
print(f'Doing task x... {i}')
if i == 2:
raise RuntimeError('BOOM X!')
i += 1
At this point, from the "main" I need to install a few things and create the loop :
def main():
try:
global newobj
loop = asyncio.get_event_loop()
logging.warning(f'Installing exception handler')
loop.set_exception_handler(handle_exception)
logging.warning(f'Creating main object')
newobj = myobj(loop)
logging.warning(f'Installing signal handlers')
signals = (signal.SIGINT, signal.SIGTERM)
for s in signals:
loop.add_signal_handler(s, lambda s=s: loop.create_task(newobj.graceful_shutdown(s)))
logging.warning(f'Running object...')
loop.run_until_complete(newobj.run())
finally:
loop.close()
logging.warning(f'object is Shutdown - Exiting program.')
sys.exit(0)
if __name__ == "__main__":
main()
But the handle_exception needs to be defined.
def handle_exception(loop, context):
# context["message"] will always be there; but context["exception"] may not
msg = context.get("exception", context["message"])
logging.error(f'Caught exception: {msg}')
logging.info(f'Shutting down from exception.')
loop.create_task(newobj.graceful_shutdown())
The problem is that it's never calling handle_exception. I need to be running this in python3.6 for some reason. What am I missing here?

How to create mutliple tasks asyncio and get returns values

I'm beggining with asyncio and I'm struggling awaiting multiple return values from differents functions.
I want to do something like thread but using asyncio since I'm using async librairies from Azure.
I need to launch in my Thread class some function and I want to launch them in conccurency and await for their results and see if the results are good (I'm in a django test).
Here my current code:
class DeviceThread(threading.Thread):
def __init__(self, test_case, device_id, module_id, cert_prim, cert_sec):
threading.Thread.__init__(self)
self.test_case = test_case
self.device_id = device_id
self.module_id = module_id
self.cert_prim = cert_prim
self.cert_sec = cert_sec
async def get_twin(self, device):
try:
# 0 - Get the twin for the device
twin = await device.get_twin()
info(twin)
return twin['desired']['test'] == "test1device"
except Exception as e:
info(format_exc())
return False
async def receive_message_cloud(self, device):
try:
# 1- Waiting for the first message from the server (Blocking call)
message = await device.receive_message_cloud()
return message.data.decode() == "testmessage1"
except Exception as e:
info(format_exc())
return False
async def send_message_cloud(self, device):
try:
# 2 -Sending a message to the backend
await device.send_message_cloud("testmessage1")
return True
except Exception as e:
info(format_exc())
return False
async def receive_direct_call_and_respond(self, device):
try:
# 3 - Receiving a direct call method from the backend
method_request = await device.receive_method_request()
# 4 - Sending result of direct call
payload = {"test": "testok"}
status = 200
await device.send_method_response(method_request, status, payload)
return (method_request.name == "testmethod1") and (method_request.payload == "testpayload1")
except Exception as e:
info(format_exc())
return False
async def update_twin_properties(self, device):
try:
# 5 - Updating twin properties
new_properties = {'test', 'test2device'}
device.patch_twin_reported_properties(new_properties)
return True
except Exception as e:
info(format_exc())
return False
async def perform(self, device):
# Creating the tasks that will execute in parrallel
twin_get_res = asyncio.create_task(self.get_twin(device))
rec_mess_cloud = asyncio.create_task(self.receive_message_cloud(device))
send_mess_cloud = asyncio.create_task(self.send_message_cloud(device))
rec_dir_call = asyncio.create_task(self.receive_direct_call_and_respond(device))
up_twin_prop = asyncio.create_task(self.update_twin_properties(device))
# Verify the execution of the routine when done
self.test_case.assertTrue(await twin_get_res)
self.test_case.assertTrue(await rec_mess_cloud)
self.test_case.assertTrue(await send_mess_cloud)
self.test_case.assertTrue(await rec_dir_call)
self.test_case.assertTrue(await up_twin_prop)
def run(self):
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# Getting writing the cert content to disk
open("cert.pem", "w").write(self.cert_prim)
# Creating a device
device = IOTHubDevice()
device.authenticate_device(self.device_id, "cert.pem")
# Removing cert previously created
remove("cert.pem")
asyncio.run(self.perform(device))
except Exception as e:
info(format_exc())
self.test_case.assertFalse(True)
As you've seen I'm in a thread and I want to verify some function of Azure IOTHub.
But I get this error :
RuntimeError: Task got Future attached to a different loop
So my question is : how can I get all these tasks running in the loop and get their individual results ?
Thanks for your answer !

Asyncio python - TypeError: A Future, a coroutine or an awaitable is required

One of the async function returns the async generator object. I added loop.run_until_complete(func()), but still, it throws the error as "TypeError: A Future, a coroutine or an awaitable is required". Below is the code. I'm trying to fetch the records from Neo4j asynchronously. I got the async "Neo4j class from a GitHub. I'm new to this async concept.
from concurrent import futures
import neo4j
from neo4j import GraphDatabase, basic_auth
import time
import traceback
import asyncio
RETRY_WAITS = [0, 1, 4] # How long to wait after each successive failure.
class Neo4j:
"""Neo4j database API."""
def __init__(self, config, loop):
self.config = config
self.loop = loop
self.executor = futures.ThreadPoolExecutor(max_workers=30)
for retry_wait in RETRY_WAITS:
try:
self.init_driver()
break
except:
if retry_wait == RETRY_WAITS[-1]:
raise
else:
print('WARNING: retrying to Init DB; err:')
traceback.print_exc()
time.sleep(retry_wait) # wait for 0, 1, 3... seconds.
def init_driver(self):
auth = basic_auth(self.config['user'], self.config['pass'])
self.driver = GraphDatabase.driver(self.config['url'], auth=auth)
async def afetch_start(self, query):
session = self.driver.session(access_mode=neo4j.READ_ACCESS)
def run():
return session.run(query).records()
return session, await self.loop.run_in_executor(self.executor, run)
async def afetch_iterate(self, session, iter):
def iterate():
try:
return next(iter)
except StopIteration:
return None
while True:
res = await self.loop.run_in_executor(self.executor, iterate)
if res is None:
return
else:
yield dict(res)
async def afetch(self, query):
for retry_wait in RETRY_WAITS:
try:
session, iter = await self.afetch_start(query)
break
except (BrokenPipeError, neo4j.exceptions.ServiceUnavailable) as e:
if retry_wait == RETRY_WAITS[-1]:
raise
else:
await asyncio.sleep(retry_wait)
await self.loop.run_in_executor(self.executor, self.init_driver)
async for x in self.afetch_iterate(session, iter):
yield x
await self.loop.run_in_executor(self.executor, session.close)
async def afetch_one(self, query):
async for i in self.afetch(query):
return i
return None
async def aexec(self, query):
async for i in self.afetch(query):
pass
return
config={'url':"bolt://localhost",'user':'neo4j','pass':'pwd'}
loop=asyncio.get_event_loop()
n=Neo4j(config,loop)
loop.run_until_complete(n.afetch("MATCH(p:Person)-[:Acted_in]->(mv:Movies) RETURN p.name as actors"))
loop.close()
--EDIT
I have modified the code to work properly. The query returns 218K rows and it takes 5 minutes to extract the complete list and the same async operation in C# completes in just 2 sec. Looks like the above code still doesnt go in async

It's very hard to tell what exactly happens without reproducible example, but I'll take a guess. You probably pass async generator object in a loop, you shouldn't do it. A way to work with async generators is to use async for. Here's example:
import asyncio
async def func(): # async generator
yield 1
yield 2
yield 3
async def main():
async for i in func(): # get values from async generator
print(i)
asyncio.run(main()) # can be used instead of loop.run_until_complete(main())

Timeout handling while using run_in_executor and asyncio

I'm using asyncio to run a piece of blocking code like this:
result = await loop.run_in_executor(None, long_running_function)
My question is: Can I impose a timeout for the execution of long_running_function?
Basically I don't want long_running_function to last more than 2 seconds and I can't do proper timeout handling within it because that function comes from a third-party library.

A warning about cancelling long running functions:
Although wrapping the Future returned by loop.run_in_executor with an asyncio.wait_for call will allow the event loop to stop waiting for long_running_function after some x seconds, it won't necessarily stop the underlying long_running_function. This is one of the shortcomings of concurrent.futures and to the best of my knowledge there is no simple way to just cancel a concurrent.futures.Future.

You could use asyncio.wait_for:
future = loop.run_in_executor(None, long_running_function)
result = await asyncio.wait_for(future, timeout, loop=loop)

although not using run_in_executor, i have some workaround about "wrap a block function async with timeout handling"
import asyncio
import threading
import time
import ctypes
def terminate_thread(t: threading.Thread, exc_type=SystemExit):
if not t.is_alive(): return
try:
tid = next(tid for tid, tobj in threading._active.items() if tobj is t)
except StopIteration:
raise ValueError("tid not found")
if ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exc_type)) != 1:
raise SystemError("PyThreadState_SetAsyncExc failed")
class AsyncResEvent(asyncio.Event):
def __init__(self):
super().__init__()
self.res = None
self.is_exc = False
self._loop = asyncio.get_event_loop()
def set(self, data=None) -> None:
self.res = data
self.is_exc = False
self._loop.call_soon_threadsafe(super().set)
def set_exception(self, exc) -> None:
self.res = exc
self.is_exc = True
self._loop.call_soon_threadsafe(super().set)
async def wait(self, timeout: float | None = None):
await asyncio.wait_for(super().wait(), timeout)
if self.is_exc:
raise self.res
else:
return self.res
async def sub_thread_async(func, *args, _timeout: float | None = None, **kwargs):
res = AsyncResEvent()
def f():
try:
res.set(func(*args, **kwargs))
except Exception as e:
res.set_exception(e)
except SystemExit:
res.set_exception(TimeoutError)
(t := threading.Thread(target=f)).start()
try:
return await res.wait(_timeout)
except TimeoutError:
raise TimeoutError
finally:
if not res.is_set():
terminate_thread(t)
_lock = threading.Lock()
def test(n):
_tid = threading.get_ident()
for i in range(n):
with _lock:
print(f'print from thread {_tid} ({i})')
time.sleep(1)
return n
async def main():
res_normal = await asyncio.gather(*(sub_thread_async(test, 5) for _ in range(2)))
print(res_normal) # [5,5]
res_normal_2 = await asyncio.gather(*(sub_thread_async(test, 2, _timeout=3) for _ in range(2)))
print(res_normal_2) # [2,2]
res_should_not_get = await asyncio.gather(*(sub_thread_async(test, 5, _timeout=3) for _ in range(2)))
print(res_should_not_get) # timeout error
if __name__ == '__main__':
asyncio.new_event_loop().run_until_complete(main())