I'm trying to complete all the tasks with "gather" and then signal that the Loop is stopped and close it. But the application gives an error after it prints "Stopped"
How can i fix it? Here is my code:
async def _main_loop_wrapper(self):
try:
await self._main_loop()
except asyncio.CancelledError:
pass
except Exception:
self.stop()
raise
async def _main_loop(self):
queue = asyncio.Queue(maxsize=self._concurrent_handlers_count)
await self._on_start(queue)
while not self.shuting_down:
await self._sem.acquire()
update, backend = await queue.get()
ctx = await Context.create(
engine=self,
config=self.config,
update=update,
backend=backend,
)
backend.prepare_context(ctx)
task = asyncio.create_task(self._handle_update_with_logger(update, ctx))
task.add_done_callback(lambda t: self._sem.release())
async def _shutdown(self):
logger.info("Gracecfully shutting down...")
# Clean up
tasks = []
tasks.append(self._backend.on_shutdown(self))
tasks.append(self._handle_event("shutdown"))
await asyncio.gather(*tasks, return_exceptions=True)
# Cancel all tasks
tasks = []
for task in asyncio.all_tasks(loop=self._loop):
if task is not asyncio.current_task():
tasks.append(task)
task.cancel()
# Wait for all tasks to be cancelled
await asyncio.gather(*tasks, return_exceptions=True)
self._loop.stop()
def run(self):
"""Run the application."""
logger.info(f"Total plugins: {len(self._plugins)}")
logger.info("Starting...")
if platform.lower().startswith("win"):
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# Set up the signal handlers
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
self._loop.create_task(self._main_loop_wrapper())
self._loop.run_forever()
self._loop.close()
logger.info("Stopped...")
sys.exit(0)
def stop(self, sig=None, frame=None):
"""Stop the application."""
# Check if shuting down is already started
if self.shuting_down:
return
self.shuting_down = True
asyncio.create_task(self._shutdown())
Here is output log:
kutana.py [ L:210 ]# INFO [21.01.23 14:52:16] Gracecfully shutting down...
kutana.py [ L:249 ]# INFO [21.01.23 14:52:16] Stopped...
Exception ignored in: <function _ProactorBasePipeTransport.__del__ at 0x00000242EA9B48B0>
Traceback (most recent call last):
File "C:\Users\root\AppData\Local\Programs\Python\Python39\lib\asyncio\proactor_events.py", line 116, in __del__
self.close()
File "C:\Users\root\AppData\Local\Programs\Python\Python39\lib\asyncio\proactor_events.py", line 108, in close
self._loop.call_soon(self._call_connection_lost, None)
File "C:\Users\root\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 746, in call_soon
self._check_closed()
File "C:\Users\root\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
I tried putting self._loop.close() into the _shutdown function, but it didn't work
Related
I have tried to build code that simultaneously prints new messages to console and gets user input to send a message to a specified channel. When the send() function and on_message() started blocking each other, I put send() in a thread. However, when the code is run and the user inputs a message to be sent, the send fails and results in
Exception in thread Thread-2 (run):
Traceback (most recent call last):
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "main.py", line 45, in send
await channel.send(mes)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/discord/abc.py", line 1538, in send
data = await state.http.send_message(channel.id, params=params)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/discord/http.py", line 624, in request
async with self.__session.request(method, url, **kwargs) as response:
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client.py", line 1141, in __aenter__
self._resp = await self._coro
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client.py", line 560, in _request
await resp.start(conn)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client_reqrep.py", line 899, in start
message, payload = await protocol.read() # type: ignore[union-attr]
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/streams.py", line 616, in read
await self._waiter
RuntimeError: Task <Task pending name='Task-13' coro=<send() running at main.py:45> cb=[_run_until_complete_cb() at /nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/base_events.py:184]> got Future <Future pending> attached to a different loop
Future exception was never retrieved
future: <Future finished exception=ServerDisconnectedError('Server disconnected')>
aiohttp.client_exceptions.ServerDisconnectedError: Server disconnected
(Line 32)
Why is this happening?
My code is below:
import discord
from asyncio import sleep,run
from aioconsole import ainput
from ast import literal_eval as l
from threading import Thread
conn = connect('logs.db')
c = conn.cursor()
intents = discord.Intents.all()
intents.members = True
client = discord.Client(intents=intents,chunk_guilds_at_startup=False)
channel = 613
def to_message_string(author,channel,message,attachments):
base = ': '.join([author,channel,message])
if l(str(attachments)) != []:
base += '\n' + attachments
return base
async def send():
while True:
mes = await ainput('Enter a message: ')
if mes == '!change':
global channel
channel = await client.fetch_channel(int(await ainput('New channel: ')))
await channel.send(mes)
#client.event
async def on_ready():
for i in client.guilds:
for c in i.channels:
print(str(c.id) + ': ' + c.name)
global channel
channel = await client.fetch_channel(await ainput('Enter Channel ID: '))
z = Thread(target=run,args=(send(),))
z.start()
z.join()
#client.event
async def on_message(message):
mess,attachment_count = message.content,len(message.attachments)
author,channel,time = str(message.author.name),': '.join([message.guild.name,message.channel.name]),str(message.created_at)
if attachment_count != 0:
attachments = [(message.attachments[i]).url for i in range(attachment_count)]
else:
attachments = []
print(to_message_string(author,channel,mess,attachments))
client.run('token here')
If you look at the code:
import asyncio
import datetime as dt
def get_results_or_none(tasks):
results = []
for task in tasks:
if task.cancelled():
print(task.get_name(), 'cancelled')
results.append(None)
else:
if task.exception():
print(task.get_name(), task.exception())
results.append(None)
else:
print(task.get_name(), 'result', task.result())
results.append(task.result())
return results
async def main():
tasks = [
asyncio.create_task(asyncio.sleep(1), name='task1'),
asyncio.create_task(asyncio.sleep(3), name='task2'),
]
await asyncio.wait(tasks, timeout=2)
return get_results_or_none(tasks)
start_time = dt.datetime.now()
task1, task2 = asyncio.run(main(), debug=True)
total_time = (dt.datetime.now() - start_time).total_seconds()
print('lasted', total_time)
print(task1)
print(task2)
You can see that task2 is intentionally too long and causes timeout. I hoped that it will be enough to just retrieve task.exception(), but I get an output like this one:
task1 result None
Traceback (most recent call last):
File "/home/user/project/chk_invalid_state_error.py", line 31, in <module>
task1, task2 = asyncio.run(main(), debug=True)
File "/home/user/miniconda3/envs/algot/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/home/user/miniconda3/envs/algot/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
return future.result()
File "/home/user/project/chk_invalid_state_error.py", line 28, in main
return get_results_or_none(tasks)
File "/home/user/project/chk_invalid_state_error.py", line 12, in get_results_or_none
if task.exception():
asyncio.exceptions.InvalidStateError: Exception is not set.
What does it mean "asyncio.exceptions.InvalidStateError: Exception is not set." in this case?
Thast error is raised if you check for exception in a task that is not done yet. (The call to .wait() timed out,but the task is still running).
You have to check if it is done, by calling the .done() method before calling .exception(): https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.exception
I've been trying to understanding asynchronous programming in Python. I tried writing a simple throttler that allows only rate_limit number of tasks to be processed at a time.
Here's the implementation:
import asyncio
import time
class Throttler:
def __init__ (
self,
rate_limit: int,
retry_interval: float
) -> None:
self.rate_limit = rate_limit
self.retry_interval = retry_interval
self._time_counter = None
self._tasks_counter = 0
self.lock = asyncio.Lock()
async def __aenter__(
self
) -> 'Throttler':
async with self.lock:
print(f'Starting {self._tasks_counter}')
if self._time_counter is None:
pass
else:
difference = time.perf_counter() - self._time_counter
# if difference < self.retry_interval:
# await asyncio.sleep(self.retry_interval - difference)
while True:
if self._tasks_counter < self.rate_limit:
break
else:
print('here')
await asyncio.sleep(self.retry_interval)
if self._time_counter is not None:
print(time.perf_counter() - self._time_counter)
self._time_counter = time.perf_counter()
self._tasks_counter += 1
return self
async def __aexit__(
self,
exc_type,
exc_val,
exc_tb
) -> None:
async with self.lock:
self._tasks_counter -= 1
print(f'Ending {self._tasks_counter}')
throttler = Throttler(rate_limit = 5, retry_interval = 2.0)
async def f ():
async with throttler:
print(42)
await asyncio.sleep(1)
async def main ():
await asyncio.gather(*[f() for i in range(10)])
asyncio.run(main())
I expected that as rate_limit is set to 5 when I declare the Throttler, it should process at most 5 requests at a time and then wait until one or more of them finish to begin processing other requests. But it doesn't work as I expect it to and raises a RuntimeError as soon as it encounters one of the asyncio.sleep statements (even the commented one if you uncomment it).
Here's the complete traceback:
Traceback (most recent call last):
File "C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py", line 288, in f
await asyncio.sleep(1)
File "C:\Programming\Python\lib\asyncio\locks.py", line 120, in acquire
await fut
RuntimeError: Task <Task pending name='Task-3' coro=<f() running at C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py:288> cb=[gather.<locals>._done_callback() at C:\Programming\Python\lib\asyncio\tasks.py:766, gather.<locals>._done_callback() at C:\Programming\Python\lib\asyncio\tasks.py:766]> got Future <Future pending> attached to a different loop
Traceback (most recent call last):
File "C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py", line 293, in <module>
asyncio.run(main())
File "C:\Programming\Python\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Programming\Python\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py", line 291, in main
await asyncio.gather(*[f() for i in range(10)])
File "C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py", line 286, in f
async with throttler:
File "C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py", line 247, in __aenter__
async with self.lock:
File "C:\Programming\Python\lib\asyncio\locks.py", line 14, in __aenter__
await self.acquire()
File "C:\Programming\Python\lib\asyncio\locks.py", line 120, in acquire
await fut
RuntimeError: Task <Task pending name='Task-8' coro=<f() running at C:\Users\Aryan V S\Desktop\Projects\General\Python\Other\Async\throttle.py:286> cb=[gather.<locals>._done_callback() at C:\Programming\Python\lib\asyncio\tasks.py:766]> got Future <Future pending> attached to a different loop
What am I doing wrong here? Does asyncio.sleep not use the currently running event loop or am I not understanding how this works? Thanks a lot for your time!
This is because you are creating the lock before the asyncio.run runs a loop, try creating the throttler in main() and sending it to f().
I am currently working on a script for some web scraping related activity and recently decided to move to asyncio on python 3.5.2 version.
There are 3 coroutines producer, processor and consumer each of them is infinitely running task.
Also inside processor and producer I have to make network request so I want to spawn those requests in separate thread and wait on them.
Though it would have been better if I could possibly also create Pool for making external requests at places loop.run_in_executor which I am not able to achieve currently.
The script for the producer consumer and processor is
import feedparser
import asyncio
def getEventLoop():
loop = None
try:
loop = asyncio.get_event_loop()
except:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
return loop
loop = getEventLoop()
link_queue = asyncio.Queue(100)
item_queue = asyncio.Queue(200)
async def addItem(item):
#Saving into database
pass
async def processor():
# logger.info("Started processor")
while True:
news_job = await link_queue.get()
link, source = news_job
try:
parsedData = await loop.run_in_executor(None, feedparser.parse, link)
map(item_queue.put, parsedData.get("items"))
except Exception as e:
# logger.error("processor {} from {}".format(link, source), e)
pass
finally:
link_queue.task_done()
async def consumer():
# logger.info("Started consumer")
while True:
item = await item_queue.get()
try:
await addItem(item)
except Exception as e:
# logger.error("consumer {}".format(item.get("link")), e)
pass
finally:
item_queue.task_done()
async def getItems(item_category):
baseUrl = "https://www.example.com/"
req = await loop.run_in_executor(None, requests.get, baseUrl+item_category)
try:
if req.status_code == 200:
return [req.text.split(" ")] # Only dummy showcased
except Exception as e:
# logger.error("Exception in getItems for category {}".format(item_category), e)
pass
return []
async def produce_link(item_category):
try:
itemlist =[ item for item in await getItems(item_category)]
await asyncio.gather(map(link_queue.put, itemList))
return True
except Exception as e:
# logger.error("Exception produce_link for {}".format(item_category), e)
return False
async def producer():
while True:
item_category_list = ["a","b"] # Some dummy item categories
def getAllItems():
loop.run_until_complete(asyncio.gather(*[ produce_link(item_category) for item_category in item_category_list]))
result = await loop.run_in_executor(None, getAllItems)
asyncio.sleep(random.randint(10,60))
# logger.info("Stopped producer")
def run():
loop.run_until_complete(asyncio.gather(producer(), processor(), consumer()))
loop.close()
I am getting following error RuntimeError: There is no current event loop in thread 'Thread-1'
traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "D:\Projects\alphine\testscript.py", line 78, in run
loop.run_until_complete(asyncio.gather(produce(), processor(), consumer()))
File "C:\Python35\Lib\asyncio\base_events.py", line 387, in run_until_complete
return future.result()
File "C:\Python35\Lib\asyncio\futures.py", line 274, in result
raise self._exception
File "C:\Python35\Lib\asyncio\tasks.py", line 241, in _step
result = coro.throw(exc)
File "D:\Projects\alphine\testscript.py", line 73, in produce
result = await loop.run_in_executor(None, getAllItems)
File "C:\Python35\Lib\asyncio\futures.py", line 361, in __iter__
yield self # This tells Task to wait for completion.
File "C:\Python35\Lib\asyncio\tasks.py", line 296, in _wakeup
future.result()
File "C:\Python35\Lib\asyncio\futures.py", line 274, in result
raise self._exception
File "C:\Python35\Lib\concurrent\futures\thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
File "D:\Projects\alphine\testscript.py", line 72, in getAllItems
loop.run_until_complete(asyncio.gather(*[ produce_link(item_category) for item_category in item_category_list]))
File "C:\Python35\Lib\asyncio\tasks.py", line 617, in gather
fut = ensure_future(arg, loop=loop)
File "C:\Python35\Lib\asyncio\tasks.py", line 546, in ensure_future
loop = events.get_event_loop()
File "C:\Python35\Lib\asyncio\events.py", line 632, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "C:\Python35\Lib\asyncio\events.py", line 578, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-1'.
What is the root problem in my script which is causing this error?
Any other inputs are also welcomed.
I'm new to asyncio and aiohttp. I am currently getting this error and not sure why I am getting InvalidStateError for my asyncio future and RuntimeError for my session:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 100, in <module>
sys.exit(main(sys.argv))
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 92, in main
poster.post()
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 87, in post
results = event_loop.run_until_complete(self.async_post_events(events))
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/base_events.py", line 387, in run_until_complete
return future.result()
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 79, in async_post_events
task.add_done_callback(self.send_oracle, task.result(), session)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 268, in result
raise InvalidStateError('Result is not ready.')
asyncio.futures.InvalidStateError: Result is not ready.
Task exception was never retrieved
future: <Task finished coro=<Poster.async_post_event() done, defined at /Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py:62> exception=RuntimeError('Session is closed',)>
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 64, in async_post_event
async with session.post(self.endpoint, data=event) as resp:
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 565, in __aenter__
self._resp = yield from self._coro
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 161, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed
What I am trying to do is POST to an endpoint, and then use the same event POSTed to post to another endpoint. This will be ran as another async method as a callback
Here is my code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
return event
async def send_oracle(self, event, session):
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
tasks = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
task = asyncio.ensure_future(self.async_post_event(event, session))
task.add_done_callback(self.send_oracle, task.result(), session)
tasks.append(task)
return await asyncio.gather(*tasks)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()
add_done_callback accepts a callback, not a coroutine.
Moreover it's a part of very low level API which should be avoided by a casual developer.
But your main mistake is calling session.post() outside of ClientSession async context manager, the stacktrace explicitly points on it.
I've modified your snippet for getting something which looks like a working code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
coros = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
coros.append(self.async_post_event(event, session))
return await asyncio.gather(*coros)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()
You can extract two posts from async_post_event into separate coroutines but the main idea remains the same.