Develop Reference

Discord aiohttp.client_exceptions.ServerDisconnectedError when printing messages while getting console input to send messages to channel

I have tried to build code that simultaneously prints new messages to console and gets user input to send a message to a specified channel. When the send() function and on_message() started blocking each other, I put send() in a thread. However, when the code is run and the user inputs a message to be sent, the send fails and results in
Exception in thread Thread-2 (run):
Traceback (most recent call last):
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "main.py", line 45, in send
await channel.send(mes)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/discord/abc.py", line 1538, in send
data = await state.http.send_message(channel.id, params=params)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/discord/http.py", line 624, in request
async with self.__session.request(method, url, **kwargs) as response:
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client.py", line 1141, in __aenter__
self._resp = await self._coro
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client.py", line 560, in _request
await resp.start(conn)
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/client_reqrep.py", line 899, in start
message, payload = await protocol.read() # type: ignore[union-attr]
File "/home/runner/DiscordTest/venv/lib/python3.10/site-packages/aiohttp/streams.py", line 616, in read
await self._waiter
RuntimeError: Task <Task pending name='Task-13' coro=<send() running at main.py:45> cb=[_run_until_complete_cb() at /nix/store/hd4cc9rh83j291r5539hkf6qd8lgiikb-python3-3.10.8/lib/python3.10/asyncio/base_events.py:184]> got Future <Future pending> attached to a different loop
Future exception was never retrieved
future: <Future finished exception=ServerDisconnectedError('Server disconnected')>
aiohttp.client_exceptions.ServerDisconnectedError: Server disconnected
(Line 32)
Why is this happening?
My code is below:
import discord
from asyncio import sleep,run
from aioconsole import ainput
from ast import literal_eval as l
from threading import Thread
conn = connect('logs.db')
c = conn.cursor()
intents = discord.Intents.all()
intents.members = True
client = discord.Client(intents=intents,chunk_guilds_at_startup=False)
channel = 613
def to_message_string(author,channel,message,attachments):
base = ': '.join([author,channel,message])
if l(str(attachments)) != []:
base += '\n' + attachments
return base
async def send():
while True:
mes = await ainput('Enter a message: ')
if mes == '!change':
global channel
channel = await client.fetch_channel(int(await ainput('New channel: ')))
await channel.send(mes)
#client.event
async def on_ready():
for i in client.guilds:
for c in i.channels:
print(str(c.id) + ': ' + c.name)
global channel
channel = await client.fetch_channel(await ainput('Enter Channel ID: '))
z = Thread(target=run,args=(send(),))
z.start()
z.join()
#client.event
async def on_message(message):
mess,attachment_count = message.content,len(message.attachments)
author,channel,time = str(message.author.name),': '.join([message.guild.name,message.channel.name]),str(message.created_at)
if attachment_count != 0:
attachments = [(message.attachments[i]).url for i in range(attachment_count)]
else:
attachments = []
print(to_message_string(author,channel,mess,attachments))
client.run('token here')

Uploading image with FastAPI cause exc.ResourceClosedError

I have an endpoint which saves uploaded image to it:
#router.post("/v1/installation/{installation_uuid}/image")
#db.create_connection
async def upload_installation_image(installation_uuid: UUID, request: Request):
content_type = request.headers["content-type"]
async with db.transaction():
installation = await get_installation_by_uuid(installation_uuid)
if not installation:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Installation {installation} not found")
try:
content = await request.body()
image_uuid = await save_installation_image(installation.uuid, content, content_type)
except Exception as e:
log.debug(f"An error raised while uploading image: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error while uploading image to db"
)
return {"image_uuid": image_uuid}
save_installation_image make simple insert into DB.
In my tests I send 3 request to this endpoint. and all works fine:
tests.py
# upload image 1
data1 = b"\x01\x02\x03\x61\x05\x61"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data1, headers={"content-type": "image/png"}
)
# upload image 2
data2 = b"\x01\x02\x03\x61\x05\x62"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data2, headers={"content-type": "image/png"}
)
# upload image 3
data3 = b"\x01\x02\x03\x61\x05\x63"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data3, headers={"content-type": "image/png"}
)
But when FE start calling that request, each request after first one start failing with this error:
Traceback (most recent call last):
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/protocols/http/httptools_impl.py", line 404, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/message_logger.py", line 86, in __call__
raise exc from None
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
await self.app(scope, inner_receive, inner_send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/applications.py", line 270, in __call__
await super().__call__(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 184, in __call__
raise exc
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 75, in __call__
raise exc
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 64, in __call__
await self.app(scope, receive, sender)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
raise e
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 680, in __call__
await route.handle(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 275, in handle
await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 65, in app
response = await func(request)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/routing.py", line 231, in app
raw_response = await run_endpoint_function(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function
return await dependant.call(**values)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 153, in wrapper
return await func(*args, **kwargs)
File "/opt/kelvatek/camlin-relink-broker/src/installation.py", line 347, in upload_installation_image
installation = await get_installation_by_uuid(installation_uuid)
File "/opt/kelvatek/camlin-relink-broker/src/installation.py", line 64, in get_installation_by_uuid
row = (await db.execute(query, installation_uuid=str(installation_uuid))).fetchone()
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 116, in execute
return await active_connection.execute(query, **params)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 50, in execute
return await t.execute(query, **params)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 30, in execute
return await self.async_connection.execute(text(query).bindparams(**params))
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/ext/asyncio/engine.py", line 453, in execute
result = await greenlet_spawn(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 115, in greenlet_spawn
result = context.switch(*args, **kwargs)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1705, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 333, in _execute_on_connection
return connection._execute_clauseelement(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement
ret = self._execute_context(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1798, in _execute_context
conn = self._revalidate_connection()
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 650, in _revalidate_connection
raise exc.ResourceClosedError("This Connection is closed")
sqlalchemy.exc.ResourceClosedError: This Connection is closed
Also, when I change by data to data which send FE, it start failing for me as well. What may cause this error, and why if I send real images connection close after first request?
P.S. If I send just 1 image, it works fine, so problem in sending several requests.
UPDATES
This is how I manage connection to db:
#asynccontextmanager
async def connection(self):
active_connection = self.get_active_connection()
if active_connection:
yield active_connection
return
async with self.engine.connect() as conn:
try:
wrapper = ConnectionWrapper(conn)
token = self.active_connection.set(wrapper)
await self.set_connection_options()
if not self.collation_check_done:
await self.check_collation()
self.collation_check_done = True
yield wrapper
finally:
self.active_connection.reset(token)
def create_connection(self, func):
#wraps(func)
async def wrapper(*args, **kwargs):
async with self.connection():
return await func(*args, **kwargs)
return wrapper
This is my Wrappers classes:
class TransactionWrapper:
def __init__(self, async_connection, async_transaction):
self.async_connection = async_connection
self.async_transaction = async_transaction
self.after_commit_callbacks = []
def add_after_commit_callback(self, fn):
self.after_commit_callbacks.append(fn)
async def execute(self, query, **params):
return await self.async_connection.execute(text(query).bindparams(**params))
async def commit(self):
await self.async_transaction.commit()
while self.after_commit_callbacks:
fn = self.after_commit_callbacks.pop(0)
fn()
async def rollback(self):
await self.async_transaction.rollback()
class ConnectionWrapper:
def __init__(self, async_connection):
self.async_connection = async_connection
self.active_transaction = contextvars.ContextVar('active_transaction')
async def execute(self, query, **params):
t = self.get_active_transaction()
if t:
return await t.execute(query, **params)
# create temporary transaction in case it doesn't exists (simulates autocommit feature)
async with self.transaction() as t:
return await t.execute(query, **params)
def get_active_transaction(self):
try:
return self.active_transaction.get()
except LookupError:
return None
#asynccontextmanager
async def transaction(self):
# reuse existing transaction
active_transaction = self.get_active_transaction()
if active_transaction:
yield active_transaction
return
async with self.async_connection.begin() as async_transaction:
wrapper = TransactionWrapper(self.async_connection, async_transaction)
token = self.active_transaction.set(wrapper)
try:
yield wrapper
except Exception:
await wrapper.rollback()
raise
else:
await wrapper.commit()
finally:
self.active_transaction.reset(token)

Python binance asyncio.exceptions.TimeoutError

Im trying to get binane.com 5 minute kline data and send data to excel.
My code
#imports section
#link to excel
pairlistcount = 321
async def factorial(x,y):
client = await AsyncClient.create()
bm = BinanceSocketManager(client)
symbol = pairs.range('A'+str(x)).value
# start any sockets here
ts = bm.kline_socket(symbol, interval=KLINE_INTERVAL_5MINUTE)
# then start receiving messages
async with ts as tscm:
while True:
do something
await client.close_connection()
args = [(str(g+1), g+1) for g in range(pairlistcount)]
tasks = itertools.starmap(factorial, args)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))
Sometimes it works perfectly, mostly it throws following error
most code deleted due to SO limit
return await self._request_api('get', path, signed, version, **kwargs)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\binance\client.py", line 7174, in _request_api
return await self._request(method, uri, signed, **kwargs)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\binance\client.py", line 7155, in _request
async with getattr(self.session, method)(uri, **kwargs) as response:
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 1138, in __aenter__
self._resp = await self._coro
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 559, in _request
await resp.start(conn)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client_reqrep.py", line 913, in start
self._continue = None
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
Please help with it.
Thanks in Advance.

impossible to catch asyncio.TimeoutError?

I'm using asyncio to fetch url's and sometimes they timeout, try as I might, I cannot catch the asyncio.TimeoutError with the following code!
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
async with session.get(url) as response:
# print(f"fetching {url}")
try:
resp = await response.read()
except asyncio.TimeoutError:
return {"results": f"timeout error on {url}"}
if response.status != 200:
return {"error": f"server returned {response.status}"}
return str(resp, 'utf-8').rstrip()
Here's the stack trace. What can I do to catch this exception and log it rather than exiting my program?
resource: {…}
severity: "ERROR"
textPayload: "Traceback (most recent call last):
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 346, in run_http_function
result = _function_handler.invoke_user_function(flask.request)
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 217, in invoke_user_function
return call_user_function(request_or_event)
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 210, in call_user_function
return self._user_function(request_or_event)
File "/user_code/main.py", line 230, in gcf_update_all_featured_podcasts
loop.run_until_complete(future) # loop until done
File "/opt/python3.7/lib/python3.7/asyncio/base_events.py", line 573, in run_until_complete
return future.result()
File "/user_code/main.py", line 201, in fetch_all
_ = await asyncio.gather(*tasks) # gather task responses
File "/user_code/main.py", line 181, in fetch
async with session.get(url) as response:
File "/env/local/lib/python3.7/site-packages/aiohttp/client.py", line 1005, in __aenter__
self._resp = await self._coro
File "/env/local/lib/python3.7/site-packages/aiohttp/client.py", line 497, in _request
await resp.start(conn)
File "/env/local/lib/python3.7/site-packages/aiohttp/client_reqrep.py", line 857, in start
self._continue = None
File "/env/local/lib/python3.7/site-packages/aiohttp/helpers.py", line 585, in __exit__
raise asyncio.TimeoutError from None
concurrent.futures._base.TimeoutError

In your traceback you can see that exception is raised from line where you are trying to make request to url, but your try block is on one level below.
Try like this:
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
try:
async with session.get(url) as response:
# print(f"fetching {url}")
resp = await response.read()
if response.status != 200:
return {"error": f"server returned {response.status}"}
return str(resp, 'utf-8').rstrip()
except asyncio.TimeoutError:
return {"results": f"timeout error on {url}"}

Adding to 4xel's solution
Its good idea to check status before reading.
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
resp = None
try:
async with session.get(url) as response:
if response.status != 200:
resp ={"error": f"server returned {response.status}"}
else:
resp = await response.read()
resp = str(resp, 'utf-8').rstrip()
except asyncio.TimeoutError:
resp = {"results": f"timeout error on {url}"}
return resp

InvalidStateError with asyncio futures and RuntimeError with aiohttp when using Futures with callback

I'm new to asyncio and aiohttp. I am currently getting this error and not sure why I am getting InvalidStateError for my asyncio future and RuntimeError for my session:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 100, in <module>
sys.exit(main(sys.argv))
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 92, in main
poster.post()
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 87, in post
results = event_loop.run_until_complete(self.async_post_events(events))
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/base_events.py", line 387, in run_until_complete
return future.result()
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 79, in async_post_events
task.add_done_callback(self.send_oracle, task.result(), session)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 268, in result
raise InvalidStateError('Result is not ready.')
asyncio.futures.InvalidStateError: Result is not ready.
Task exception was never retrieved
future: <Task finished coro=<Poster.async_post_event() done, defined at /Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py:62> exception=RuntimeError('Session is closed',)>
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 64, in async_post_event
async with session.post(self.endpoint, data=event) as resp:
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 565, in __aenter__
self._resp = yield from self._coro
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 161, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed
What I am trying to do is POST to an endpoint, and then use the same event POSTed to post to another endpoint. This will be ran as another async method as a callback
Here is my code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
return event
async def send_oracle(self, event, session):
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
tasks = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
task = asyncio.ensure_future(self.async_post_event(event, session))
task.add_done_callback(self.send_oracle, task.result(), session)
tasks.append(task)
return await asyncio.gather(*tasks)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()

add_done_callback accepts a callback, not a coroutine.
Moreover it's a part of very low level API which should be avoided by a casual developer.
But your main mistake is calling session.post() outside of ClientSession async context manager, the stacktrace explicitly points on it.
I've modified your snippet for getting something which looks like a working code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
coros = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
coros.append(self.async_post_event(event, session))
return await asyncio.gather(*coros)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()
You can extract two posts from async_post_event into separate coroutines but the main idea remains the same.