I'm using asyncio to fetch url's and sometimes they timeout, try as I might, I cannot catch the asyncio.TimeoutError with the following code!
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
async with session.get(url) as response:
# print(f"fetching {url}")
try:
resp = await response.read()
except asyncio.TimeoutError:
return {"results": f"timeout error on {url}"}
if response.status != 200:
return {"error": f"server returned {response.status}"}
return str(resp, 'utf-8').rstrip()
Here's the stack trace. What can I do to catch this exception and log it rather than exiting my program?
resource: {…}
severity: "ERROR"
textPayload: "Traceback (most recent call last):
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 346, in run_http_function
result = _function_handler.invoke_user_function(flask.request)
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 217, in invoke_user_function
return call_user_function(request_or_event)
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 210, in call_user_function
return self._user_function(request_or_event)
File "/user_code/main.py", line 230, in gcf_update_all_featured_podcasts
loop.run_until_complete(future) # loop until done
File "/opt/python3.7/lib/python3.7/asyncio/base_events.py", line 573, in run_until_complete
return future.result()
File "/user_code/main.py", line 201, in fetch_all
_ = await asyncio.gather(*tasks) # gather task responses
File "/user_code/main.py", line 181, in fetch
async with session.get(url) as response:
File "/env/local/lib/python3.7/site-packages/aiohttp/client.py", line 1005, in __aenter__
self._resp = await self._coro
File "/env/local/lib/python3.7/site-packages/aiohttp/client.py", line 497, in _request
await resp.start(conn)
File "/env/local/lib/python3.7/site-packages/aiohttp/client_reqrep.py", line 857, in start
self._continue = None
File "/env/local/lib/python3.7/site-packages/aiohttp/helpers.py", line 585, in __exit__
raise asyncio.TimeoutError from None
concurrent.futures._base.TimeoutError
In your traceback you can see that exception is raised from line where you are trying to make request to url, but your try block is on one level below.
Try like this:
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
try:
async with session.get(url) as response:
# print(f"fetching {url}")
resp = await response.read()
if response.status != 200:
return {"error": f"server returned {response.status}"}
return str(resp, 'utf-8').rstrip()
except asyncio.TimeoutError:
return {"results": f"timeout error on {url}"}
Adding to 4xel's solution
Its good idea to check status before reading.
async def fetch(url, session):
"""Fetch a url, using specified ClientSession."""
resp = None
try:
async with session.get(url) as response:
if response.status != 200:
resp ={"error": f"server returned {response.status}"}
else:
resp = await response.read()
resp = str(resp, 'utf-8').rstrip()
except asyncio.TimeoutError:
resp = {"results": f"timeout error on {url}"}
return resp
Related
I have an endpoint which saves uploaded image to it:
#router.post("/v1/installation/{installation_uuid}/image")
#db.create_connection
async def upload_installation_image(installation_uuid: UUID, request: Request):
content_type = request.headers["content-type"]
async with db.transaction():
installation = await get_installation_by_uuid(installation_uuid)
if not installation:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Installation {installation} not found")
try:
content = await request.body()
image_uuid = await save_installation_image(installation.uuid, content, content_type)
except Exception as e:
log.debug(f"An error raised while uploading image: {e}")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Error while uploading image to db"
)
return {"image_uuid": image_uuid}
save_installation_image make simple insert into DB.
In my tests I send 3 request to this endpoint. and all works fine:
tests.py
# upload image 1
data1 = b"\x01\x02\x03\x61\x05\x61"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data1, headers={"content-type": "image/png"}
)
# upload image 2
data2 = b"\x01\x02\x03\x61\x05\x62"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data2, headers={"content-type": "image/png"}
)
# upload image 3
data3 = b"\x01\x02\x03\x61\x05\x63"
response = session.post(
f"/v1/installation/{installation_uuid}/image", data=data3, headers={"content-type": "image/png"}
)
But when FE start calling that request, each request after first one start failing with this error:
Traceback (most recent call last):
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/protocols/http/httptools_impl.py", line 404, in run_asgi
result = await app( # type: ignore[func-returns-value]
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
return await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/message_logger.py", line 86, in __call__
raise exc from None
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
await self.app(scope, inner_receive, inner_send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/applications.py", line 270, in __call__
await super().__call__(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/applications.py", line 124, in __call__
await self.middleware_stack(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 184, in __call__
raise exc
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/errors.py", line 162, in __call__
await self.app(scope, receive, _send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 75, in __call__
raise exc
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 64, in __call__
await self.app(scope, receive, sender)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
raise e
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 680, in __call__
await route.handle(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 275, in handle
await self.app(scope, receive, send)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/starlette/routing.py", line 65, in app
response = await func(request)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/routing.py", line 231, in app
raw_response = await run_endpoint_function(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/fastapi/routing.py", line 160, in run_endpoint_function
return await dependant.call(**values)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 153, in wrapper
return await func(*args, **kwargs)
File "/opt/kelvatek/camlin-relink-broker/src/installation.py", line 347, in upload_installation_image
installation = await get_installation_by_uuid(installation_uuid)
File "/opt/kelvatek/camlin-relink-broker/src/installation.py", line 64, in get_installation_by_uuid
row = (await db.execute(query, installation_uuid=str(installation_uuid))).fetchone()
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 116, in execute
return await active_connection.execute(query, **params)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 50, in execute
return await t.execute(query, **params)
File "/opt/kelvatek/camlin-relink-broker/toolbox/asyncio_db.py", line 30, in execute
return await self.async_connection.execute(text(query).bindparams(**params))
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/ext/asyncio/engine.py", line 453, in execute
result = await greenlet_spawn(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 115, in greenlet_spawn
result = context.switch(*args, **kwargs)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1705, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 333, in _execute_on_connection
return connection._execute_clauseelement(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement
ret = self._execute_context(
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1798, in _execute_context
conn = self._revalidate_connection()
File "/opt/kelvatek/camlin-relink-broker/venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 650, in _revalidate_connection
raise exc.ResourceClosedError("This Connection is closed")
sqlalchemy.exc.ResourceClosedError: This Connection is closed
Also, when I change by data to data which send FE, it start failing for me as well. What may cause this error, and why if I send real images connection close after first request?
P.S. If I send just 1 image, it works fine, so problem in sending several requests.
UPDATES
This is how I manage connection to db:
#asynccontextmanager
async def connection(self):
active_connection = self.get_active_connection()
if active_connection:
yield active_connection
return
async with self.engine.connect() as conn:
try:
wrapper = ConnectionWrapper(conn)
token = self.active_connection.set(wrapper)
await self.set_connection_options()
if not self.collation_check_done:
await self.check_collation()
self.collation_check_done = True
yield wrapper
finally:
self.active_connection.reset(token)
def create_connection(self, func):
#wraps(func)
async def wrapper(*args, **kwargs):
async with self.connection():
return await func(*args, **kwargs)
return wrapper
This is my Wrappers classes:
class TransactionWrapper:
def __init__(self, async_connection, async_transaction):
self.async_connection = async_connection
self.async_transaction = async_transaction
self.after_commit_callbacks = []
def add_after_commit_callback(self, fn):
self.after_commit_callbacks.append(fn)
async def execute(self, query, **params):
return await self.async_connection.execute(text(query).bindparams(**params))
async def commit(self):
await self.async_transaction.commit()
while self.after_commit_callbacks:
fn = self.after_commit_callbacks.pop(0)
fn()
async def rollback(self):
await self.async_transaction.rollback()
class ConnectionWrapper:
def __init__(self, async_connection):
self.async_connection = async_connection
self.active_transaction = contextvars.ContextVar('active_transaction')
async def execute(self, query, **params):
t = self.get_active_transaction()
if t:
return await t.execute(query, **params)
# create temporary transaction in case it doesn't exists (simulates autocommit feature)
async with self.transaction() as t:
return await t.execute(query, **params)
def get_active_transaction(self):
try:
return self.active_transaction.get()
except LookupError:
return None
#asynccontextmanager
async def transaction(self):
# reuse existing transaction
active_transaction = self.get_active_transaction()
if active_transaction:
yield active_transaction
return
async with self.async_connection.begin() as async_transaction:
wrapper = TransactionWrapper(self.async_connection, async_transaction)
token = self.active_transaction.set(wrapper)
try:
yield wrapper
except Exception:
await wrapper.rollback()
raise
else:
await wrapper.commit()
finally:
self.active_transaction.reset(token)
Im trying to get binane.com 5 minute kline data and send data to excel.
My code
#imports section
#link to excel
pairlistcount = 321
async def factorial(x,y):
client = await AsyncClient.create()
bm = BinanceSocketManager(client)
symbol = pairs.range('A'+str(x)).value
# start any sockets here
ts = bm.kline_socket(symbol, interval=KLINE_INTERVAL_5MINUTE)
# then start receiving messages
async with ts as tscm:
while True:
do something
await client.close_connection()
args = [(str(g+1), g+1) for g in range(pairlistcount)]
tasks = itertools.starmap(factorial, args)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))
Sometimes it works perfectly, mostly it throws following error
most code deleted due to SO limit
return await self._request_api('get', path, signed, version, **kwargs)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\binance\client.py", line 7174, in _request_api
return await self._request(method, uri, signed, **kwargs)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\binance\client.py", line 7155, in _request
async with getattr(self.session, method)(uri, **kwargs) as response:
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 1138, in __aenter__
self._resp = await self._coro
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 559, in _request
await resp.start(conn)
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client_reqrep.py", line 913, in start
self._continue = None
File "C:\Users\ELCOT\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\helpers.py", line 721, in __exit__
raise asyncio.TimeoutError from None
asyncio.exceptions.TimeoutError
Please help with it.
Thanks in Advance.
I'm trying to use await for json method,but it tell me error.
Traceback (most recent call last):
File "D:/project/shoes_crawler/shoes_crawler/spiders/tes.py", line 73, in <module>
loop.run_until_complete(main2())
File "C:\Miniconda3\envs\py35\lib\asyncio\base_events.py", line 467, in run_until_complete
return future.result()
File "C:\Miniconda3\envs\py35\lib\asyncio\futures.py", line 294, in result
raise self._exception
File "C:\Miniconda3\envs\py35\lib\asyncio\tasks.py", line 240, in _step
result = coro.send(None)
File "D:/project/shoes_crawler/shoes_crawler/spiders/tes.py", line 50, in main2
js = await r.json()
File "C:\Miniconda3\envs\py35\lib\site-packages\aiohttp\client_reqrep.py", line 1021, in json
await self.read()
File "C:\Miniconda3\envs\py35\lib\site-packages\aiohttp\client_reqrep.py", line 973, in read
self._body = await self.content.read()
File "C:\Miniconda3\envs\py35\lib\site-packages\aiohttp\streams.py", line 334, in read
raise self._exception
aiohttp.client_exceptions.ClientConnectionError: Connection closed
Here is code.
async def get_web():
async with aiohttp.ClientSession() as session:
async with session.get(url) as r:
# print(r)
return r
async def main2():
r = await get_web()
if r.status == 200:
print('200')
js = await r.json()
#do someting with js
await asyncio.sleep(1)
loop = asyncio.get_event_loop()
loop.run_until_complete(main2())
Besides,anybody know how to let main2() run forever?I want to check the target web if it update new thing
I'm trying to rewrite my flask application to FastAPI which uses uvloop as the base.
When I execute code without the routes, I see no issue and executes as expected. But throws me error after I add the routes and hit a request using Postman.
> INFO: ('172.17.0.1', 42352) - "GET /api/v1/getpass HTTP/1.1" 500
> ERROR: Exception in ASGI application Traceback (most recent call
> last): File
> "/usr/local/lib/python3.6/site-packages/uvicorn/protocols/http/httptools_impl.py",
> line 375, in run_asgi
> result = await app(self.scope, self.receive, self.send) File "/usr/local/lib/python3.6/site-packages/starlette/applications.py",
> line 133, in __call__
> await self.error_middleware(scope, receive, send) File "/usr/local/lib/python3.6/site-packages/starlette/middleware/errors.py",
> line 177, in __call__
> raise exc from None File "/usr/local/lib/python3.6/site-packages/starlette/middleware/errors.py",
> line 155, in __call__
> await self.app(scope, receive, _send) File "/usr/local/lib/python3.6/site-packages/starlette/exceptions.py", line
> 73, in __call__
> raise exc from None File "/usr/local/lib/python3.6/site-packages/starlette/exceptions.py", line
> 62, in __call__
> await self.app(scope, receive, sender) File "/usr/local/lib/python3.6/site-packages/starlette/routing.py", line
> 590, in __call__
> await route(scope, receive, send) File "/usr/local/lib/python3.6/site-packages/starlette/routing.py", line
> 208, in __call__
> await self.app(scope, receive, send) File "/usr/local/lib/python3.6/site-packages/starlette/routing.py", line
> 41, in app
> response = await func(request) File "/usr/local/lib/python3.6/site-packages/fastapi/routing.py", line 111,
> in app
> raw_response = await dependant.call(**values) File "./iowait.py", line 51, in get_pvt
> print(kpair) UnboundLocalError: local variable 'kpair' referenced before assignment
My Code:
import random
import asyncio as aio
from fastapi import FastAPI
from pydantic import BaseModel
get_id = '/api/v1/getid'
get_pass = '/api/v1/getpass'
app = FastAPI()
key_pass_list = [
{
'uid' : 'uid1',
'pass' : 'pass1'
},
{
'uid' : 'uid2',
'pass' : 'pass2'
},
{
'uid' : 'uid3',
'pass' : 'pass3'
},
]
id_list = [_['uid'] for _ in key_pass_list]
class KeyDoc(BaseModel):
uid : str
async def key_pass(uid):
for keypair in key_pass_list:
if keypair['uid'] == uid:
return keypair
return {uid: 'Key not Found'}
#app.get(get_pass)
async def get_pvt(key_doc: KeyDoc):
uid = key_doc.uid
print('uid :' + uid)
try:
myloop = aio.get_event_loop()
kpair = myloop.run_until_complete(aio.wait([key_pass(uid)]))
await kpair
except Exception as err:
print(err)
print(kpair)
return None
#app.get(get_id)
async def get_pub():
return random.choice(id_list)
#app.get("/")
async def root():
return {"message": "Test-AIO"}
NOTE
get_id and / route works anytime because there is no logic, it fails with the get_pass requests.
Any help or pointers on this is very much appreciated.
Thank you
Two things here. Firstly, using aio.get_event_loop() is throwing an exception This event loop is already running before await kpair which throws the UnboundLocalError you're seeing when trying to print a non-existent variable. You can see this behavior by adding kpair = None under your first print statement in get_pvt. Below is how you would structure this using asyncio.wait.
#app.get(get_pass)
async def get_pvt(key_doc: KeyDoc):
uid = key_doc.uid
print('uid :' + uid)
done = set()
try:
done, pending = await aio.wait([key_pass(uid)])
except Exception as err:
print(err)
if done:
for task in done:
print(task.result())
return None
I'm new to asyncio and aiohttp. I am currently getting this error and not sure why I am getting InvalidStateError for my asyncio future and RuntimeError for my session:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 100, in <module>
sys.exit(main(sys.argv))
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/tests/tracer.py", line 92, in main
poster.post()
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 87, in post
results = event_loop.run_until_complete(self.async_post_events(events))
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/base_events.py", line 387, in run_until_complete
return future.result()
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 79, in async_post_events
task.add_done_callback(self.send_oracle, task.result(), session)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/futures.py", line 268, in result
raise InvalidStateError('Result is not ready.')
asyncio.futures.InvalidStateError: Result is not ready.
Task exception was never retrieved
future: <Task finished coro=<Poster.async_post_event() done, defined at /Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py:62> exception=RuntimeError('Session is closed',)>
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/asyncio/tasks.py", line 239, in _step
result = coro.send(None)
File "/Users/bli1/Development/QE/idea/trinity-tracer/tracer/utils/poster.py", line 64, in async_post_event
async with session.post(self.endpoint, data=event) as resp:
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 565, in __aenter__
self._resp = yield from self._coro
File "/Users/bli1/Development/QE/idea/trinity-tracer/lib/python3.5/site-packages/aiohttp/client.py", line 161, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed
What I am trying to do is POST to an endpoint, and then use the same event POSTed to post to another endpoint. This will be ran as another async method as a callback
Here is my code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
return event
async def send_oracle(self, event, session):
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
tasks = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
task = asyncio.ensure_future(self.async_post_event(event, session))
task.add_done_callback(self.send_oracle, task.result(), session)
tasks.append(task)
return await asyncio.gather(*tasks)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()
add_done_callback accepts a callback, not a coroutine.
Moreover it's a part of very low level API which should be avoided by a casual developer.
But your main mistake is calling session.post() outside of ClientSession async context manager, the stacktrace explicitly points on it.
I've modified your snippet for getting something which looks like a working code:
async def async_post_event(self, event, session):
async with session.post(self.endpoint, data=event) as resp:
event["tracer"]["post"]["timestamp"] = time.time() * 1000.0
event["tracer"]["post"]["statusCode"] = await resp.status
async with session.post(self.oracle, data=event) as resp:
return event["event"]["event_header"]["event_id"], await resp.status
async def async_post_events(self, events):
coros = []
conn = aiohttp.TCPConnector(verify_ssl=self.secure)
async with aiohttp.ClientSession(connector=conn) as session:
for event in events:
coros.append(self.async_post_event(event, session))
return await asyncio.gather(*coros)
def post(self):
event_loop = asyncio.get_event_loop()
try:
events = [self.gen_random_event() for i in range(self.num_post)]
results = event_loop.run_until_complete(self.async_post_events(events))
print(results)
finally:
event_loop.close()
You can extract two posts from async_post_event into separate coroutines but the main idea remains the same.