I am writing a helper class for handling multiple urls request in asynchronous way. The code is following.
class urlAsyncClient(object):
def __init__(self, url_arr):
self.url_arr = url_arr
async def async_worker(self):
result = await self.__run()
return result
async def __run(self):
pending_req = []
async with aiohttp.ClientSession() as session:
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#Awaiting the results altogether instead of one by one
result = await asyncio.wait(pending_req)
return result
#staticmethod
async def __fetch(session, url):
async with session.get(url) as response: #ERROR here
status_code = response.status
if status_code == 200:
return await response.json()
else:
result = await response.text()
print('Error ' + str(response.status_code) + ': ' + result)
return {"error": result}
As awaiting the result one by one seems meaningless in asynchronous. I put them into an array and wait together by await asyncio.wait(pending_req).
But seems like it is not the correct way to do it as I get the following error
in __fetch async with session.get(url) as response: RuntimeError: Session is closed
May I know the correct way to do it? Thanks.
because session has closed before you await it
async with aiohttp.ClientSession() as session:
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#session closed hear
you can make session an argument to __run, like this
async def async_worker(self):
async with aiohttp.ClientSession() as session:
result = await self.__run(session)
return result
# session will close hear
async def __run(self, session):
pending_req = []
for url in self.url_arr:
r = self.__fetch(session, url)
pending_req.append(r)
#Awaiting the results altogether instead of one by one
result = await asyncio.wait(pending_req)
return result
Related
import asyncio
from aiohttp import ClientSession
async def download(session: ClientSession, link: str):
print("init")
async with session.get(link) as resp:
print("hello")
data = await resp.content.read()
print("bye")
return
async def amain():
link = "https://www.google.com/"
async with ClientSession() as session:
tasks = []
for _ in range(100_000):
tasks.append(asyncio.create_task(download(session, link)))
await asyncio.sleep(0) # <------------ Label 1
await asyncio.gather(*tasks)
if __name__ == '__main__':
asyncio.run(amain())
If u comment "Label 1" line, u got unexpected output like:
init
init
...
...
init
and then mixed 'hello' and 'bye' as expected.
but if that line uncommented we got 'init' 'hello' 'bye' mixed as u expected from async tasks.
Can anybody explain me, why async with session.get(link) as resp: block task until all tasks created, if they created without await asyncio.sleep(0)?
↓↓↓ works unexpected too:
async def amain():
link = "https://www.google.com/"
async with ClientSession() as session:
await asyncio.gather(*(download(session, link) for _ in range(100_000)))
I'm trying to test an async request but I didn't find how to do. I tried with patch decorator, with AsyncMock... Everytime, I had either aexit error or AsyncMock can't be used in await expression... Where am I wrong ?
class RequestService:
async def requestPostPicture(self, session: aiohttp.ClientSession, photoData: dict):
try:
with aiohttp.MultipartWriter('form-data') as mpwriter:
part = mpwriter.append(photoData['file'][1],{'content-type': photoData['file'][2]})
part.set_content_disposition('form-data', name='file', filename=photoData['file'][0])
async with session.post('https://www.api-url.com', data=mpwriter, headers=self.headers) as resp:
if isinstance(resp, dict):
return resp
apiResponse = await resp.json
return apiResponse
except Exception as error:
return {'error': str(error)}
My test :
class TestRequestService(IsolatedAsyncioTestCase):
#patch('aiohttp.ClientSession.post')
async def testRequestPostPictureDict(self, mockPost):
mockPost.__aenter__.return_value = {"error": "test"}
requestservice = RequestService()
pictureTest = {'file': ('photodatatest.jpg', 'photodatatest', 'image/jpeg')}
connector = aiohttp.TCPConnector(limit=15)
async with aiohttp.ClientSession(connector=connector) as sessionPicture:
returnValue = await requestservice.requestPostPicture(sessionPicture, pictureTest)
self.assertEqual(returnValue, {'error': 'test'})
async def testRequestPostPictureDict(self):
mock = aiohttp.ClientSession
mock.post = MagicMock()
mock.post.return_value.__aenter__.return_value = {'error': 'test'}
I want to create simple async web app
the app.py looks like this
async def get_favicon(link: str):
return await _get_favicon(link)
async def handle(request):
url: Optional[str] = request.query['url']
result = await get_favicon(url)
return web.json_response(result, status=200)
async def init():
app = web.Application()
app.router.add_get("/", handle)
return app
if __name__ == "__main__":
application = init()
web.run_app(application, port=8000)
Inside of get_favicon i have a _get_favicon function
async def _get_favicon(self, url: str, biggest: bool = True, **request_kwargs) -> Dict:
request_kwargs.setdefault('headers', self.HEADERS)
request_kwargs.setdefault('allow_redirects', True)
request_kwargs.setdefault('verify', False)
response = await requests_async.get(url, **request_kwargs)
response.raise_for_status()
icons = set()
default_icon = await default(response.url, **request_kwargs)
and default fucntion
async def default(url: str, **request_kwargs: Dict) -> Optional[Icon]:
parsed = urlparse(url)
favicon_url = urlunparse((parsed.scheme, parsed.netloc, 'favicon.ico', '', '', ''))
response = await requests_async.head(favicon_url, **request_kwargs)
if response.status_code == 200:
return Icon(response.url, 0, 0, 'ico')
When i try to run it, i got RuntimeError: This event loop is already running in default function at response = await requests_async.head(favicon_url, **request_kwargs). I don't understand when i got wrong. I don't create another loop. If i run default_icon = self.default(response.url, **request_kwargs) without await i ll get coroutine object which is not exactly what i am looking for. I also tried to default_icon = await asyncio.gather(default(response.url, **request_kwargs)) and got the same error.
I want multi requests using aiohttp.
I was wrapping aiohttp like this, and i was test like this
my code
import asyncio
from aiohttp import ClientSession as AioClientSession
class ClientSession(AioClientSession):
async def _get(self, session, url, params=None, **kwargs):
async with session.get(url, params=params, **kwargs) as response:
return await response.json()
async def _post(self, session, url, data=None, **kwargs):
async with session.post(url, data=data, **kwargs) as response:
return await response.json()
async def fetch_all(self, method, urls, loop, data=None, params=None, **kwargs):
async with AioClientSession(loop=loop) as session:
if method == "GET":
results = await asyncio.gather(*[self._get(session, url, params=params, **kwargs) for url in urls])
elif method == "POST":
results = await asyncio.gather(*[self._post(session, url, data=data, **kwargs) for url in urls])
else:
assert False
return results
def multi_requests_get(urls, params=None, **kwargs):
session = ClientSession()
loop = asyncio.get_event_loop()
result = loop.run_until_complete(session.fetch_all("GET", urls, loop, params=params, **kwargs))
session.close()
return result
def multi_requests_post(urls, data=None, **kwargs):
session = ClientSession()
loop = asyncio.get_event_loop()
result = loop.run_until_complete(session.fetch_all("POST", urls, loop, data=data, **kwargs))
session.close()
return result
test code
urls = ["https://httpbin.org/get?{}={}".format(x, x) for x in range(10)]
result = multi_requests_get(urls=urls)
assert result
assert result[0]["args"] == {"0": "0"}
assert result[1]["args"] == {"1": "1"}
but this test return Warning like this:
The object should be created from async function
loop=loop)
How can I avoid this warning?
Here is the full traceback
============================================================================= warnings summary ==============================================================================
base/tests/test_aiohttp.py::AioHttpTest::test_get
/path/server/base/requests.py:122: DeprecationWarning: The object should be created from async function
session = ClientSession()
base/tests/test_aiohttp.py::AioHttpTest::test_get
base/tests/test_aiohttp.py::AioHttpTest::test_post
/env_path/lib/python3.6/site-packages/aiohttp/connector.py:730: DeprecationWarning: The object should be created from async function
loop=loop)
base/tests/test_aiohttp.py::AioHttpTest::test_get
base/tests/test_aiohttp.py::AioHttpTest::test_post
/env_path/lib/python3.6/site-packages/aiohttp/connector.py:735: DeprecationWarning: The object should be created from async function
resolver = DefaultResolver(loop=self._loop)
base/tests/test_aiohttp.py::AioHttpTest::test_get
base/tests/test_aiohttp.py::AioHttpTest::test_post
/env_path/lib/python3.6/site-packages/aiohttp/cookiejar.py:55: DeprecationWarning: The object should be created from async function
super().__init__(loop=loop)
base/tests/test_aiohttp.py::AioHttpTest::test_get
/path/darae/server/base/requests.py:125: RuntimeWarning: coroutine 'ClientSession.close' was never awaited
session.close()
base/tests/test_aiohttp.py::AioHttpTest::test_post
/path/server/base/requests.py:131: DeprecationWarning: The object should be created from async function
session = ClientSession()
base/tests/test_aiohttp.py::AioHttpTest::test_post
/path/server/base/requests.py:134: RuntimeWarning: coroutine 'ClientSession.close' was never awaited
session.close()
-- Docs: https://docs.pytest.org/en/latest/warnings.html
=================================================================== 2 passed, 10 warnings in 1.93 seconds ===================================================================
You may take a look at the simple working example from here and find a place where to place your aiohttp.ClientSession() as client:
import aiohttp
import asyncio
async def fetch(client):
async with client.get('http://python.org') as resp:
assert resp.status == 200
return await resp.text()
async def main():
async with aiohttp.ClientSession() as client:
html = await fetch(client)
print(html)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
aiohttp.ClientSession class MUST be instantiated inside coroutine function, not just a function.
All you need to do:
Remove parent class from your ClientSession - you already use it explicitly in your fetch_all and you don't need it there anymore.
Remove calls of session.close() - session DO close automatically by context manager in fetch_all.
Using Python 3.6 and asyncio and aiohttp I wrote a simple async program:
from aiohttp import ClientSession
import asyncio, ssl, time
base_url = 'https://my-base-url.com/api'
async def fetch(session, id):
query_params = {'qp1':'v1','qp2':'v2', 'id': id}
async with session.get(base_url, params=query_params, ssl=ssl.SSLContext()) as response:
res_json = await response.json()
if response.status == 200:
time.sleep(2)
min_rating = res_json.get('minRating')
max_rating = res_json.get('maxRating')
print("id = %s, min = %s, max = %s" % (id, min_rating, max_rating))
async def run(ids):
tasks = []
async with ClientSession() as session:
for id in ids:
task = asyncio.ensure_future(fetch(session, id))
tasks.append(task)
responses = await asyncio.gather(*tasks)
return responses
if __name__ == '__main__':
ids = [123, 456, 789]
future = asyncio.ensure_future(run(ids))
event_loop = asyncio.get_event_loop()
event_loop.run_until_complete(future)
print("\n\ndone")
The time.sleep(2) inside fetch(session, id) makes it seem like this program is not asynchronous because it gets one response, sleeps, gets another, sleeps, so on and so forth. When I remove the sleep call, it does seem to be async/concurrent because the responses come back in a random order. What is sleep doing in this case? Is it locking all threads? Why does it appear to be sequential instead of parallel?
time.sleep(2) is a synchronous (blocking) call hence you are stopping the asynchronous call with it, you should use await asyncio.sleep(2) which it will "liberate" the resource.