Example
Here's my code:
from typing import List
from fastapi import FastAPI, File, UploadFile
import asyncio
import concurrent.futures
app = FastAPI()
#app.post("/send_images")
async def update_item(
files: List[UploadFile] = File(...),
):
return {"res": len(files)}
And I send requests to this server with (these specific urls are just for example here):
import requests
import os
import json
import numpy as np
import time
import io
import httpx
import asyncio
from datetime import datetime
import pandas as pd
from random import shuffle
import pandas as pd
import urllib
import io
from PIL import Image
from matplotlib import pyplot as plt
import concurrent.futures
urls = ['https://sun9-63.userapi.com/c638920/v638920705/1a54d/xSREwpakJD4.jpg',
'https://sun9-28.userapi.com/c854024/v854024084/1160d8/LDMVHYgguAw.jpg',
'https://sun9-54.userapi.com/c854220/v854220084/111f66/LdcbEpPR6tg.jpg',
'https://sun9-40.userapi.com/c841420/v841420283/4c8bb/Mii6GSCrmpo.jpg',
'https://sun6-16.userapi.com/CPQpllJ0KtaArvQKkPsHTZDCupqjRJ_8l07ejA/iyg2hRR_kM4.jpg',
'https://sun9-1.userapi.com/c638920/v638920705/1a53b/SMta6Bv-k7s.jpg',
'https://sun9-36.userapi.com/c857332/v857332580/56ad/rJCGKFw03FQ.jpg',
'https://sun6-14.userapi.com/iPsfmW0ibE8RsMh0k2lUFdRxHZ4Q41yctB7L3A/ajJHY3WN6Xg.jpg',
'https://sun9-28.userapi.com/c854324/v854324383/1c1dc3/UuFigBF7WDI.jpg',
'https://sun6-16.userapi.com/UVXVAT-tYudG5_24FMaBWTB9vyW8daSrO2WPFQ/RMjv7JZvowA.jpg']
os.environ['NO_PROXY'] = '127.0.0.1'
async def request_get_4(list_urls):
async with httpx.AsyncClient() as client:
r = httpx.post("http://127.0.0.1:8001/send_images", files={f'num_{ind}': el for ind, el in enumerate(list_urls)})
print(r.text)
return r
async def request_get_3(url):
async with httpx.AsyncClient() as client:
return await client.get(url)
from collections import defaultdict
async def main():
start = datetime.now()
tasks = [asyncio.create_task(request_get_3(url)) for url in urls[0:10]]
result = await asyncio.gather(*tasks)
data_to_send = []
for ind, resp in enumerate(result):
if resp.status_code == 200:
image_bytes = io.BytesIO(resp.content)
image_bytes.seek(0)
data_to_send.append(image_bytes)
end = datetime.now()
print(result)
print(len(data_to_send))
batch_size = 2
batch_num = len(data_to_send) // batch_size
tasks = [asyncio.create_task(request_get_4(data_to_send[i * batch_size: (i+1) * batch_size])) for i in range(batch_num)]
result = await asyncio.gather(*tasks)
left_data = data_to_send[batch_size*(batch_num):]
print(len(left_data))
print(result)
asyncio.run(main())
I am trying to load image which are contained in urls, then form batches of them and send them to FastAPI server. But it doesn't work.
I get the following error:
{"detail":[{"loc":["body","files"],"msg":"field required","type":"value_error.missing"}]}
How can I fix my problem and be able to send multiple files through httpx to FastAPI?
The problem is that HTTPX 0.13.3 doesn't support multiple file uploading as this arg wants dictionary and dictionary can't have same key values.
We can use this pull request https://github.com/encode/httpx/pull/1032/files to fix this problem (now it can also accept List[Tuple[str, FileTypes]]])!
UPDATE:
this problem is solved now!
Update2:
Here I show how I use httpx for different requests:
async def request_post_batch(fastapi_url: str, url_content_batch: List[BinaryIO]) -> httpx.Response:
"""
Send batch to FastAPI server.
"""
async with httpx.AsyncClient(timeout=httpx.Timeout(100.0)) as client:
r = await client.post(
fastapi_url,
files=[('bytes_image', url_content) for url_content in url_content_batch]
)
return r
async def request_post_logs(logstash_url: str, logs: List[Dict]) -> httpx.Response:
"""
Send logs to logstash
"""
async with httpx.AsyncClient(timeout=httpx.Timeout(100.0)) as client:
r = await client.post(
logstash_url,
json=logs
)
return r
This example will pass multiple files under the files field.
async with httpx.AsyncClient(timeout=httpx.Timeout(100.0)) as client:
response = await client.post(f"/api/v1/upload-files",
files=[("files", ("image.png", b"{}", "image/png")), ("files", ("image2.png", b"{}", "image/png"))],
)
Related
I have a python code and I want to speed it up using threads but when I try to I get the same lines getting duplicated, is there is any way I could speed it up without getting duplicate lines
code
import requests
import json
f = open("urls.json")
data = json.load(f)
def urls():
for i in data['urls']:
r = requests.get("https://" + i)
print(r.headers)
You can use ThreadPoolExecutor class from concurrent.futures. It is efficient way according to Thread class.
You can change the max_workers value according to your task
Here is the piece of code:
import requests
from concurrent.futures import ThreadPoolExecutor
import json
with open("urls.json") as f:
data = json.load(f)
def urls():
urls = ["https://" + url for url in data['urls']]
print(urls)
with ThreadPoolExecutor(max_workers=5) as pool:
iterator = pool.map(requests.get,urls)
for response in iterator:
print(response.headers)
print("\n")
Make async or threaded calls.
So, you would do something like this:
import aiohttp
import asyncio
import time
start_time = time.time()
async def main():
async with aiohttp.ClientSession() as session:
for number in range(1, 151):
pokemon_url = f'https://pokeapi.co/api/v2/pokemon/{number}'
async with session.get(pokemon_url) as resp:
pokemon = await resp.json()
print(pokemon['name'])
asyncio.run(main())
Could also do multiprocessing as per the comment, but async is better for i/o type tasks.
I'm trying to create a POST route where the media type would be set to "image/*" but when my route requires an argument, it's media type become "application/json" (even when I specify the response_class value).
Do you have an idea why?
Here is my source code:
from io import BytesIO
from fastapi.responses import ORJSONResponse
from fastapi import FastAPI
from fastapi.responses import StreamingResponse, JSONResponse, FileResponse
app = FastAPI(default_response_class=ORJSONResponse)
#app.post("/working", response_class=FileResponse)
async def send_w():
with open("/tmp/test/b.jpg", "rb") as f:
image = BytesIO(f.read())
image.seek(0)
return StreamingResponse(image, media_type="image/jpeg")
#app.post("/a", response_class=FileResponse)
async def send_a(a):
with open("/tmp/test/b.jpg", "rb") as f:
image = BytesIO(f.read())
image.seek(0)
return StreamingResponse(image, media_type="image/jpeg")
#app.post("/b", response_class=FileResponse)
async def send_b(b):
return "/tmp/test/b.jpg"
/a route:
You need to create function args to handle file type.
e.g.
from fastapi import FastAPI, File, UploadFile
#app.post("/image")
async def upload(data : UploadFile = File(...)):
image = await data.read()
# do your process here
read more:
https://fastapi.tiangolo.com/tutorial/request-files/
I have a python application running on my localhost:3978. Is it possible to make an api call to http://localhost:3978/api/users from http://localhost:3978/api/accounts?
#routes.get("/api/accounts")
async def accounts(request):
api_url = "http://127.0.0.1:3978/api/users"
response = requests.get(api_url)
return json_response(data=respone.json())
#routes.get("/api/users")
async def users(request):
pass
You can use url_path_for() to feed in RedirectResponse of starlette.responses as stated here. For example:
from fastapi import FastAPI
from starlette.responses import RedirectResponse
app = FastAPI()
#app.get("/a")
async def a():
return {"message": "Hello World"}
#app.get('/b')
async def b():
url = app.url_path_for("a")
response = RedirectResponse(url=url)
return response
I'm in a situation where I need to retry async request even when the request returns 200 response. For some specific cases, I need to check if there's a key in the output. If so, we need to retry. The following sample code (which can be executed in a Jupyter notebook) works for retries whenever the request fails (non-200). How can I tweak this to cater to this particular need?
P.S. Ideally, the response should've been non-200 but this is the option I'm left with.
# load required libraries
import json
import asyncio
import aiohttp
from async_retrying import retry
base_url = "http://localhost:1050/hello?rid="
# async ginger call
#retry(attempts=3)
async def async_ginger_call():
connector = aiohttp.TCPConnector(limit=3)
async with aiohttp.ClientSession(connector=connector) as session:
async with session.post(url, raise_for_status=True, timeout=300) as response:
result = await response.text()
# condition here; if key in result then retry
return json.loads(result)
reqs = 2
tasks = []
connector = aiohttp.TCPConnector(limit=reqs)
async with aiohttp.ClientSession(connector=connector) as session:
for i in range(reqs):
url = base_url + str(i)
# encode sentence
tasks.append(async_ginger_call())
results = await asyncio.gather(*tasks, return_exceptions=True)
Sample flask server code
# sample api
import time
import json
import datetime
from flask import Flask, request
from flask import Response
app = Flask(__name__)
#app.route('/hello', methods=['GET', 'POST'])
def welcome():
rid = request.args.get('rid', default=3, type=int)
valid_response = json.dumps({
"Result": {
"Warnings": [
{
"Code": 1100,
"Message": "A technical error occurred during execution."
}
]
}
}
)
# testing for failure
if rid == 2:
# pass
# return valid_response
return Response("{'Result': ''}", status=500, mimetype='application/json')
return valid_response
if __name__ == '__main__':
app.run(host='0.0.0.0', port=1050)
So I am trying to get a part of the URL like http://example/home?code=123456789 this 123456789 changes every time since it is oauth
so I am trying to get it
This is the py file
from aiohttp import web, web_urldispatcher
import discord
from discord.ext import commands
import aiohttp_jinja2
import jinja2
from pathlib import Path
from oauth import Ouath
#aiohttp_jinja2.template('home.html')
async def start(request):
raise web.HTTPSeeOther(location=Ouath.discord_login_url)
#aiohttp_jinja2.template('home.html')
async def login(request):
return
app = web.Application(loop=self.client.loop)
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(str(here)))
app.router.add_get('/', start)
app.router.add_get('/home', login)
runner = web.AppRunner(app)
await runner.setup()
self.site = web.TCPSite(runner, '127.0.0.1', 5000)
await self.client.wait_until_ready()
await self.site.start()
i want to print it in the html file
but i don't know how to get that part
note: i edit the code box
Since you're using web from aiohttp you can add a route that accepts a parameter
routes = web.RouteTableDef()
#routes.get('/guild/{guild}')
async def guild(request):
gid = request.match_info['guild']
The url would be http://localhost:PORT/guild/123456
Once you've fetched the required details, you're free to render a template or return a response.
After digging in source code of aiohttp_jinja2 and aiohttp it seems you can get it with request.query.get('code')
#aiohttp_jinja2.template('home.html')
async def login(request):
#print('code:', request.query.get('code'))
return {'code': request.query.get('code')}
If there is not ?code=... in URL then it gives None but you can set other default value using request.query.get('code', some_default_value)
Doc aiohttp: web.BaseRequest.query
If you have string then you can use
URL = 'http://example/home?code=123456789'
code = URL.split('?code=')[-1]
or if number is always at the end and it has always the same lenght
URL = 'http://example/home?code=123456789'
code = URL[-9:]
But there is also urllib.parse and
URL = 'http://example/home?code=123456789'
data = urllib.parse.parse_qs(urllib.parse.urlsplit(URL).query)
gives dictionary
{'code': ['123456789']}
and you can do
code = data.get('code')
and it will gives expected code or None if there was no ?code=... in url.
EDIT Probably you have to use request.url
#aiohttp_jinja2.template('home.html')
async def login(request):
data = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)
code = data.get('code')
return {'code': code}
Because data is dictionary with "code" so you could use return data
#aiohttp_jinja2.template('home.html')
async def login(request):
data = urllib.parse.parse_qs(urllib.parse.urlsplit(request.url).query)
return data
#aiohttp_jinja2.template('home.html')
async def login(request):
code = urllib.parse.parse_qs(urllib.parse.urlsplit(request).query)
return {'code': code}
like this ?