How can I use the Flask test_client to upload multiple files to one API endpoint?
I'm trying to use the Flask test_client to upload multiple files to a web service that accepts multiple files to combine them into one large file.
My controller looks like this:
#app.route("/combine/file", methods=["POST"])
#flask_login.login_required
def combine_files():
user = flask_login.current_user
combined_file_name = request.form.get("file_name")
# Store file locally
file_infos = []
for file_data in request.files.getlist('file[]'):
# Get the content of the file
file_temp_path="/tmp/{}-request.csv".format(file_id)
file_data.save(file_temp_path)
# Create a namedtuple with information about the file
FileInfo = namedtuple("FileInfo", ["id", "name", "path"])
file_infos.append(
FileInfo(
id=file_id,
name=file_data.filename,
path=file_temp_path
)
)
...
My test code looks like this:
def test_combine_file(get_project_files):
project = get_project_files["project"]
r = web_client.post(
"/combine/file",
content_type='multipart/form-data',
buffered=True,
follow_redirects=True,
data={
"project_id": project.project_id,
"file_name": "API Test Combined File",
"file": [
(open("data/CC-Th0-MolsPerCell.csv", "rb"), "CC-Th0-MolsPerCell.csv"),
(open("data/CC-Th1-MolsPerCell.csv", "rb"), "CC-Th1-MolsPerCell.csv")
]})
response_data = json.loads(r.data)
assert "status" in response_data
assert response_data["status"] == "OK"
However, I can't get the test_client to actually upload both files. With more than one file specified, the file_data is empty when the API code loops. I have tried my own ImmutableDict with two "file" entries, a list of file tuples, a tuple of file tuples, anything I could think of.
What is the API to specify multiple files for upload in the Flask test_client? I can't find this anywhere on the web! :(
The test client takes a list of file objects (as returned by open()), so this is the testing utility I use:
def multi_file_upload(test_client, src_file_paths, dest_folder):
files = []
try:
files = [open(fpath, 'rb') for fpath in src_file_paths]
return test_client.post('/api/upload/', data={
'files': files,
'dest': dest_folder
})
finally:
for fp in files:
fp.close()
I think if you lose your tuples (but keeping the open()s) then your code might work.
You should just send data object with your files named as you want:
test_client.post('/api/upload',
data={'title': 'upload sample',
'file1': (io.BytesIO(b'get something'), 'file1'),
'file2': (io.BytesIO(b'forthright'), 'file2')},
content_type='multipart/form-data')
Another way of doing this- if you want to explicitly name your file uploads here (my use case was for two CSVs, but could be anything) with test_client is like this:
resp = test_client.post(
'/data_upload_api', # flask route
file_upload_one=[open(FILE_PATH, 'rb')],
file_upload_two=[open(FILE_PATH_2, 'rb')]
)
Using this syntax, these files would be accessible as:
request.files['file_upload_one'] # etc.
Related
I am developing an azure function that receives in input several files of different formats (eg xlsx, csv, txt, pdf, png) through the form-data format. The idea is to develop a function that can take files and store them one by one inside a blob. At the moment, my code is as follows:
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
filename, contents = False, False
try:
files = req.files.values()
for file in files:
filename = str(file.filename)
logging.info(type(file.stream.read()))
contents = file.stream.read().decode('utf-8')
except Exception as ex:
logging.error(str(type(ex)) + ': ' + str(ex))
return func.HttpResponse(body=str(ex), status_code=400)
Then i write the content variable inside the blob but the files inside the blob had 0 as size and if i try to download the file, the files are empty. How can i manage this operation to store different format files inside a blob? Thanks a lot for your support!
Below is the code to upload multiple files of different formats:
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient,PublicAccess
import os
def UploadFiles():
CONNECTION_STRING="ENTER_STORAGE_CONNECTION_STRING"
Container_name="uploadcontainer"
service_client=BlobServiceClient.from_connection_string(CONNECTION_STRING)
container_client = service_client.get_container_client(Container_name)
ReplacePath = "C:\\"
local_path = "C:\Testupload" #the local folder
for r,d,f in os.walk(local_path):
if f:
for file in f:
AzurePath = os.path.join(r,file).replace(ReplacePath,"")
LocalPath = os.path.join(r,file)
blob_client = container_client.get_blob_client(AzurePath)
with open(LocalPath,'rb') as data:
blob_client.upload_blob(data)
if __name__ == '__main__':
UploadFiles()
print("Files Copied")
As from your question I am not able to get how your function is getting triggered, and where you are uploading your files.
So as per your logic you can use the above piece of code to upload all type of files.
Currently above code can be used to upload all the files in a local folder. Below is the screenshot for a repro:
import glob
import os
import requests
import shutil
class file_service:
def file():
dir_name = '/Users/TEST/Downloads/TU'
os.chdir(dir_name)
pattern='TU_*.csv'
for x in glob.glob(pattern):
file_name=os.path.join(dir_name,x)
print (file_name)
from datetime import date
dir_name_backup = '/Users/Zill/Downloads/backup'
today = date.today()
backup_file_name = f'Backup_TU_{today.year}{today.month:02}{today.day:02}.csv'
backup_file_name_directory= os.path.join(dir_name_backup,backup_file_name)
print(backup_file_name_directory)
newPath = shutil.copy(file_name, backup_file_name_directory)
url = "google.com"
payload = {'name': 'file'}
files = [
('file', open(file_name,'rb'))
]
headers = {
'X-API-TOKEN': '12312'
}
response = requests.request("POST", url, headers=headers, data = payload, files = files)
print(response.text.encode('utf8'))
files.close()
os.remove(file_name)
file()
To provide an overall context, I am trying to retrieve a file from my OS and using POST method I am trying to post the content of the file into an application. Its working as expected so far, the details are getting pushed into application as expected. As part of my next step I am trying to remove the file from my existing directory using os.remove(). But I am getting a Win32 error as my file is not closed when it was opened in read-only mode in the POST call. I am trying to close it but I am unable to do so.
Can anyone please help me out with it.
Thanks!
I'm not sure I understand your code correctly. Could you try replacing
files.close()
with
for _, file in files:
file.close()
and check if it works?
Explanation:
In
files = [('file', open(file_name,'rb'))]
you create a list containing exactly one tuple that has the string 'file' as first element and a file object as second element:
[('file', file_object)]
The loop takes the tuple from the list, ignores its first element (_), takes its second element, the file object, and uses its close method to close it.
I've just now realised the list contains only one tuple. So there's no need for a loop:
files[0][1].close()
should do it.
The best way would be to use with (the file gets automatically closed once you leave the with block):
payload = {'name': 'file'}
with open(file_name, 'rb') as file:
files = [('file', file)]
headers = {'X-API-TOKEN': '12312'}
response = requests.request("POST", url, headers=headers, data = payload, files = files)
print(response.text.encode('utf8'))
os.remove(file_name)
I'm quite new on Django and i'm looking for a way to dwonload a zip file from my django site but i have some issue when i'm running this piece of code:
def download(self):
dirName = settings.DEBUG_FOLDER
name = 'test.zip'
with ZipFile(name, 'w') as zipObj:
# Iterate over all the files in directory
for folderName, subfolders, filenames in os.walk(dirName):
for filename in filenames:
# create complete filepath of file in directory
filePath = os.path.join(folderName, filename)
# Add file to zip
zipObj.write(filePath, basename(filePath))
path_to_file = 'http://' + sys.argv[-1] + '/' + name
resp= {}
# Grab ZIP file from in-memory, make response with correct MIME-type
resp = HttpResponse(content_type='application/zip')
# ..and correct content-disposition
resp['Content-Disposition'] = 'attachment; filename=%s' % smart_str(name)
resp['X-Sendfile'] = smart_str(path_to_file)
return resp
I get:
Exception Value:
<HttpResponse status_code=200, "application/zip"> is not JSON serializable
I tried to change the content_type to octet-stream but it doesn't work
And to use a wrapper as followw:
wrapper = FileWrapper(open('test.zip', 'rb'))
content_type = 'application/zip'
content_disposition = 'attachment; filename=name'
# Grab ZIP file from in-memory, make response with correct MIME-type
resp = HttpResponse(wrapper, content_type=content_type)
# ..and correct content-disposition
resp['Content-Disposition'] = content_disposition
I didn't find useful answer so far but maybe I didn't search well, so if it seems my problem had been already traited, feel free to notify me
Thank you very much for any help
You have to send the zip file as byte
response = HttpResponse(zipObj.read(), content_type="application/zip")
response['Content-Disposition'] = 'attachment; filename=%s' % smart_str(name)
return response
I would do like this:
(Caveat I use wsl so the python function will make use of cmd lines)
In view:
import os
def zipdownfun(request):
""" Please establish in settings.py where media file should be downloaded from.
In my case is media with a series of other folders inside. Media folder is at the same level of project root folder, where settings.py is"""
file_name = os.path.join(MEDIA_URL,'folder_where_your_file_is','file_name.zip')
"""let us put the case that you have zip folder in media folder"""
file_folder_path = os.path.join(MEDIA_URL,'saving_folder')
"""The command line takes as first variable the name of the
future zip file and as second variable the destination folder"""
cmd = f'zip {file_name} {file_folder_path}'
"""With os I open a process in the background so that some magic
happens"""
os.system(cmd)
"""I don't know what you want to do with this, but I placed the
URL of the file in a button for the download, so you will need
the string of the URL to place in href of an <a> element"""
return render(request,'your_html_file.html', {'url':file_name})
The db I have created, will be updated very often. I used a slightly different version of this function with -r clause since I had to zip, each time, a folder. Why I did this? The database I have created has to allow the download of this zipped folder. This folder will be updated daily. So this function basically overwrites the file each time that is downloaded. It will be so fresh of new data each time.
Please refer to this page to understand how to create a button for the download of the generated file.
Take as reference approach 2. The URL variable that you are passing to the Django template should be used at the place of the file (screenshot attached)
I hope it can help!
I am developing a flask application which uploads a file to IBM Bluemix Cloudant DB. I need to save the contents of the file as a key value pair in Cloudant.
If I try to save a text file, it reads the content correctly. For other type of files it does not work.
Following is my flask REST API CODE:
#app.route('/upload', methods=['POST'])
def upload_file():
file_to_upload = request.files['file_upload'];
response = CloudantDB().upload_file_to_db(file_to_upload);
//tHE FUNCTION upload_file under CloudantDB is as shown below.
file_name = file.filename;
uploaded_file_content = file.read();
data = {
'file_name': file_name,
'file_contents': uploaded_file_content,
'version': version
}
my_doc = self.database.create_document(data);
I know the error is because "uploaded_file_content" is in a different format (i.e. For PDFs, JPGs etc).
Is there anyway I can overcome this?
Thanks!
The difference is that text files contain ordinary text whereas JPG, PNG etc. contain binary data.
Binary data should be uploaded as an attachment with a mime type, and you need to base64 encode the data. You don't show what create_document() is doing, but it's unlikely that it is able to treat binary data as an attachment. This might fix it for you:
from base64 import b64encode
uploaded_file_content = b64encode(file.read());
data = {
'file_name': file_name,
'version': version,
'_attachments': {
file_name : {
'content-type': 'image/png',
'data': uploaded_file_content
}
}
}
my_doc = self.database.create_document(data);
It should also be possible with your current code to simply base64 encode the file content and upload it. So that you know what type of data is stored should you later retrieve it, you will need to add another key value pair to store the mime type as content-type does above.
Attachments have advantages in that they can be individually addressed, read, deleted, updated without affecting the containing document, so you are probably better off using them.
I have an app that takes in some information, performs some calculations using pandas, and turns the final pandas data frame into a CSV that is then downloaded using the Flask app. How do I download multiple CSVs within one view? It seems that I can only return a single response at a time.
An example snippet:
def serve_csv(dataframe,filename):
buffer = StringIO.StringIO()
dataframe.to_csv(buffer, encoding='utf-8', index=False)
buffer.seek(0)
return send_file(buffer,
attachment_filename=filename,
mimetype='text/csv')
def make_calculation(arg1, arg2):
'''Does some calculations.
input: arg1 - string, arg2- string
returns: a pandas data frame'''
#app.route('test_app', methods=['GET', 'POST'])
def test_app():
form = Form1()
if form.validate_on_submit():
calculated_dataframe = make_calculation(str(form.input_1.data), str(form.input_2.data))
return serve_csv(calculated_dataframe, 'Your_final_output.csv')
return render_template('test_app.html', form=form)
So let's say in that example above that make_calculation returned two pandas data frames. How would I print both of them to a CSV?
This is all the code you need using the Zip files. It will return a zip file with all of your files.
In my program everything I want to zip is in an output folder so i just use os.walk and put it in the zip file with write. Before returning the file you need to close it, if you don't close it will return an empty file.
import zipfile
import os
from flask import send_file
#app.route('/download_all')
def download_all():
zipf = zipfile.ZipFile('Name.zip','w', zipfile.ZIP_DEFLATED)
for root,dirs, files in os.walk('output/'):
for file in files:
zipf.write('output/'+file)
zipf.close()
return send_file('Name.zip',
mimetype = 'zip',
attachment_filename= 'Name.zip',
as_attachment = True)
In the html I simply call the route:
DOWNLOAD ALL
I hope this helped somebody. :)
You could return a MIME Multipart response, a zip file, or a TAR ball (please note the linked RFC is somewhat out of date, but is easier to quickly get up to speed with because it's in HTML; the official one is here).
If you choose to do a MIME multipart response, a good starting point might be to look at the MultipartEncoder and MultipartDecoder in requests toolbelt; you may be able to use them directly, or at least subclass/compose using those to get your desired behavior. Zip files and TAR balls can be implemented using standard library modules.
An alternative would be to design your API so that you were returning JSON, use a header (or XML element or JSON field) to indicate that additional CSVs could be obtained by another request, or similar.
Building on #desfido's answer above, here would be some code implementation that does not involve using zip, and instead downloads two different files:
from requests_toolbelt import MultipartEncoder
def make_calculation(arg1, arg2):
'''Does some calculations.
input: arg1 - string, arg2- string
puts results in two different dataframes
and stores them in two different files,
returns the names of those two files'''
return filename1, filename2
#app.route('test_app', methods=['GET', 'POST'])
def test_app():
form = Form1()
if form.validate_on_submit():
f1, f2 = make_calculation(str(form.input_1.data), str(form.input_2.data))
m = MultipartEncoder({
'field1': (f1, open(f1, 'rb'), 'text/plain'),
'field2': (f2, open(f2, 'rb'), 'text/plain')
})
return Response(m.to_string(), mimetype=m.content_type)
return render_template('test_app.html', form=form)
Also you may try this, using zip module --
import zipfile
from os.path import basename
UPLOAD_PATH = <upload_location>
base_files = ["file1.csv", "file2.csv"]
with zipfile.ZipFile(UPLOAD_PATH + 'Test.zip', 'w') as zipF:
for file in base_files:
zipF.write(UPLOAD_PATH + file, basename(UPLOAD_PATH + file), compress_type=zipfile.ZIP_DEFLATED)
zipF.close()
return send_file(METAFILE_UPLOADS+'Test.zip', mimetype='zip', attachment_filename='Test.zip', as_attachment=True)