I'm trying to parse a .csv file using this code
class uploadHandler(tornado.web.RequestHandler):
def get(self):
self.render("index.html")
def post(self):
file = self.request.files['uploadedFile'][0]
filename = file['filename']
output = open("file/" + filename, 'wb')
output.write(file['body'])
self.write("http://localhost:8080/file/" + filename)
self.finish("uploaded")
df = pandas.read_csv("file\\" + filename)
print(df)
if (__name__ == "__main__"):
app = tornado.web.Application([
("/", uploadHandler),
("/file/(.*)", tornado.web.StaticFileHandler, {"path" : "file"})
])
app.listen(8080)
print("Listening on port 8080")
tornado.ioloop.IOLoop.instance().start()
I get the error
File "pandas\_libs\parsers.pyx", line 554, in pandas._libs.parsers.TextReader.__cinit__
pandas.errors.EmptyDataError: No columns to parse from file
How can I parse this file?
I tried accessing this file in different parts of the code but I get the same error. The file gets uploaded correctly.
You have to close the output handler before read the file with Pandas:
...
output = open("file/" + filename, 'wb')
output.write(file['body'])
output.close() # <- HERE
...
But use a context manager instead of closing the file yourself:
...
with open("file/" + filename, 'wb') as output:
output.write(file['body'])
...
Related
I've been trying for the past 24 hours but can't find a solution.
This is the code:
import os
from pathlib import Path
import requests
PINATA_BASE_URL = "https://api.pinata.cloud/"
endpoint = "pinning/pinFileToIPFS"
# Change this filepath
filepath = "C:/Users/acer/Desktop/Ciao"
filename = os.listdir(filepath)
print(filename)
headers = {
"pinata_api_key": os.getenv("PINATA_API_KEY"),
"pinata_secret_api_key": os.getenv("PINATA_API_SECRET"),
}
def main():
with Path(filepath).open("rb") as fp:
image_binary = filepath.read()
print(image_binary)
response = requests.post(
PINATA_BASE_URL + endpoint,
files={"file": (filename, image_binary)},
headers=headers,
)
print(response.json())
if __name__ == "__main__":
main()
I tried to open the folder where the metadata was stored and than i sent the request with the list of files in the folder.
This is the error:
['no.txt', 'yeah.txt']
Traceback (most recent call last):
File "C:\Users\acer\Desktop\SOLIDITY_PYTHON\nft-bored-ape\scripts\upload_to_pinata.py", line 30, in <module>
main()
File "C:\Users\acer\Desktop\SOLIDITY_PYTHON\nft-bored-ape\scripts\upload_to_pinata.py", line 18, in main
with Path(filepath).open("rb") as fp:
File "C:\Users\acer\AppData\Local\Programs\Python\Python310\lib\pathlib.py", line 1119, in open
return self._accessor.open(self, mode, buffering, encoding, errors,
PermissionError: [Errno 13] Permission denied: 'C:\\Users\\acer\\Desktop\\Ciao'
Nevermind...
After some research i found the answer to my own question.
Here is the code:
# Tulli's script :-)
from brownie import config
import requests, os, typing as tp
PINATA_BASE_URL = "https://api.pinata.cloud/"
endpoint = "pinning/pinFileToIPFS"
# Here you could use os.getenv("VARIABLE_NAME"),
# i used config from my .yaml file. Your choice!
headers = {
"pinata_api_key": config["pinata"]["api-keys"],
"pinata_secret_api_key": config["pinata"]["api-private"],
}
def get_all_files(directory: str) -> tp.List[str]:
"""get a list of absolute paths to every file located in the directory"""
paths: tp.List[str] = []
for root, dirs, files_ in os.walk(os.path.abspath(directory)):
for file in files_:
paths.append(os.path.join(root, file))
return paths
def upload_folder_to_pinata(filepath):
all_files: tp.List[str] = get_all_files(filepath)
# The replace function is a must,
# pinata servers doesn't recognize the backslash.
# Your filepath is probably different than mine,
# so in the split function put your "penultimate_file/".
# Strip the square brackets and the apostrophe,
# because we don't want it as part of the metadata ipfs name
files = [
(
"file",
(
str(file.replace("\\", "/").split("Desktop/")[-1:])
.strip("[]")
.strip("'"),
open(file, "rb"),
),
)
for file in all_files
]
response: requests.Response = requests.post(
PINATA_BASE_URL + endpoint,
files=files,
headers=headers,
)
# If you want to see all the stats then do this:
# return/print/do both separately response.json()
return "ipfs.io/ipfs/" + response.json()["IpfsHash"]
def main():
upload_folder_to_pinata("Put your full filepath here")
if __name__ == "__main__":
main()
I'm going to create a zip file from some of the image files stored on my server.
I've used the following function to do this:
def create_zip_file(user, examination):
from lms.models import StudentAnswer
f = BytesIO()
zip = zipfile.ZipFile(f, 'w')
this_student_answer = StudentAnswer.objects.filter(student_id=user.id, exam=examination)
for answer in this_student_answer:
if answer.answer_file:
answer_file_full_path = answer.answer_file.path
fdir, fname = os.path.split(answer_file_full_path)
zip.writestr(fname, answer_file_full_path)
zip.close() # Close
zip_file_name = "student-answers_"+ str(examination.id)+"_" + str(user.id) + "_" + date=datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") + '.zip'
response = HttpResponse(f.getvalue(), content_type="application/x-zip-compressed")
response['Content-Disposition'] = 'attachment; filename=%s' % zip_file_name
return response
Everything is fine and all photos are made in zip file but there is only one problem.
The problem is that the photos won't open and this error will appear in Windows:
Its look like we don't support this file format.
What is wrong with my codes?
To append data from file you have to use
write(filename)
Using writestr(filename) you add only string from variable filename but not from file.
I am having problem tracking when a file download has been completed when StreamingHttpResponse is used. My intention is to delete the file once it has been downloaded by the user.
Doing the following returned an exception in terminal killing the server.
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv_path = os.path.join(fh.path, fh.filename)
csv_file = open(csv_path)
response = StreamingHttpResponse(csv_file, content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
csv_file.close()
# I can now delete the file using os.remove(csv_path). Not sure since response has not been returned
return response
return HttpResponseRedirect('/b2b/export/')
Trace back:
----------------------------------------
Exception happened during processing of request from ('127.0.0.1', 59899)
Traceback (most recent call last):
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 599, in process_request_thread
self.finish_request(request, client_address)
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 334, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/Users/Michael/.virtualenvs/scrape/lib/python2.7/site-packages/django/core/servers/basehttp.py", line 102, in __init__
super(WSGIRequestHandler, self).__init__(*args, **kwargs)
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/SocketServer.py", line 655, in __init__
self.handle()
File "/Users/Michael/.virtualenvs/scrape/lib/python2.7/site-packages/django/core/servers/basehttp.py", line 182, in handle
handler.run(self.server.get_app())
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/handlers.py", line 92, in run
self.close()
File "/usr/local/Cellar/python/2.7.10_2/Frameworks/Python.framework/Versions/2.7/lib/python2.7/wsgiref/simple_server.py", line 33, in close
self.status.split(' ',1)[0], self.bytes_sent
AttributeError: 'NoneType' object has no attribute 'split'
----------------------------------------
What works is as follows but am not sure when to delete the file or know when download has been completed. Most importantly how to close the file:
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv_path = os.path.join(fh.path, fh.filename)
response = StreamingHttpResponse(open(csv_path), content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
return response
return HttpResponseRedirect('/b2b/export/')
I have tried the above suggestion by #mwag; however with FileWrapper.
In my case, wanted to zip up directories and get rid of the archive when download is done.
import os, time, zipfile
from django.http import StreamingHttpResponse
from wsgiref.util import FileWrapper
class FileDeleteWrapper(FileWrapper):
def __init__(self, filepath, *args, **kwargs):
self.filepath = filepath
super(FileDeleteWrapper, self).__init__(*args, **kwargs)
def __del__(self, *args, **kwargs):
os.remove(self.filepath)
# View function
def zipFiles(request, assetId):
asset = get_object_or_404(Asset, id=assetId)
try:
files = File.objects.filter(asset=asset)
prefix = str(time.time()) +'_'
zipPath = os.path.join(
settings.ZIPPED_FILES_DIR,
prefix + asset.label+'.zip'
)
z = zipfile.ZipFile(zipPath, 'w', zipfile.ZIP_DEFLATED)
for f in files:
path = os.path.join(
mainSet.MEDIA_ROOT,
str(f.file)
)
z.write(path, str(f))
z.close()
chunkSize = 16384
response = StreamingHttpResponse(
FileDeleteWrapper(
filepath = zipPath,
filelike=open(zipPath, 'rb'),
blksize=chunkSize
)
)
response['Content-Length'] = os.path.getsize(zipPath)
response['Content-Disposition'] = "attachment; filename=%s" % asset.label+'.zip'
return response
except Exception as e:
if mainSet.DEBUG:
print(type(e))
else:
# log expception
raise Http404
Try creating a class that will delete the file when it is gc'd. For example, something like the below might work:
class open_then_delete(object):
def __init__(self, filename, mode='rb'):
self.filename = filename
self.file_obj = open(filename, mode)
def __del__(self):
self.close()
def close(self):
if self.file_obj:
self.file_obj.close()
self.file_obj = None
self.cleanup()
def cleanup(self):
if self.filename:
try:
sys.stderr.write('open_then_delete: del ' + self.filename)
os.remove(self.filename)
except:
pass
self.filename = None
def __getattr__(self, attr):
return getattr(self.file_obj, attr)
def __iter__(self):
return iter(self.file_obj)
# below is your code, modified to use use_then_delete
def down(request, file_name):
if request.method == 'GET':
if file_name:
import os
fh = get_object_or_404(FileHandler, filename=file_name)
csv = open_then_delete(os.path.join(fh.path, fh.filename))
response = StreamingHttpResponse(csv.open(), content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}"'.format(fh.filename)
return response
return HttpResponseRedirect('/b2b/export/')
I've tried the script below:
import os
from ftplib import FTP
ftp = FTP("ftpsite","myuser", "mypass")
ftp.login()
ftp.retrlines("LIST")
ftp.cwd("folderOne")
ftp.cwd("subFolder")
listing = []
ftp.retrlines("LIST", listing.append)
words = listing[0].split(None, 8)
filename = words[-1].lstrip()
#download the file
local_filename = os.path.join(r"C:\example", file)
lf = open(local_filename, "wb")
ftp.retrbinary("RETR " + filename, lf.write, 8*1024)
lf.close()
But everytime I run the script, it says:
Traceback (most recent call last):
File "C:\User\Desktop\sample\ex.py", line 4, in <module>
ftp = FTP("ftpsite", "myuser", "mypass")
File "C:\Python27\lib\ftplib.py", line 119, in __init__
self.login(user, passwd, acct)
File "C:\Python27\lib\ftplib.py", line 387, in login
resp = self.sendcmd('USER ' + user)
File "C:\Python27\lib\ftplib.py", line 244, in sendcmd
return self.getresp()
File "C:\Python27\lib\ftplib.py", line 219, in getresp
raise error_perm, resp
error_perm: 530 Permission denied.
I don't know what 530 Permission Denied means.Can anyone tell me what does that means?
It seems like the ftp server allows anonymous access; You don't need pass username, password.
FTP constructor accept hostname(or IP), not URL.
import sys
import os
from ftplib import FTP
ftp = FTP("ftpsite.com")
ftp.login()
ftp.cwd("/ftp/site/directory/")
listing = []
ftp.retrlines("LIST", listing.append)
words = listing[0].split(None, 8)
filesize = int(words[4])
filename = words[-1].lstrip()
class VerboseWriter:
def __init__(self, lf, filesize):
self.progress = 0
self.lf = lf
self.filesize = filesize
def write(self, data):
self.lf.write(data)
self.progress += len(data)
sys.stdout.write('\r{}/{} ({:.1%})'.format(self.progress, self.filesize, float(self.progress)/self.filesize))
sys.stdout.flush()
#download the file
with open(os.path.join(r"c:\example", filename), 'wb') as f:
ftp.retrbinary("RETR " + filename, VerboseWriter(lf, filesize).write, 8*1024)
print
ftp.quit()
i wrote a small app that downloads a zip file (with a different extension) when provided by a link and extracts the file to a renamed folder.
For some reason its working for some of my zip files, but not for all of them.
I get a :
Traceback (most recent call last):
File "download_unzip.py", line 48, in <module>
main()
File "download_unzip.py", line 42, in main
shutil.move(unzip_file(temp_kmz),'temp_extracted/')
File "download_unzip.py", line 26, in unzip_file
fd = open(name, 'w')
IOError: [Errno 2] No such file or directory: 'models/model.dae'
My code is :
import sys , urllib , zipfile , os.path , argparse , shutil
parser = argparse.ArgumentParser(description="Download and Unzip")
parser.add_argument('url', help='The action to take (e.g. install, remove, etc.)')
args = parser.parse_args()
print args.url
url = args.url
temp_kmz="temp_kmz"
def unzip_file(path):
zfile = zipfile.ZipFile(path)
extracted_filename = zfile.infolist()[0].filename[:-1]
for name in zfile.namelist():
(dirname, filename) = os.path.split(name)
#print "Decompressing " + filename + " on " + dirname
if filename == '':
# directory
if not os.path.exists(dirname):
os.mkdir(dirname)
else:
# file
fd = open(name, 'w')
fd.write(zfile.read(name))
fd.close()
zfile.close()
return extracted_filename
def download_file():
urllib.urlretrieve (url, temp_kmz)
return True
def main():
if (download_file()):
print "Now deleting temp..."
shutil.rmtree('temp_extracted/')
print "unzipping.. and renaming folder"
shutil.move(unzip_file(temp_kmz),'temp_extracted/')
print "Finished!!"
else:
print "Error downloading file"
main()
my working downloaded file:
python download_unzip.py "http://dl.dropbox.com/u/2971439/dae.kmz"
The one that is not working:
python download_unzip.py
"http://dl.dropbox.com/u/2971439/rally_car_youbeq.kmz"
Please note that both files extract properly with my OS (Ubuntu)
fixed my problem with some heavy code changes:
import urllib2 ,argparse, shutil, urlparse , os , zipfile, os.path
from zipfile import ZipFile as zip
parser = argparse.ArgumentParser(description="Download and Unzip")
parser.add_argument('url', help='The action to take (e.g. install, remove, etc.)')
args = parser.parse_args()
print args.url
url = args.url
temp_kmz="temp_kmz"
def extractAll(zipName):
z = zip(zipName)
for f in z.namelist():
if f.endswith('/'):
os.makedirs(f)
else:
z.extract(f)
def download(url, fileName=None):
def getFileName(url,openUrl):
if 'Content-Disposition' in openUrl.info():
# If the response has Content-Disposition, try to get filename from it
cd = dict(map(
lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),
openUrl.info()['Content-Disposition'].split(';')))
if 'filename' in cd:
filename = cd['filename'].strip("\"'")
if filename: return filename
# if no filename was found above, parse it out of the final URL.
return os.path.basename(urlparse.urlsplit(openUrl.url)[2])
r = urllib2.urlopen(urllib2.Request(url))
try:
fileName = fileName or getFileName(url,r)
with open(fileName, 'wb') as f:
shutil.copyfileobj(r,f)
finally:
r.close()
def main():
download(url,temp_kmz)
extractAll(temp_kmz)
main()