Develop Reference

Os.walk to exctract .gz files from all subdirectories using python

I'm using the following code to extract .gz files from all the subdirectories.
I am able to extract the files but the extracted files are not being saved in the subdirectory rather they are getting saved in the main directory/root:
import os, gzip, shutil
dir_name = r'C:\Users\ngowda\Downloads\DO_driver_logs'
def gz_extract(directory):
extension = ".gz"
os.chdir(directory)
for root,dirs,files in os.walk(directory):
for f in files:
if f.endswith(extension): # check for ".gz" extension
gz_name = os.path.join(root, f) # get full path of files
print('gz_name',gz_name)
file_name = (os.path.basename(gz_name)).rsplit('.',1)[0]
print('file_name',file_name)
with gzip.open(gz_name,"rb") as f_in, open(file_name,"wb") as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(gz_name) # delete zipped fil
I want all the extracted .gz files to be saved in same subdirectory and not in main directory. Can someone help me on this?

You are writing to file described by file_name which is
file_name = (os.path.basename(gz_name)).rsplit('.',1)[0]
os.path.basename does Returns the final component of a pathname therefore it ends in current working directory. As
gz_name = os.path.join(root, f) # get full path of files
is already absolute path, it should be sufficient to eliminate os.path.basename i.e. replacing
file_name = (os.path.basename(gz_name)).rsplit('.',1)[0]
using
file_name = gz_name.rsplit('.',1)[0]

Maybe it is because of file_name not having the full path. Change this line:
with gzip.open(gz_name,"rb") as f_in, open(file_name,"wb") as f_out:
to
with gzip.open(gz_name,"rb") as f_in, open(os.path.join(root, file_name),"wb") as f_out:
Let me know if it worked.

Streamlit, Flask, Fastapi: How to upload zipped files to my web app and make them downloadable?

I am working on an app using streamlit to upload a zipped file and enable users to download that zipped file. For a reference example, see how online services take in a word file, convert it into pdf and make that pdf downloadable automatically.
I have been able to complete the first 2 steps. Requesting assistance with uploading the file and making it downloadable. Thank you.

This is how it is done.
import streamlit as st
# zipping the different parts of the song
def zipdir(dst, src):
zf = zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED)
abs_src = os.path.abspath(src)
for dirname, subdirs, files in os.walk(src):
for filename in files:
absname = os.path.abspath(os.path.join(dirname, filename))
arcname = absname[len(abs_src) + 1:]
print('zipping %s as %s' % (os.path.join(dirname, filename),
arcname))
zf.write(absname, arcname)
zf.close()
zip_path = 'Python.zip'
zipdir(zip_path, path)
# send to webapp to make downloadable link
with open(zip_path, "rb") as f:
bytes_read = f.read()
b64 = base64.b64encode(bytes_read).decode()
href = f'<a href="data:file/zip;base64,{b64}" download=\'{title}.zip\'>\
Click to download\
</a>'
st.sidebar.markdown(href, unsafe_allow_html=True)

Problems downloading files from ftp server to directory

I am trying to download files from an ftp server to a local folder. Below I have included my code, the script is able to access the ftp succesfully and list the names of the files on the ftp server however where my issue lies is I am not sure how to download them to a local directory.
from ftplib import FTP
import os, sys, os.path
def handleDownload(block):
file.write(block)
ddir='U:/Test Folder'
os.chdir(ddir)
ftp = FTP('sidads.colorado.edu')
ftp.login()
print ('Logging in.')
directory = '/pub/DATASETS/NOAA/G02158/unmasked/2012/04_Apr/'
print ('Changing to ' + directory)
ftp.cwd(directory)
ftp.retrlines('LIST')
print ('Accessing files')
filenames = ftp.nlst() # get filenames within the directory
print (filenames)
I tried using the following code below however it gives me the error in the photo below.
for filename in filenames:
if filename != '.':
local_filename = os.path.join('U:/Test Folder/', filename)
file = open(local_filename, 'wb')
ftp.retrbinary('RETR '+ filename, file.write)
file.close()
ftp.quit()

Try the following:
for filename in filenames:
if filename not in ['.', '..']:
local_filename = os.path.join(ddir, filename)
with open(local_filename, 'wb') as f_output:
ftp.retrbinary('RETR '+ filename, f_output.write)
ftp.quit()
Your code was still attempting to download . filenames. You need to indent the writing of the valid filenames. Also by using with it will close the file automatically.

How do I upload full directory on FTP in python?

Ok, so I have to upload a directory, with subdirectories and files inside, on a FTP server. But I can't seem to get it right. I want to upload the directory as it is, with it's subdirectories and files where they were.
ftp = FTP()
ftp.connect('host',port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles():
for root,dirnames,filenames in os.walk(filenameCV):
for files in filenames:
print(files)
ftp.storbinary('STOR ' + files, open(files,'rb'))
ftp.quit()
placeFiles()

There are multiple problems with your code: First, the filenames array will only contain the actual filenames, not the entire path, so you need to join it with fullpath = os.path.join(root, files) and then use open(fullpath). Secondly, you quit the FTP connection inside the loop, move that ftp.quit() down on the level of the placeFiles() function.
To recursively upload your directory, you have to walk through your root directories and at the same time through your remote directory, uploading files on the go.
Full example code:
import os.path, os
from ftplib import FTP, error_perm
host = 'localhost'
port = 21
ftp = FTP()
ftp.connect(host,port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles(ftp, path):
for name in os.listdir(path):
localpath = os.path.join(path, name)
if os.path.isfile(localpath):
print("STOR", name, localpath)
ftp.storbinary('STOR ' + name, open(localpath,'rb'))
elif os.path.isdir(localpath):
print("MKD", name)
try:
ftp.mkd(name)
# ignore "directory already exists"
except error_perm as e:
if not e.args[0].startswith('550'):
raise
print("CWD", name)
ftp.cwd(name)
placeFiles(ftp, localpath)
print("CWD", "..")
ftp.cwd("..")
placeFiles(ftp, filenameCV)
ftp.quit()

os.renames for ftp in python

I want to move a large number of files from a windows system to a unix ftp server using python. I have a csv which has the current full path and filename and the new base bath to send it to (see here for an example dataset).
I have got a script using os.renames to do the transfer and directory creation in windows but can figure out a way to easily do it via ftp.
import os, glob, arcpy, csv, sys, shutil, datetime
top=os.getcwd()
RootOutput = top
startpath=top
FileList = csv.reader(open('FileList.csv'))
filecount=0
successcount=0
errorcount=0
# Copy/Move to FTP when required
ftp = ftplib.FTP('xxxxxx')
ftp.login('xxxx', 'xxxx')
directory = '/TransferredData'
ftp.cwd(directory)
##f = open(RootOutput+'\\Success_LOG.txt', 'a')
##f.write("Log of files Succesfully processed. RESULT of process run #:"+str(datetime.datetime.now())+"\n")
##f.close()
##
for File in FileList:
infile=File[0]
# local network ver
#outfile=RootOutput+File[4]
#os.renames(infile, outfile)
# ftp netowrk ver
# outfile=RootOutput+File[4]
# ftp.mkd(directory)
print infile, outfile
I tried the process in http://forums.arcgis.com/threads/17047-Upload-file-to-FTP-using-Python-ftplib but this is for moving all files in a directory, I have the old and new full file names and just need it to create the intermediate directories.
Thanks,

The following might work (untested):
def mkpath(ftp, path):
path = path.rsplit('/', 1)[0] # parent directory
if not path:
return
try:
ftp.cwd(path)
except ftplib.error_perm:
mkpath(ftp, path)
ftp.mkd(path)
ftp = FTP(...)
directory = '/TransferredData/'
for File in FileList:
infile = File[0]
outfile = File[4].split('\\') # need forward slashes in FTP
outfile = directory + '/'.join(outfile)
mkpath(ftp, outfile)
ftp.storbinary('STOR '+outfile, open(infile, 'rb'))