Develop Reference

How do I upload full directory on FTP in python?

Ok, so I have to upload a directory, with subdirectories and files inside, on a FTP server. But I can't seem to get it right. I want to upload the directory as it is, with it's subdirectories and files where they were.
ftp = FTP()
ftp.connect('host',port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles():
for root,dirnames,filenames in os.walk(filenameCV):
for files in filenames:
print(files)
ftp.storbinary('STOR ' + files, open(files,'rb'))
ftp.quit()
placeFiles()

There are multiple problems with your code: First, the filenames array will only contain the actual filenames, not the entire path, so you need to join it with fullpath = os.path.join(root, files) and then use open(fullpath). Secondly, you quit the FTP connection inside the loop, move that ftp.quit() down on the level of the placeFiles() function.
To recursively upload your directory, you have to walk through your root directories and at the same time through your remote directory, uploading files on the go.
Full example code:
import os.path, os
from ftplib import FTP, error_perm
host = 'localhost'
port = 21
ftp = FTP()
ftp.connect(host,port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles(ftp, path):
for name in os.listdir(path):
localpath = os.path.join(path, name)
if os.path.isfile(localpath):
print("STOR", name, localpath)
ftp.storbinary('STOR ' + name, open(localpath,'rb'))
elif os.path.isdir(localpath):
print("MKD", name)
try:
ftp.mkd(name)
# ignore "directory already exists"
except error_perm as e:
if not e.args[0].startswith('550'):
raise
print("CWD", name)
ftp.cwd(name)
placeFiles(ftp, localpath)
print("CWD", "..")
ftp.cwd("..")
placeFiles(ftp, filenameCV)
ftp.quit()

Python - Watch a folder for new .zip file and upload via FTP

I am working on creating a script to watch a folder, grab any new .zip files, and then upload them via FTP to a predetermined area. Right now FTP testing is being performed Locally, since the environment isnt yet created.
The strategy I am taking is to first, unzip into a local folder. Then, perform ftplib.storbinary , on the file from the local folder, to the ftpdestination. However, the unzipping process appears to be working but I am getting a "file does not exist" error, all though I can see it in the folder itself.
Also, is there anyway to unzip directly into an FTP location? I havent been able to find a way hence the approach I am taking.
Thanks, local ftp info removed from code. All paths that are relevant in this code will be changed, most likely to dynamic fashion, but for now this is a local environment
extractZip2.py
import zipfile
import ftplib
import os
import logging
import time
from socket import error as socket_error
#Logging Setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('__name__')
FTPaddress = ''
FTPusername = ''
FTPpassword = ''
ftp_destination_location = ''
path_to_watch = "C:/Users/206420055/Desktop/test2/"
before = dict ([(f,None) for f in os.listdir(path_to_watch)])
temp_destination_location = "C:/Users/206420055/Desktop/temp/"
def unzip(fullPath,temporaryPath):
with zipfile.ZipFile(fullPath, "r") as z :
logger.info("Unzipping {0}".format(fullPath))
z.extractall(temporaryPath)
logger.info("Unzipped into local directory {0}".format(temp_destination_location))
def check_or_create_ftp(session, folder):
"""
Checks to see if necessary folder for currenttab is available.
Creates the folder if not found, and enters it.
"""
if folder not in session.nlst():
logger.info('Directory for {0} does not exist, creating directory\n'.format(folder))
session.mkd(folder)
session.cwd(folder)
def check_or_create(temp_destination):
"""
Checks to see if local savepath exists. Will create savepath if not exists.
"""
if not os.path.exists(temp_destination):
logger.info('Directory for %s does not exist, creating directory\n' % temp_destination)
os.makedirs(str(temp_destination))
def transfer(address,username,password,filename,destination):
logger.info("Creating Session")
try:
session = session_init(address,username,password,destination)
except (socket_error,ftplib.error_perm) as e:
logger.error(str(e))
logger.error("Error in Session Init")
else:
try:
logger.info("Sending File {0}".format(filename))
send_file(session,filename)
except (IOError, OSError, ftplib.error_perm) as e:
logger.error(e)
def session_init(address,username,password,path):
session = ftplib.FTP(address,username,password)
check_or_create_ftp(session,path)
logger.info("Session Established")
return session
def send_file(session,filename):
file = open(filename,'rb')
logger.info('Sending File : STOR '+filename)
session.storbinary('STOR '+ filename, file)
file.close()
def delete_local_files(savepath, file):
logger.info("Cleaning Up Folder {0}".format(savepath))
os.remove(file)
while 1:
time.sleep(5)
after = dict ([(f,None) for f in os.listdir(path_to_watch)])
added = [f for f in after if not f in before]
removed = [f for f in before if not f in after]
if added: print "Added: ",", ".join(added)
before = after
check_or_create(temp_destination_location)
if added :
for file in added:
print file
if file.endswith('.zip'):
unzip(path_to_watch+file, temp_destination_location)
temp_files = os.listdir(temp_destination_location)
print("Temp Files {0}".format(temp_files))
for tf in temp_files:
print("TF {0}".format(tf))
transfer(FTPaddress,FTPusername,FTPpassword,tf,ftp_destination_location)
#delete_local_files(temp_destination_location,tf)
else:
pass
edit: adding error image
Seen above, we see the file in the temp folder. But the console obviously shows the error.

just change it to
from glob import glob
zips_in_path = dict ([(f,None) for f in glob("{base_path}/*.zip".format(base_path = path_to_watch)])
os.listdir does not include the path_to_watch part of the path it is just the filenames, however glob does.
so you could also do
after = dict ([(os.path.join(path_to_watch,f),None) for f in os.listdir(path_to_watch)])
using either of these methods you should be able to get the full path to the files in the path

I'm trying to download all the files from a directory using Python's Paramiko module

I've got pretty much everything else worked out except this so any help here would be greatly appreciated as I've been trying to get this part to work for hours now and nothing I search for seems to be helping.
The problem is that I can't seem to get the download function (get()) to work. My upload function (put()) works perfectly fine.. What this code does is creates a list of all the files that aren't yet on the server and then, using glob.glob to target all the files, it uploads the ones that aren't already there directly onto the server.
All I need to figure out, is how to download all the files from a directory onto my computer so that I can upload them to a different directory on the same server..if there is an easier way to do this please let me know.
import paramiko
import glob
import os
import fileinput
import shutil
from paramiko import sftp
from os import path
##This script uploads files from a local folder to a remote folder via
##SFTP and keeps track of which files have already been uploaded using "list.txt".
remotee = "//development//"
locall = "C:\\ftp\\Pangea"
complete = "C:\\moved"
mylist = "C:\\moved\\list.txt"
lastlist = []
class Server(object):
def __init__(self, username, password, host, port=22):
self.transport = paramiko.Transport((host, port))
self.transport.connect(username=username, password=password)
self.sftp = paramiko.SFTPClient.from_transport(self.transport)
def upload(self, local, remote):
self.sftp.put(local, remote)
def download(self, remote, local):
self.sftp.get(remote, local)
def close(self):
if self.transport.is_active():
self.sftp.close()
self.transport.close()
def __enter__(self):
return self
def __exit__(self, type, value, tb):
self.close()
with Server("username", "pass", "x.x.x.x") as server:
for image in glob.glob("C:\\ftp\\Pangea\\*.*"):
base = path.basename(image)
server.upload(image, path.join(remotee, base))
for line in fileinput.input(locall + "\\list.txt"):
lastlist.append(line.rstrip("\n"))
currentlist = os.listdir(locall)
newfiles = list(set(currentlist) - set(lastlist))
if len(newfiles) == 0:
print("No files need to be uploaded")
for src_dir, dirs, files in os.walk(locall):
dst_dir = src_dir.replace(locall, complete)
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
for file_ in files:
src_file = os.path.join(src_dir, file_)
dst_file = os.path.join(dst_dir, file_)
if os.path.exists(dst_file):
os.remove(dst_file)
shutil.move(src_file, dst_dir)
shutil.copy(mylist, locall)
else:
for needupload in newfiles:
print("Uploading " + locall + "\\" + needupload)
with Server("username", "pass", "x.x.x.x") as server:
for image in glob.glob("C:\\ftp\\Pangea\\*.*"):
base = path.basename(image)
server.upload(image, path.join(remotee, base))
with open(locall + "\\list.txt", "a") as myfile:
myfile.write(needupload + "\n")
with open(complete + "\\list.txt", "a") as myfile:
myfile.write(needupload + "\n")

Create missing directories in ftplib storbinary

I was using pycurl to transfer files over ftp in python. I could create the missing directories automatically on my remote server using:
c.setopt(pycurl.FTP_CREATE_MISSING_DIRS, 1)
for some reasons, I have to switch to ftplib. But I don't know how to to the same here. Is there any option to add to storbinary function to do that? or I have to create the directories manually?

FTP_CREATE_MISSING_DIRS is a curl operation (added here). I'd hazard a guess that you have to do it manually with ftplib, but I'd love to be proven wrong, anyone?
I'd do something like the following: (untested, and need to catch ftplib.all_errors)
ftp = ... # Create connection
# Change directories - create if it doesn't exist
def chdir(dir):
if directory_exists(dir) is False: # (or negate, whatever you prefer for readability)
ftp.mkd(dir)
ftp.cwd(dir)
# Check if directory exists (in current location)
def directory_exists(dir):
filelist = []
ftp.retrlines('LIST',filelist.append)
for f in filelist:
if f.split()[-1] == dir and f.upper().startswith('D'):
return True
return False
Or you could do directory_exists like this: (a bit harder to read?)
# Check if directory exists (in current location)
def directory_exists(dir):
filelist = []
ftp.retrlines('LIST',filelist.append)
return any(f.split()[-1] == dir and f.upper().startswith('D') for f in filelist)

I know it's kind of an old post but I just needed this and came up with a very simple function. I'm new to Python so I'd appreciate any feedback.
from ftplib import FTP
ftp = FTP('domain.com', 'username', 'password')
def cdTree(currentDir):
if currentDir != "":
try:
ftp.cwd(currentDir)
except IOError:
cdTree("/".join(currentDir.split("/")[:-1]))
ftp.mkd(currentDir)
ftp.cwd(currentDir)
Usage example:
cdTree("/this/is/an/example")

I tried adding this as a comment to the #Alex L 's answer, but it was too long. You need to descend recursively when changing directory if you want to create directories on the way. E.g.
def chdir(ftp, directory):
ch_dir_rec(ftp,directory.split('/'))
# Check if directory exists (in current location)
def directory_exists(ftp, directory):
filelist = []
ftp.retrlines('LIST',filelist.append)
for f in filelist:
if f.split()[-1] == directory and f.upper().startswith('D'):
return True
return False
def ch_dir_rec(ftp, descending_path_split):
if len(descending_path_split) == 0:
return
next_level_directory = descending_path_split.pop(0)
if not directory_exists(ftp,next_level_directory):
ftp.mkd(next_level_directory)
ftp.cwd(next_level_directory)
ch_dir_rec(ftp,descending_path_split)

I am using the following lines to resolve missing directory paths for FTP file copy
import os
ftps = FTP_TLS('ftps_server')
ftps.connect()
ftps.login()
destination_dir_path = 'some/dir/path' # directory path on FTP
dir_path = ''
for i in destination_dir_path.split('/'):
dir_path = os.path.join(dir_path,i)
if i not in ftps.nlst(os.path.dirname(dir_path)):
ftps.mkd(dir_path) # create directory on the FTP
ftps.storbinary(...) # store file using the binary mode

An alternative is to simply loop through each of the path elements, create the next and change into the newly-created directory. My use case was fairly straightforward though as I was copying items from one FTP server to another.
def create_ftp_path(session: ftplib.FTP, required_dir: str):
required_dir = required_dir.split('/')[:-1]
for path_item in required_dir:
if path_item.strip() == '':
continue
path_item = path_item.replace('/', '')
try:
session.cwd(path_item)
except:
session.mkd(path_item)
session.cwd(path_item)
Considerations:
This function assumes you have already changed directory for your FTP session to some base path and the required_dir is a path from that base path.
required_dir includes file name as the last element.
I'm removing any / characters because in my case they were causing 553 permission denied exception.
The exception handling is lacking, but in my case upload validation is happening further in the code so even if it fails it will be caught further down.

A more robust and reliable solution:
using ftplib
Hostname = yourhostname.com
Username = yourusername
Password = yourpassword
def mkdirs(path):
ftp = FTP(Hostname,Username,Password)
items = path.split('/')
cwd = "/"
for i in range(len(items)):
list = ftp.nlst()
if(not '.' in items[i] and not items[i] in list):
ftp.mkd(cwd + items[i] + "/")
cwd += items[i] + '/'
ftp.cwd(cwd)
ftp.quit()
mkdirs('path/to/directory/file.name')
This will create directories on your server if they do not exist.
Limitations: This will not work on folders with names that contain ..

This code will create all missing folders in path:
...
def chdir(ftp_path, ftp_conn):
dirs = [d for d in ftp_path.split('/') if d != '']
for p in dirs:
print(p)
check_dir(p, ftp_conn)
def check_dir(dir, ftp_conn):
filelist = []
ftp_conn.retrlines('LIST', filelist.append)
found = False
for f in filelist:
if f.split()[-1] == dir and f.lower().startswith('d'):
found = True
if not found:
ftp_conn.mkd(dir)
ftp_conn.cwd(dir)
if __name__ == '__main__':
ftp_conn = ... # ftp connection
t = 'FTP/for_Vadim/1/2/3/'
chdir(t, ftp_conn)
This code will check all dirs in path and create missing dirs
before "FTP/for_Vadim/"
after "FTP/for_Vadim/1/2/3/"

I'm using something like this (without cwd):
# -*- coding:utf-8 -*-
from ftplib import FTP, error_perm
def createDirs(ftp, dirpath):
"""
Create dir with subdirs.
:param ftp: connected FTP
:param dirpath: path (like 'test/test1/test2')
:type ftp: FTP
:type dirpath: str
:rtype: None
"""
dirpath = dirpath.replace('\\', '/')
tmp = dirpath.split('/')
dirs = []
for _ in tmp:
if len(dirs) == 0:
dirs.append(_)
continue
dirs.append(dirs[-1] + '/' + _)
for _ in dirs:
try:
ftp.mkd(_)
except error_perm as e:
e_str = str(e)
if '550' in e_str and 'File exists' in e_str:
continue
if __name__ == '__main__':
# init ftp
createDirs(ftp=ftp, dirpath='test/1/2/3')

Delete all files and folders after connecting to FTP

I want to connect through FTP to an address and then delete all contents. Currently I am using this code:
from ftplib import FTP
import shutil
import os
ftp = FTP('xxx.xxx.xxx.xxx')
ftp.login("admin", "admin")
for ftpfile in ftp.nlst():
if os.path.isdir(ftpfile)== True:
shutil.rmtree(ftpfile)
else:
os.remove(ftpfile)
My problem is I always get this error when he is trying to delete the first file:
os.remove(ftpfile)
WindowsError: [Error 2] The system cannot find the file specified: somefile.sys
Anyone has an idea why?

for something in ftp.nlst():
try:
ftp.delete(something)
except Exception:
ftp.rmd(something)
Any other ways?

This function deletes any given path recursively:
# python 3.6
from ftplib import FTP
def remove_ftp_dir(ftp, path):
for (name, properties) in ftp.mlsd(path=path):
if name in ['.', '..']:
continue
elif properties['type'] == 'file':
ftp.delete(f"{path}/{name}")
elif properties['type'] == 'dir':
remove_ftp_dir(ftp, f"{path}/{name}")
ftp.rmd(path)
ftp = FTP(HOST, USER, PASS)
remove_ftp_dir(ftp, 'path_to_remove')

from ftplib import FTP
import shutil
import os
ftp = FTP("hostname")
ftp.login("username", "password")
def deletedir(dirname):
ftp.cwd(dirname)
print(dirname)
for file in ftp.nlst():
try:
ftp.delete(file)
except Exception:
deletedir(file)
ftp.cwd("..")
ftp.rmd(dirname)
for ftpfile in ftp.nlst():
try:
ftp.delete(ftpfile)
except Exception:
deletedir(ftpfile)

ftp.nlst()
The above statement returns a list of file names.
os.remove()
The above statement requires a file path.

from ftplib import FTP
#--------------------------------------------------
class FTPCommunicator():
def __init__(self):
self.ftp = FTP()
self.ftp.connect('server', port=21, timeout=30)
self.ftp.login(user="user", passwd="password")
def getDirListing(self, dirName):
listing = self.ftp.nlst(dirName)
# If listed a file, return.
if len(listing) == 1 and listing[0] == dirName:
return []
subListing = []
for entry in listing:
subListing += self.getDirListing(entry)
listing += subListing
return listing
def removeDir(self, dirName):
listing = self.getDirListing(dirName)
# Longest path first for deletion of sub directories first.
listing.sort(key=lambda k: len(k), reverse=True)
# Delete files first.
for entry in listing:
try:
self.ftp.delete(entry)
except:
pass
# Delete empty directories.
for entry in listing:
try:
self.ftp.rmd(entry)
except:
pass
self.ftp.rmd(dirName)
def quit(self):
self.ftp.quit()
#--------------------------------------------------
def main():
ftp = FTPCommunicator()
ftp.removeDir("/Untitled")
ftp.quit()
#--------------------------------------------------
if __name__ == "__main__":
main()
#--------------------------------------------------

#Dmytro Gierman has the best option in my opinion, however, I don't want to delete the main folder, only the subfolder and all files inside.
Here is a script in order to make it work
import ftplib
# python 3.9
def remove_ftp_dir(ftp, path, mainpath):
for (name, properties) in ftp.mlsd(path=path):
if name in ['.', '..']:
continue
elif properties['type'] == 'file':
ftp.delete(f"{path}/{name}")
elif properties['type'] == 'dir':
remove_ftp_dir(ftp, f"{path}/{name}", mainpath)
if path != mainpath: #only deletes subdirectories not he mainpath
ftp.rmd(path)