I'm trying to loop through a folder structure in python and upload each file it finds to a specified folder. The problem is that it's uploading a file with the correct name, however there is no content and the file size is only 10 bytes.
import dropbox, sys, os
try:
dbx = dropbox.Dropbox('some_access_token')
user = dbx.users_get_current_account()
except:
print ("Negative, Ghostrider")
sys.exit()
rootdir = os.getcwd()
print ("Attempting to upload...")
for subdir, dirs, files in os.walk(rootdir):
for file in files:
try:
dbx.files_upload("afolder",'/bfolder/' + file, mute=True)
print("Uploaded " + file)
except:
print("Failed to upload " + file)
print("Finished upload.")
Your call to dbx.files_upload("afolder",'/bfolder/' + file, mute=True) says: "Send the text afolder and write it as a file named '/bfolder/' + file".
From doc:
files_upload(f, path, mode=WriteMode('add', None), autorename=False, client_modified=None, mute=False)
Create a new file with the contents provided in the request.
Parameters:
f – A string or file-like obj of data.
path (str) – Path in the user’s Dropbox to save the file.
....
Meaning that f must be the content of the file (and not the filename string).
Here is a working example:
import dropbox, sys, os
dbx = dropbox.Dropbox('token')
rootdir = '/tmp/test'
print ("Attempting to upload...")
# walk return first the current folder that it walk, then tuples of dirs and files not "subdir, dirs, files"
for dir, dirs, files in os.walk(rootdir):
for file in files:
try:
file_path = os.path.join(dir, file)
dest_path = os.path.join('/test', file)
print 'Uploading %s to %s' % (file_path, dest_path)
with open(file_path) as f:
dbx.files_upload(f, dest_path, mute=True)
except Exception as err:
print("Failed to upload %s\n%s" % (file, err))
print("Finished upload.")
EDIT: For Python3 the following should be used:
dbx.files_upload(f.read(), dest_path, mute=True)
For Dropbox Business API below python code helps uploading files to dropbox.
#function code
import dropbox
def dropbox_file_upload(access_token,dropbox_file_path,local_file_name):
'''
The function upload file to dropbox.
Parameters:
access_token(str): Access token to authinticate dropbox
dropbox_file_path(str): dropboth file path along with file name
Eg: '/ab/Input/f_name.xlsx'
local_file_name(str): local file name with path from where file needs to be uploaded
Eg: 'f_name.xlsx' # if working directory
Returns:
Boolean:
True on successful upload
False on unsuccessful upload
'''
try:
dbx = dropbox.DropboxTeam(access_token)
# get the team member id for common user
members = dbx.team_members_list()
for i in range(0,len(members.members)):
if members.members[i].profile.name.display_name == logged_in_user:
member_id = members.members[i].profile.team_member_id
break
# connect to dropbox with member id
dbx = dropbox.DropboxTeam(access_token).as_user(member_id)
# upload local file to dropbox
f = open(local_file_name, 'rb')
dbx.files_upload(f.read(),dropbox_file_path)
return True
except Exception as e:
print(e)
return False
Related
I'd like to download tar.gz files over FTP. It uses FTP_TLS.
Files are downloaded but when I try to open them on Windows (I use 7zip), it's not working.
Error is :
"File [...] cannot be opened as an archive"
This is my code (It needs improvment I know I'm quite newbee :) ):
def get_ftp(ip, login, passwd, path):
""" Connexion FTP """
try:
with ftplib.FTP_TLS(ip, login, passwd) as ftps:
ftps.prot_p()
# ftp.login(login, passwd)
files = ftps.nlst('/home/user/dir/' + path)
# ftp.retrlines('LIST')
if files:
for file in files:
if file.endswith('.tar.gz'):
if file + '.md5' in files:
localfile = join(path_recu, basename(file))
with open(localfile, 'wb') as binary_file:
response = ftps.retrbinary('RETR %s' % file, binary_file.write, blocksize=8192, rest=None)
if response.startswith('226'):
with open(localfile, 'w') as text_file:
ftps.retrlines('RETR %s' % file + '.md5', text_file.write)
except ftplib.error_perm as resp:
if str(resp):
logger.critical('ERREUR : ' + repr(resp))
raise
else:
return files
I've tried with "blocksize=4096": same error.
Any ideas ?
I'm working on uploading some countries' admin data to my Dropbox app. Here below is my code that does that:
# importing the required libraries
import dropbox, sys, os
import requests
# Get your app key and secret from the Dropbox developer website
app_key = 'qie********'
app_secret = 'qom**********'
dbx = dropbox.Dropbox('YYPRp-*******************_JzclLe-***************-3Js')
# verify if the account is connected
dbx.users_get_current_account()
#find all the folders present
for entry in dbx.files_list_folder('').entries:
print(entry.name)
# creating a path to where all the data to be uploaded is
root_dir = "H:/WORK/Upwork/Project 7 - Python School Data Analysis/Planning and Costing Model/Updated Script/Extracted"
print ("Attempting to upload...")
z = 1
for dir, dirs, files in os.walk(root_dir):
# the first dir is the root dir itself so we skip it
if z == 1:
z = z + 1
continue
# uploading contents of the file path
elif z > 15:
# split the path to get the country, which is the very last item after split (-1)
split_dir = dir.split('\\')
folder_name = split_dir[-1] # country name
# creating a new folder in my Dropbox for each country
country_name = dbx.files_create_folder('/Data/'+ folder_name)
dropbox_folder = country_name.path_display #obtaining the name of the folder
folder_split = dropbox_folder.split('/') # splitting the path to get root folder and created folder
folder_created = folder_split[-1] #created/country folder
dest_path = os.path.join('/Data/', folder_created) #joining the two to make a full path
print(dest_path)
# looping through the files in each of the country's folder
for file in files:
try:
# getting the path for each of the file in the folder
file_path = os.path.join(dir, file)
print(f'Uploading to {folder_name} in Dropbox')
f = open(file_path, 'rb')
connect = '/' # will be used to separate the destination path and the file
# this is where the file will be saved
d_path = os.path.join(dest_path, connect, file)
dbx.files_upload(f.read(), d_path, mode=dropbox.files.WriteMode.overwrite)
print(dest_path)
print(file_path)
print(dir)
print('\n')
except Exception as err:
print("Error!", file, err)
z = z + 1
The code runs successfully with no errors. Here is how it looks at the console:
It successfully creates the folders for each of the countries. Note that in my countries' folders, it has several files (max of 15). When I visit my dropbox app, the folders are there but nothing is inside the folders. There are completely not files, I receive the message notification that says:
This Folder is Empty
See below the images:
With folders created
One of the countries, with no files:
I have given it over an hour but nothing changes. Also note that I configured all permissions necessary for writing files and folders. Could there be something I may have done wrong? I will appreciate any assistance. Thanks!
After some help from Greg, I was able to find where the issue was. The files_upload class functions expects a working path as part of its parameters. The path provided could not be found in the App and so I added the following to make it work: d_path = dest_path+d_path
Here is the full working code:
# importing the required libraries
import dropbox, sys, os
import requests
# Get your app key and secret from the Dropbox developer website
app_key = 'qie********'
app_secret = 'qom**********'
dbx = dropbox.Dropbox('YYPRp-*******************_JzclLe-***************-3Js')
# verify if the account is connected
dbx.users_get_current_account()
#find all the folders present
for entry in dbx.files_list_folder('').entries:
print(entry.name)
# creating a path to where all the data to be uploaded is
root_dir = "H:/WORK/Upwork/Project 7 - Python School Data Analysis/Planning and Costing Model/Updated Script/Extracted"
print ("Attempting to upload...")
z = 1
for dir, dirs, files in os.walk(root_dir):
# the first dir is the root dir itself so we skip it
if z == 1:
z = z + 1
continue
# uploading contents of the file path
elif z > 15:
# split the path to get the country, which is the very last item after split (-1)
split_dir = dir.split('\\')
folder_name = split_dir[-1] # country name
# creating a new folder in my Dropbox for each country
country_name = dbx.files_create_folder('/Data/'+ folder_name)
dropbox_folder = country_name.path_display #obtaining the name of the folder
folder_split = dropbox_folder.split('/') # splitting the path to get root folder and created folder
folder_created = folder_split[-1] #created/country folder
dest_path = os.path.join('/Data/', folder_created) #joining the two to make a full path
print(dest_path)
# looping through the files in each of the country's folder
for file in files:
try:
# getting the path for each of the file in the folder
file_path = os.path.join(dir, file)
print(f'Uploading to {folder_name} in Dropbox')
f = open(file_path, 'rb')
connect = '/' # will be used to separate the destination path and the file
# this is where the file will be saved
d_path = os.path.join(dest_path, connect, file)
d_path = dest_path+d_path
dbx.files_upload(f.read(), d_path, mode=dropbox.files.WriteMode.overwrite)
print(dest_path)
print(file_path)
print(dir)
print('\n')
except Exception as err:
print("Error!", file, err)
z = z + 1
In my instance in the s3 I have a folder with N files, I need to iterate in this using this script bellow, I need to get all files and convert it, this script is hosted on a ec2 instance working with django.
I tryed a lot, by using the boto3 function get_object but all I get is nothing.
Can someone tell please how can I do something like that?, I'll need to download this files before convert or can I do it directly?
def upload_folder_to_s3(local_folder, destination_folder, s3_bucket):
'''
Function to upload a specific local folder to S3 bucket.
Parameters:
local_folder (str): Path to local folder.
destination_folder (str): Path to destination folder on S3.
s3_bucket (str): Bucket name on S3.
Return:
'''
# Global variables
global client
# Iterate over files on folder
for root, dirs, files in os.walk(local_folder):
for filename in files:
print(filename)
# construct the full local path
local_path = os.path.join(root, filename)
# construct the full Dropbox path
relative_path = os.path.relpath(local_path, local_folder)
s3_path = os.path.join(destination_folder, relative_path)
# relative_path = os.path.relpath(os.path.join(root, filename))
print('Searching "%s" in "%s"' % (s3_path, s3_bucket))
try:
client.get_object(Bucket=s3_bucket, Key=s3_path)
print("Path found on S3! Skipping %s..." % s3_path)
# try:
# client.delete_object(Bucket=bucket, Key=s3_path)
# except:
# print "Unable to delete %s..." % s3_path
except:
print("Uploading %s..." % s3_path)
client.upload_file(local_path, s3_bucket, s3_path)
return local_folder
Ok, so I have to upload a directory, with subdirectories and files inside, on a FTP server. But I can't seem to get it right. I want to upload the directory as it is, with it's subdirectories and files where they were.
ftp = FTP()
ftp.connect('host',port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles():
for root,dirnames,filenames in os.walk(filenameCV):
for files in filenames:
print(files)
ftp.storbinary('STOR ' + files, open(files,'rb'))
ftp.quit()
placeFiles()
There are multiple problems with your code: First, the filenames array will only contain the actual filenames, not the entire path, so you need to join it with fullpath = os.path.join(root, files) and then use open(fullpath). Secondly, you quit the FTP connection inside the loop, move that ftp.quit() down on the level of the placeFiles() function.
To recursively upload your directory, you have to walk through your root directories and at the same time through your remote directory, uploading files on the go.
Full example code:
import os.path, os
from ftplib import FTP, error_perm
host = 'localhost'
port = 21
ftp = FTP()
ftp.connect(host,port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles(ftp, path):
for name in os.listdir(path):
localpath = os.path.join(path, name)
if os.path.isfile(localpath):
print("STOR", name, localpath)
ftp.storbinary('STOR ' + name, open(localpath,'rb'))
elif os.path.isdir(localpath):
print("MKD", name)
try:
ftp.mkd(name)
# ignore "directory already exists"
except error_perm as e:
if not e.args[0].startswith('550'):
raise
print("CWD", name)
ftp.cwd(name)
placeFiles(ftp, localpath)
print("CWD", "..")
ftp.cwd("..")
placeFiles(ftp, filenameCV)
ftp.quit()
This will not download the contents of sub-directories; how can I do so?
import ftplib
import configparser
import os
directories = []
def add_directory(line):
if line.startswith('d'):
bits = line.split()
dirname = bits[8]
directories.append(dirname)
def makeDir(archiveTo):
for dir in directories:
newDir = os.path.join(archiveTo, dir)
if os.path.isdir(newDir) == True:
print("Directory \"" + dir + "\" already exists!")
else:
os.mkdir(newDir)
def getFiles(archiveTo, ftp):
files = ftp.nlst()
for filename in files:
try:
directories.index(filename)
except:
ftp.retrbinary('RETR %s' % filename, open(os.path.join(archiveTo, filename), 'wb').write)
def runBackups():
#Load INI
filename = 'connections.ini'
config = configparser.SafeConfigParser()
config.read(filename)
connections = config.sections()
i = 0
while i < len(connections):
#Load Settings
uri = config.get(connections[i], "uri")
username = config.get(connections[i], "username")
password = config.get(connections[i], "password")
backupPath = config.get(connections[i], "backuppath")
archiveTo = config.get(connections[i], "archiveto")
#Start Back-ups
ftp = ftplib.FTP(uri)
ftp.login(username, password)
ftp.cwd(backupPath)
#Map Directory Tree
ftp.retrlines('LIST', add_directory)
#Make Directories Locally
makeDir(archiveTo)
#Gather Files
getFiles(archiveTo, ftp)
#End connection and increase counter.
ftp.quit()
i += 1
print()
print("Back-ups complete.")
print()
this should do the trick :)
import sys
import ftplib
import os
from ftplib import FTP
ftp=FTP("ftp address")
ftp.login("user","password")
def downloadFiles(path,destination):
#path & destination are str of the form "/dir/folder/something/"
#path should be the abs path to the root FOLDER of the file tree to download
try:
ftp.cwd(path)
#clone path to destination
os.chdir(destination)
os.mkdir(destination[0:len(destination)-1]+path)
print destination[0:len(destination)-1]+path+" built"
except OSError:
#folder already exists at destination
pass
except ftplib.error_perm:
#invalid entry (ensure input form: "/dir/folder/something/")
print "error: could not change to "+path
sys.exit("ending session")
#list children:
filelist=ftp.nlst()
for file in filelist:
try:
#this will check if file is folder:
ftp.cwd(path+file+"/")
#if so, explore it:
downloadFiles(path+file+"/",destination)
except ftplib.error_perm:
#not a folder with accessible content
#download & return
os.chdir(destination[0:len(destination)-1]+path)
#possibly need a permission exception catch:
with open(os.path.join(destination,file),"wb") as f:
ftp.retrbinary("RETR "+file, f.write)
print file + " downloaded"
return
source="/ftproot/folder_i_want/"
dest="/systemroot/where_i_want_it/"
downloadFiles(source,dest)
This is a very old question, but I had a similar need that i wanted to satisfy in a very general manner. I ended up writing my own solution that works very well for me. I've placed it on Gist here https://gist.github.com/Jwely/ad8eb800bacef9e34dd775f9b3aad987
and pasted it below in case i ever take the gist offline.
Example usage:
import ftplib
ftp = ftplib.FTP(mysite, username, password)
download_ftp_tree(ftp, remote_dir, local_dir)
The code above will look for a directory called "remote_dir" on the ftp host, and then duplicate the directory and its entire contents into the "local_dir".
It invokes the script below.
import ftplib
import os
def _is_ftp_dir(ftp_handle, name, guess_by_extension=True):
""" simply determines if an item listed on the ftp server is a valid directory or not """
# if the name has a "." in the fourth to last position, its probably a file extension
# this is MUCH faster than trying to set every file to a working directory, and will work 99% of time.
if guess_by_extension is True:
if name[-4] == '.':
return False
original_cwd = ftp_handle.pwd() # remember the current working directory
try:
ftp_handle.cwd(name) # try to set directory to new name
ftp_handle.cwd(original_cwd) # set it back to what it was
return True
except:
return False
def _make_parent_dir(fpath):
""" ensures the parent directory of a filepath exists """
dirname = os.path.dirname(fpath)
while not os.path.exists(dirname):
try:
os.mkdir(dirname)
print("created {0}".format(dirname))
except:
_make_parent_dir(dirname)
def _download_ftp_file(ftp_handle, name, dest, overwrite):
""" downloads a single file from an ftp server """
_make_parent_dir(dest)
if not os.path.exists(dest) or overwrite is True:
with open(dest, 'wb') as f:
ftp_handle.retrbinary("RETR {0}".format(name), f.write)
print("downloaded: {0}".format(dest))
else:
print("already exists: {0}".format(dest))
def _mirror_ftp_dir(ftp_handle, name, overwrite, guess_by_extension):
""" replicates a directory on an ftp server recursively """
for item in ftp_handle.nlst(name):
if _is_ftp_dir(ftp_handle, item):
_mirror_ftp_dir(ftp_handle, item, overwrite, guess_by_extension)
else:
_download_ftp_file(ftp_handle, item, item, overwrite)
def download_ftp_tree(ftp_handle, path, destination, overwrite=False, guess_by_extension=True):
"""
Downloads an entire directory tree from an ftp server to the local destination
:param ftp_handle: an authenticated ftplib.FTP instance
:param path: the folder on the ftp server to download
:param destination: the local directory to store the copied folder
:param overwrite: set to True to force re-download of all files, even if they appear to exist already
:param guess_by_extension: It takes a while to explicitly check if every item is a directory or a file.
if this flag is set to True, it will assume any file ending with a three character extension ".???" is
a file and not a directory. Set to False if some folders may have a "." in their names -4th position.
"""
os.chdir(destination)
_mirror_ftp_dir(ftp_handle, path, overwrite, guess_by_extension)
this is an alternative. you can try using ftputil package. You can then use it to walk the remote directories and get your files
Using ftp.mlsd() instead of ftp.nlst():
import sys
import ftplib
import os
from ftplib import FTP
def fetchFiles(ftp, path, destination, overwrite=True):
'''Fetch a whole folder from ftp. \n
Parameters
----------
ftp : ftplib.FTP object
path : string ('/dir/folder/')
destination : string ('D:/dir/folder/') folder where the files will be saved
overwrite : bool - Overwrite file if already exists.
'''
try:
ftp.cwd(path)
os.mkdir(destination[:-1] + path)
print('New folder made: ' + destination[:-1] + path)
except OSError:
# folder already exists at the destination
pass
except ftplib.error_perm:
# invalid entry (ensure input form: "/dir/folder/")
print("error: could not change to " + path)
sys.exit("ending session")
# list children:
filelist = [i for i in ftp.mlsd()]
print('Current folder: ' + filelist.pop(0)[0])
for file in filelist:
if file[1]['type'] == 'file':
fullpath = os.path.join(destination[:-1] + path, file[0])
if (not overwrite and os.path.isfile(fullpath)):
continue
else:
with open(fullpath, 'wb') as f:
ftp.retrbinary('RETR ' + file[0], f.write)
print(file[0] + ' downloaded')
elif file[1]['type'] == 'dir':
fetchFiles(ftp, path + file[0] + '/', destination, overwrite)
else:
print('Unknown type: ' + file[1]['type'])
if __name__ == "__main__":
ftp = FTP('ftp address')
ftp.login('user', 'password')
source = r'/Folder/'
dest = r'D:/Data/'
fetchFiles(ftp, source, dest, overwrite=True)
ftp.quit()
Using ftputil, a fast solution could be:
def download(folder):
for item in ftp.walk(folder):
print("Creating dir " + item[0])
os.mkdir(item[0])
for subdir in item[1]:
print("Subdirs " + subdir)
for file in item[2]:
print(r"Copying File {0} \ {1}".format(item[0], file))
ftp.download(ftp.path.join(item[0],file), os.path.join(item[0],file))
It is non-trivial at least. In the simplest case, you only assume you have files and directories. This isn't always the case, there are softlinks and hardlinks and Windows-style shortcut. Softlink and directory shortcut are particularly problematic since they make recursive directory possible, which would confuse naive-ly implemented ftp grabber.
How would you handle such recursive directory depends on your need; you might simply not follow softlinks or you might try to detect recursive links. Detecting recursive link is inherently tricky, you cannot do it reliably.