In my instance in the s3 I have a folder with N files, I need to iterate in this using this script bellow, I need to get all files and convert it, this script is hosted on a ec2 instance working with django.
I tryed a lot, by using the boto3 function get_object but all I get is nothing.
Can someone tell please how can I do something like that?, I'll need to download this files before convert or can I do it directly?
def upload_folder_to_s3(local_folder, destination_folder, s3_bucket):
'''
Function to upload a specific local folder to S3 bucket.
Parameters:
local_folder (str): Path to local folder.
destination_folder (str): Path to destination folder on S3.
s3_bucket (str): Bucket name on S3.
Return:
'''
# Global variables
global client
# Iterate over files on folder
for root, dirs, files in os.walk(local_folder):
for filename in files:
print(filename)
# construct the full local path
local_path = os.path.join(root, filename)
# construct the full Dropbox path
relative_path = os.path.relpath(local_path, local_folder)
s3_path = os.path.join(destination_folder, relative_path)
# relative_path = os.path.relpath(os.path.join(root, filename))
print('Searching "%s" in "%s"' % (s3_path, s3_bucket))
try:
client.get_object(Bucket=s3_bucket, Key=s3_path)
print("Path found on S3! Skipping %s..." % s3_path)
# try:
# client.delete_object(Bucket=bucket, Key=s3_path)
# except:
# print "Unable to delete %s..." % s3_path
except:
print("Uploading %s..." % s3_path)
client.upload_file(local_path, s3_bucket, s3_path)
return local_folder
Related
I want to download all images in a folder from a container i tried to download one image and it works, i need just to put the name of the image (Exemple: download_Image(bird.png)). It works only with this path path="files/exemple.png".
def download_Image(blobname):
blob_service_client = BlobServiceClient.from_connection_string(azure_storage_setting)
blob_client = blob_service_client.get_blob_client(container="images", blob=blobname)
try:
path="files/exemple.png"
download_file_path = os.path.join(path)
message='Download done!'
os.makedirs(os.path.dirname(download_file_path), exist_ok=True)
with open(download_file_path, "wb") as download_file:
download_file.write(blob_client.download_blob().readall())
except ResourceNotFoundError:
print("No blob found.")
message = "No blob found."
return message
download_Image(a)
I just want to download images with original names so i changed the path like this path = "files" and i had this error:
FileNotFoundError: [WinError 3] Das System kann den angegebenen Pfad nicht finden: ''
I tried to reproduce in my environment works fine successfully.
I have created a path folder along with sample images in blob storage like below.
To download all images in a folder from a container in azure blob storage make use of below code to execute.
import os
from azure.storage.blob import BlobServiceClient, BlobClient
MY_CONNECTION_STRING = "CONNECTION_STRING"
MY_BLOB_CONTAINER = "name" eplace with the local folder where you want files to be downloaded
LOCAL_BLOB_PATH = "Blobsss"
BLOBNAME="test"
class AzureBlobFileDownloader:
def __init__(self):
print("Intializing AzureBlobFileDownloader")
self.blob_service_client = BlobServiceClient.from_connection_string(MY_CONNECTION_STRING)
self.my_container = self.blob_service_client.get_container_client(MY_BLOB_CONTAINER)
def save_blob(self, file_name, file_content):
# Get full path to the file
download_file_path = os.path.join(LOCAL_BLOB_PATH, file_name)
os.makedirs(os.path.dirname(download_file_path), exist_ok=True)
with open(download_file_path, "wb") as file:
file.write(file_content)
def download_all_blobs_in_container(self):
my_blobs = self.my_container.list_blobs()
for blob in my_blobs:
print(blob.name)
bytes = self.my_container.get_blob_client(blob).download_blob().readall()
self.save_blob(blob.name, bytes)
azure_blob_file_downloader = AzureBlobFileDownloader()
azure_blob_file_downloader.download_all_blobs_in_container()
All images in the folder path are downloaded successfully like below.
Output:
What should I change in my code so that I can upload my entire folder from local system to a particular folder present in my s3 bucket.
import os
import boto3
s3_resource = boto3.resource("s3", region_name="ap-south-1")
def upload_objects():
try:
bucket_name = "<S3 bucket-name>"
root_path = '<local folder path>'
bucket_folder = '<S3 folder name>'
my_bucket = s3_resource.Bucket(bucket_name)
# s3 = boto3.resource('s3')
for path, subdirs, files in os.walk(root_path):
path = path.replace("\\","/")
directory_name = path.replace(root_path,"")
for file in files:
my_bucket.upload_file(os.path.join(path, file), directory_name+'/'+file)
except Exception as err:
print(err)
if __name__ == '__main__':
upload_objects()
You are not using your bucket_folder at all. This should be the beginning of your S3 prefix as in the S3 there are no folders. Its all about key names and prefixes.
So it should be something as the following:
my_bucket.upload_file(os.path.join(path, file), bucket_folder + '/' + directory_name+'/'+file)
I'm trying to upload a whole folder to dropbox but only the files get uploaded. Should I create a folder programatically or can I solve the folder-uploading so simple? Thanks
import os
import dropbox
access_token = '***********************'
dbx = dropbox.Dropbox(access_token)
dropbox_destination = '/live'
local_directory = 'C:/Users/xoxo/Desktop/man'
for root, dirs, files in os.walk(local_directory):
for filename in files:
local_path = root + '/' + filename
print("local_path", local_path)
relative_path = os.path.relpath(local_path, local_directory)
dropbox_path = dropbox_destination + '/' + relative_path
# upload the file
with open(local_path, 'rb') as f:
dbx.files_upload(f.read(), dropbox_path)
error:
dropbox.exceptions.ApiError: ApiError('xxf84e5axxf86', UploadError('path', UploadWriteFailed(reason=WriteError('disallowed_name', None), upload_session_id='xxxxxxxxxxx')))
[Cross-linking for reference: https://www.dropboxforum.com/t5/API-support/UploadWriteFailed-reason-WriteError-disallowed-name-None/td-p/245765 ]
There are a few things to note here:
In your sample, you're only iterating over files, so you won't get dirs uploaded/created.
The /2/files/upload endpoint only accepts file uploads, not folders. If you want to create folders, use /2/files/create_folder_v2. You don't need to explicitly create folders for any parent folders in the path for files you upload via /2/files/upload though. Those will be automatically created with the upload.
Per the /2/files/upload documentation, disallowed_name means:
Dropbox will not save the file or folder because of its name.
So, it's likely you're getting this error because you're trying to upload an ignored filed, e.g., ".DS_STORE". You can find more information on those in this help article under "Ignored files".
I'm trying to loop through a folder structure in python and upload each file it finds to a specified folder. The problem is that it's uploading a file with the correct name, however there is no content and the file size is only 10 bytes.
import dropbox, sys, os
try:
dbx = dropbox.Dropbox('some_access_token')
user = dbx.users_get_current_account()
except:
print ("Negative, Ghostrider")
sys.exit()
rootdir = os.getcwd()
print ("Attempting to upload...")
for subdir, dirs, files in os.walk(rootdir):
for file in files:
try:
dbx.files_upload("afolder",'/bfolder/' + file, mute=True)
print("Uploaded " + file)
except:
print("Failed to upload " + file)
print("Finished upload.")
Your call to dbx.files_upload("afolder",'/bfolder/' + file, mute=True) says: "Send the text afolder and write it as a file named '/bfolder/' + file".
From doc:
files_upload(f, path, mode=WriteMode('add', None), autorename=False, client_modified=None, mute=False)
Create a new file with the contents provided in the request.
Parameters:
f – A string or file-like obj of data.
path (str) – Path in the user’s Dropbox to save the file.
....
Meaning that f must be the content of the file (and not the filename string).
Here is a working example:
import dropbox, sys, os
dbx = dropbox.Dropbox('token')
rootdir = '/tmp/test'
print ("Attempting to upload...")
# walk return first the current folder that it walk, then tuples of dirs and files not "subdir, dirs, files"
for dir, dirs, files in os.walk(rootdir):
for file in files:
try:
file_path = os.path.join(dir, file)
dest_path = os.path.join('/test', file)
print 'Uploading %s to %s' % (file_path, dest_path)
with open(file_path) as f:
dbx.files_upload(f, dest_path, mute=True)
except Exception as err:
print("Failed to upload %s\n%s" % (file, err))
print("Finished upload.")
EDIT: For Python3 the following should be used:
dbx.files_upload(f.read(), dest_path, mute=True)
For Dropbox Business API below python code helps uploading files to dropbox.
#function code
import dropbox
def dropbox_file_upload(access_token,dropbox_file_path,local_file_name):
'''
The function upload file to dropbox.
Parameters:
access_token(str): Access token to authinticate dropbox
dropbox_file_path(str): dropboth file path along with file name
Eg: '/ab/Input/f_name.xlsx'
local_file_name(str): local file name with path from where file needs to be uploaded
Eg: 'f_name.xlsx' # if working directory
Returns:
Boolean:
True on successful upload
False on unsuccessful upload
'''
try:
dbx = dropbox.DropboxTeam(access_token)
# get the team member id for common user
members = dbx.team_members_list()
for i in range(0,len(members.members)):
if members.members[i].profile.name.display_name == logged_in_user:
member_id = members.members[i].profile.team_member_id
break
# connect to dropbox with member id
dbx = dropbox.DropboxTeam(access_token).as_user(member_id)
# upload local file to dropbox
f = open(local_file_name, 'rb')
dbx.files_upload(f.read(),dropbox_file_path)
return True
except Exception as e:
print(e)
return False
Ok, so I have to upload a directory, with subdirectories and files inside, on a FTP server. But I can't seem to get it right. I want to upload the directory as it is, with it's subdirectories and files where they were.
ftp = FTP()
ftp.connect('host',port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles():
for root,dirnames,filenames in os.walk(filenameCV):
for files in filenames:
print(files)
ftp.storbinary('STOR ' + files, open(files,'rb'))
ftp.quit()
placeFiles()
There are multiple problems with your code: First, the filenames array will only contain the actual filenames, not the entire path, so you need to join it with fullpath = os.path.join(root, files) and then use open(fullpath). Secondly, you quit the FTP connection inside the loop, move that ftp.quit() down on the level of the placeFiles() function.
To recursively upload your directory, you have to walk through your root directories and at the same time through your remote directory, uploading files on the go.
Full example code:
import os.path, os
from ftplib import FTP, error_perm
host = 'localhost'
port = 21
ftp = FTP()
ftp.connect(host,port)
ftp.login('user','pass')
filenameCV = "directorypath"
def placeFiles(ftp, path):
for name in os.listdir(path):
localpath = os.path.join(path, name)
if os.path.isfile(localpath):
print("STOR", name, localpath)
ftp.storbinary('STOR ' + name, open(localpath,'rb'))
elif os.path.isdir(localpath):
print("MKD", name)
try:
ftp.mkd(name)
# ignore "directory already exists"
except error_perm as e:
if not e.args[0].startswith('550'):
raise
print("CWD", name)
ftp.cwd(name)
placeFiles(ftp, localpath)
print("CWD", "..")
ftp.cwd("..")
placeFiles(ftp, filenameCV)
ftp.quit()