I want to convert m4a files uploaded to S3 to mp3. The files would only be 15 seconds max, so using Elastic Transcoder would be overkill. I downloaded a binary from https://www.johnvansickle.com/ffmpeg/. But I am very new to AWS and im still not sure how uploading binaries works. How would I would include it so that I could convert a file?
import boto3
import urllib
print('Loading function')
s3 = boto3.client('s3')
def lambda_handler(event, context):
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(event['Records'][0]['s3']['object']['key'].encode('utf8'))
try:
#convert to mp3
#upload to bucket
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
Related
Requirement: Find out unencrypted s3 buckets from AWS account and add tag to it.
Implemented so far
import boto3
from botocore.exceptions import ClientError
# Retrieve the list of existing buckets
s3 = boto3.client('s3')
response = s3.list_buckets()
# Find out unencrypted bucket list
for bucket in response['Buckets']:
try:
enc = s3.get_bucket_encryption(Bucket=bucket["Name"])
except ClientError as e:
if e.response['Error']['Code'] == 'ServerSideEncryptionConfigurationNotFoundError':
print('Bucket with no server-side encryption: %s' % (bucket['Name']))
else:
print("Bucket with unexpected error: %s, unexpected error: %s" % (bucket['Name'], e))
Following line gives me the unencrypted bucketslist:
print('Bucket with no server-side encryption: %s' % (bucket['Name']))
Result:
Bucket with no server-side encryption: xyz1
Bucket with no server-side encryption: xyz2
Need support for following
I can get the list of unencrypted s3 buckets but not sure how to use the output from except python code and utilize unencrypted bucket names to add tag later.
If you declare a list outside of your try-catch, you can access it later on
E.g.
import boto3
from botocore.exceptions import ClientError
#this is our new list
buckets = []
# Retrieve the list of existing buckets
s3 = boto3.client('s3')
response = s3.list_buckets()
# Find out unencrypted bucket list
for bucket in response['Buckets']:
try:
enc = s3.get_bucket_encryption(Bucket=bucket["Name"])
except ClientError as e:
if e.response['Error']['Code'] == 'ServerSideEncryptionConfigurationNotFoundError':
#add the bucket name to our new list
buckets.append(bucket['Name'])
print('Bucket with no server-side encryption: %s' % (bucket['Name']))
else:
print("Bucket with unexpected error: %s, unexpected error: %s" % (bucket['Name'], e))
#now you can use the "buckets" variable and it will contain all the unencrypted buckets
for bucket in buckets:
print(bucket)
I've been scratching my head on this for days now. I still couldn't solve the problem. Basically, I just wanted to put a CSV file in a LocalStack S3, and I can't get it working.
Here's the snippet of my code:
api.py
from files import s3, AWS_S3_BUCKET_NAME, upload_file_to_bucket
#router.post('/api/customer/generate/upload',
name='Upload CSV to AWS S3 Bucket',
status_code=201)
async def post_upload_user_csv(file_obj: UploadFile = File(...)):
upload_obj = upload_file_to_bucket(s3_client=s3(),
file_obj=file_obj.file,
bucket=AWS_S3_BUCKET_NAME,
folder='CSV', # To Be updated
object_name=file_obj.filename)
if upload_obj:
return JSONResponse(content="Object has been uploaded to bucket successfully",
status_code=status.HTTP_201_CREATED)
else:
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="File could not be uploaded")
files.py
import os
import boto3
import logging
from botocore.client import BaseClient
from botocore.exceptions import ClientError
AWS_ACCESS_KEY_ID = os.getenv('POSTGRES_HOST')
AWS_SECRET_KEY = os.getenv('AWS_SECRET_KEY')
AWS_S3_BUCKET_NAME = os.getenv('AWS_S3_BUCKET_NAME')
def s3() -> BaseClient:
client = boto3.client(service_name='s3',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_KEY,
endpoint_url='http://localhost:4566/') # Use LocalStack Endpoint
return client
def upload_file_to_bucket(s3_client, file_obj, bucket, folder, object_name=None):
"""Upload a file to an S3 bucket
:param s3_client: S3 Client
:param file_obj: File to upload
:param bucket: Bucket to upload to
:param folder: Folder to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_obj
# Upload the file
try:
# with open("files", "rb") as f:
s3_client.upload_fileobj(file_obj, bucket, f"{folder}/{object_name}")
except ClientError as e:
logging.error(e)
return False
return True
The problem is that s3_client needs to open the file in binary mode first before I can upload it to s3 bucket. However, this can't be done directly and the file needs to be saved temporarily on the FastAPI server but I really don't want to do that for obvious reasons.
Any help will be much appreciated. Thank you in advance!
Can you show the error first?
Bassicly. I have done it before. Upload directly a file from the front-end to aws-s3. Can you try to add ContentType to upload_fileobj. Here is my code
content_type = mimetypes.guess_type(fpath)[0]
s3.Bucket(bucket_name).upload_fileobj(Fileobj=file, Key=file_path,
ExtraArgs={"ACL": "public-read",
"ContentType": content_type})
another way. You should try to convert files to io.BytesIO
def s3_upload(self, file, file_path, bucket_name, width=None, height=None, make_thumb=False, make_cover=False):
s3 = boto3.resource(service_name='s3')
obj = BytesIO(self.image_optimize_from_buffer(file, width, height, make_thumb, make_cover))
s3.Bucket(bucket_name).upload_fileobj(Fileobj=obj, Key=file_path,
ExtraArgs={"ACL": "public-read", "ContentType": file.content_type})
return f'https://{bucket_name}.s3.amazonaws.com/{file_path}'
I have flask python rest api which is called by another flask rest api.
the input for my api is one parquet file (FileStorage object) and ECS connection and bucket details.
I want to save parquet file to ECS in a specific folder using boto or boto3
the code I have tried
def uploadFileToGivenBucket(self,inputData,file):
BucketName = inputData.ecsbucketname
calling_format = OrdinaryCallingFormat()
client = S3Connection(inputData.access_key_id, inputData.secret_key, port=inputData.ecsport,
host=inputData.ecsEndpoint, debug=2,
calling_format=calling_format)
#client.upload_file(BucketName, inputData.filename, inputData.folderpath)
bucket = client.get_bucket(BucketName,validate=False)
key = boto.s3.key.Key(bucket, inputData.filename)
fileName = NamedTemporaryFile(delete=False,suffix=".parquet")
file.save(fileName)
with open(fileName.name) as f:
key.send_file(f)
but it is not working and giving me error like...
signature_host = '%s:%d' % (self.host, port)
TypeError: %d format: a number is required, not str
I tried google but no luck Can anyone help me with this or any sample code for the same.
After a lot of hit and tried and time, I finally got the solution. I posting it for everyone else who are facing the same issue.
You need to use Boto3 and here is the code...
def uploadFileToGivenBucket(self,inputData,file):
BucketName = inputData.ecsbucketname
#bucket = client.get_bucket(BucketName,validate=False)
f = NamedTemporaryFile(delete=False,suffix=".parquet")
file.save(f)
endpointurl = "<your endpoints>"
s3_client = boto3.client('s3',endpoint_url=endpointurl, aws_access_key_id=inputData.access_key_id,aws_secret_access_key=inputData.secret_key)
try:
newkey = 'yourfolderpath/anotherfolder'+inputData.filename
response = s3_client.upload_file(f.name, BucketName,newkey)
except ClientError as e:
logging.error(e)
return False
return True
I have the following code posted below which gets all the s3 bucket list on aws and I am trying to write code that checks if the buckets are encrypted in python but I am having trouble figuring out how to do that. Can anyone tell me how to modify my code to do that. I tried online examples and looked at the documentation.
my code is:
from __future__ import print_function
import boto3
import os
os.environ['AWS_DEFAULT_REGION'] = "us-east-1"
# Create an S3 client
s3 = boto3.client('s3')
# Call S3 to list current buckets
response = s3.list_buckets()
# Get a list of all bucket names from the response
buckets = [bucket['Name'] for bucket in response['Buckets']]
# Print out the bucket list
print("Bucket List: %s" % buckets)
Tried the following codes but they don't work:
s3 = boto3.resource('s3')
bucket = s3.Bucket('my-bucket-name')
for obj in bucket.objects.all():
key = s3.Object(bucket.name, obj.key)
print key.server_side_encryption
and
#!/usr/bin/env python
import boto3
s3_client = boto3.client('s3')
head = s3_client.head_object(
Bucket="<S3 bucket name>",
Key="<S3 object key>"
)
if 'ServerSideEncryption' in head:
print head['ServerSideEncryption']
It's first worth understanding a few things about S3 and encryption.
When you enable default encryption on an S3 bucket, you're actually configuring a server-side encryption configuration rule on the bucket that will cause S3 to encrypt every object uploaded to the bucket after the rule was configured.
Unrelated to #1, you can apply an S3 bucket policy to a bucket, denying any uploads of objects that are not encrypted. This will prevent you from adding unencrypted data but it will not automatically encrypt anything.
You can encrypt uploads on an object-by-object basis; encryption does not have to be bucket-wide.
So, one way to find out which buckets fall into category #1 (will automatically encrypt anything uploaded to them), you can do this:
import boto3
from botocore.exceptions import ClientError
s3 = boto3.client('s3')
response = s3.list_buckets()
for bucket in response['Buckets']:
try:
enc = s3.get_bucket_encryption(Bucket=bucket['Name'])
rules = enc['ServerSideEncryptionConfiguration']['Rules']
print('Bucket: %s, Encryption: %s' % (bucket['Name'], rules))
except ClientError as e:
if e.response['Error']['Code'] == 'ServerSideEncryptionConfigurationNotFoundError':
print('Bucket: %s, no server-side encryption' % (bucket['Name']))
else:
print("Bucket: %s, unexpected error: %s" % (bucket['Name'], e))
This will result in output like this:
Bucket: mycats, no server-side encryption
Bucket: mydogs, no server-side encryption
Bucket: mytaxreturn, Encryption: [{'ApplyServerSideEncryptionByDefault': {'SSEAlgorithm': 'AES256'}}]
i am trying to write my streamed data(json format) to s3 bucket. i am using below code but not able to write. No error while executing the below code. but no json files in s3
class TweetsListener( StreamListener):
def __init__(self,path):
self.path = path
def on_data(self, data):
try:
s3 = boto3.resource('s3')
s3.put_object(Bucket='bucket',Body=data.encode('UTF-8'),Key='/A/'+self.path+'/test.json')
return True
except BaseException as e:
print("Error on_data: %s" % str(e))
return True
def on_error(self, status):
print(status)
return True
From what I can tell, you are trying to use put_object action on boto3's S3 Service Resource instead of the S3 Client.
The Service Resource doesn't have a put_object method.
Also, you should remove the leading / in the Key, and make sure that your bucket is already created.