Develop Reference

How to calculate EC2 instance price using Python script [duplicate]

Now that AWS have a Pricing API, how could one use Boto3 to fetch the current hourly price for a given on-demand EC2 instance type (e.g. t2.micro), region (e.g. eu-west-1) and operating system (e.g. Linux)? I only want the price returned. Based on my understanding, having those four pieces of information should be enough to filter down to a singular result.
However, all the examples I've seen fetch huge lists of data from the API that would have to be post-processed in order to get what I want. I would like to filter the data on the API side, before it's being returned.

Here is the solution I ended up with. Using Boto3's own Pricing API with a filter for the instance type, region and operating system. The API still returns a lot of information, so I needed to do a bit of post-processing.
import boto3
import json
from pkg_resources import resource_filename
# Search product filter. This will reduce the amount of data returned by the
# get_products function of the Pricing API
FLT = '[{{"Field": "tenancy", "Value": "shared", "Type": "TERM_MATCH"}},'\
'{{"Field": "operatingSystem", "Value": "{o}", "Type": "TERM_MATCH"}},'\
'{{"Field": "preInstalledSw", "Value": "NA", "Type": "TERM_MATCH"}},'\
'{{"Field": "instanceType", "Value": "{t}", "Type": "TERM_MATCH"}},'\
'{{"Field": "location", "Value": "{r}", "Type": "TERM_MATCH"}},'\
'{{"Field": "capacitystatus", "Value": "Used", "Type": "TERM_MATCH"}}]'
# Get current AWS price for an on-demand instance
def get_price(region, instance, os):
f = FLT.format(r=region, t=instance, o=os)
data = client.get_products(ServiceCode='AmazonEC2', Filters=json.loads(f))
od = json.loads(data['PriceList'][0])['terms']['OnDemand']
id1 = list(od)[0]
id2 = list(od[id1]['priceDimensions'])[0]
return od[id1]['priceDimensions'][id2]['pricePerUnit']['USD']
# Translate region code to region name. Even though the API data contains
# regionCode field, it will not return accurate data. However using the location
# field will, but then we need to translate the region code into a region name.
# You could skip this by using the region names in your code directly, but most
# other APIs are using the region code.
def get_region_name(region_code):
default_region = 'US East (N. Virginia)'
endpoint_file = resource_filename('botocore', 'data/endpoints.json')
try:
with open(endpoint_file, 'r') as f:
data = json.load(f)
# Botocore is using Europe while Pricing API using EU...sigh...
return data['partitions'][0]['regions'][region_code]['description'].replace('Europe', 'EU')
except IOError:
return default_region
# Use AWS Pricing API through Boto3
# API only has us-east-1 and ap-south-1 as valid endpoints.
# It doesn't have any impact on your selected region for your instance.
client = boto3.client('pricing', region_name='us-east-1')
# Get current price for a given instance, region and os
price = get_price(get_region_name('eu-west-1'), 't3.micro', 'Linux')
print(price)
This example outputs 0.0114000000 (hourly price in USD) fairly quickly. (This number was verified to match the current value listed here at the date of this writing)

If you don't like the native function, then look at Lyft's awspricing library for Python. Here's an example:
import awspricing
ec2_offer = awspricing.offer('AmazonEC2')
p = ec2_offer.ondemand_hourly(
't2.micro',
operating_system='Linux',
region='eu-west-1'
)
print(p) # 0.0126
I'd recommend enabling caching (see AWSPRICING_USE_CACHE) otherwise it will be slow.

I have updated toringe's solution a bit to handle different key errors
def price_information(self, instance_type, os, region):
# Search product filter
FLT = '[{{"Field": "operatingSystem", "Value": "{o}", "Type": "TERM_MATCH"}},' \
'{{"Field": "instanceType", "Value": "{t}", "Type": "TERM_MATCH"}}]'
f = FLT.format(t=instance_type, o=os)
try:
data = self.pricing_client.get_products(ServiceCode='AmazonEC2', Filters=json.loads(f))
instance_price = 0
for price in data['PriceList']:
try:
first_id = list(eval(price)['terms']['OnDemand'].keys())[0]
price_data = eval(price)['terms']['OnDemand'][first_id]
second_id = list(price_data['priceDimensions'].keys())[0]
instance_price = price_data['priceDimensions'][second_id]['pricePerUnit']['USD']
if float(price) > 0:
break
except Exception as e:
print(e)
print(instance_price)
return instance_price
except Exception as e:
print(e)
return 0

Based on other answers, here's some code that returns the On Demand prices for all instance types (or for a given instance type, if you add the search filter), gets some relevant attributes for each instance type, and pretty-prints the data.
It assumes pricing is the AWS Pricing client.
import json
def ec2_get_ondemand_prices(Filters):
data = []
reply = pricing.get_products(ServiceCode='AmazonEC2', Filters=Filters, MaxResults=100)
data.extend([json.loads(r) for r in reply['PriceList']])
while 'NextToken' in reply.keys():
reply = pricing.get_products(ServiceCode='AmazonEC2', Filters=Filters, MaxResults=100, NextToken=reply['NextToken'])
data.extend([json.loads(r) for r in reply['PriceList']])
print(f"\x1b[33mGET \x1b[0m{len(reply['PriceList']):3} \x1b[94m{len(data):4}\x1b[0m")
instances = {}
for d in data:
attr = d['product']['attributes']
type = attr['instanceType']
if type in data: continue
region = attr.get('location', '')
clock = attr.get('clockSpeed', '')
type = attr.get('instanceType', '')
market = attr.get('marketoption', '')
ram = attr.get('memory', '')
os = attr.get('operatingSystem', '')
arch = attr.get('processorArchitecture', '')
region = attr.get('regionCode', '')
storage = attr.get('storage', '')
tenancy = attr.get('tenancy', '')
usage = attr.get('usagetype', '')
vcpu = attr.get('vcpu', '')
terms = d['terms']
ondemand = terms['OnDemand']
ins = ondemand[next(iter(ondemand))]
pricedim = ins['priceDimensions']
price = pricedim[next(iter(pricedim))]
desc = price['description']
p = float(price['pricePerUnit']['USD'])
unit = price['unit'].lower()
if 'GiB' not in ram: print('\x1b[31mWARN\x1b[0m')
if 'hrs'!=unit: print('\x1b[31mWARN\x1b[0m')
if p==0.: continue
instances[type] = {'type':type, 'market':market, 'vcpu':vcpu, 'ram':float(ram.replace('GiB','')), 'ondm':p, 'unit':unit, 'terms':list(terms.keys()), 'desc':desc}
instances = {k:v for k,v in sorted(instances.items(), key=lambda e: e[1]['ondm'])}
for ins in instances.values():
p = ins['ondm']
print(f"{ins['type']:32} {ins['market'].lower()}\x1b[91m: \x1b[0m{ins['vcpu']:3} vcores\x1b[91m, \x1b[0m{ins['ram']:7.1f} GB, \x1b[0m{p:7.4f} \x1b[95m$/h\x1b[0m, \x1b[0m\x1b[0m{p*720:8,.1f} \x1b[95m$/m\x1b[0m, \x1b[0m\x1b[0m{p*720*12:7,.0f} \x1b[95m$/y\x1b[0m, \x1b[0m{ins['unit']}\x1b[91m, \x1b[0m{ins['terms']}\x1b[0m")
# print(desc, , sep='\n')
print(f'\x1b[92m{len(instances)}\x1b[0m')
flt = [
# {'Field': 'instanceType', 'Value': 't4g.nano', 'Type': 'TERM_MATCH'}, # enable this filter to select only 1 instance type
{'Field': 'regionCode', 'Value': 'us-east-2', 'Type': 'TERM_MATCH'}, # alternative notation?: {'Field': 'location', 'Value': 'US East (Ohio)', 'Type': 'TERM_MATCH'},
{'Field': 'operatingSystem', 'Value': 'Linux', 'Type': 'TERM_MATCH'},
{'Field': 'tenancy', 'Value': 'shared', 'Type': 'TERM_MATCH'},
{'Field': 'capacitystatus', 'Value': 'Used', 'Type': 'TERM_MATCH'},
]
ec2_get_ondemand_prices(Filters=flt)

S3 show buckets last modified

I'm trying to list the last modified file in S3 buckets for a report but the report is showing the first modified (ie when the first file was uploaded not the last file).
I'm using this:
top_level_folders[folder]['modified'] = obj.last_modified
and adding to the report here:
report.add_row([folder[1]['name'], folder[1]['objects'],
str(round(folder[1]['size'],2)), status, folder[1]['modified']])
I've tried adding
=obj.last_modified, reverse=True but keep getting invalid syntax errors.
This is what the report looks like:

I'm not exactly sure as to what you're doing when it comes to writing to the report, but the code below will return a list of dictionaries with the name of each bucket and the time the last-modified file was last modified. E.g.,
[
{
'Folder': 'bucket_1',
'Last Modified': '2021-11-30 13:10:32+00:00'
},
{
'Folder': 'bucket_2',
'Last Modified': '2021-09-27 17:18:27+00:00'
}
]
import datetime
import boto3
s3_client = boto3.client('s3',
aws_access_key_id="AKXXXXXXXXXXXXXXXXXX",
aws_secret_access_key="YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY",
region_name="eu-west-2"
)
def find_last_modified_file_in_bucket(bucket_name: str) -> datetime:
last_modified = []
for bucket_object in s3_client.list_objects(Bucket=bucket_name)["Contents"]:
last_modified.append(bucket_object["LastModified"])
return max(last_modified)
def fetch_last_modified() -> [{}]:
last_modified_file_by_bucket: list[{}] = []
for bucket_name in list(map(lambda bucket: bucket["Name"], s3_client.list_buckets()["Buckets"])):
latest_time_of_last_modified_file: datetime = find_last_modified_file_in_bucket(bucket_name)
last_modified_file_by_bucket.append(
{
"Folder": bucket_name,
"Last Modified": str(latest_time_of_last_modified_file)
}
)
return last_modified_file_by_bucket
Without the source code or knowledge of the type of folder, I can't say with certainty how you would use the above code to update the folder, but it will likely come down to iterating over the dict returned by fetch_last_modified(). E.g.,
def update_report(report: Report, folder_with_last_modified: dict):
for folder in folder_with_last_modified:
report.add_row(folder['Folder'], folder['Last Modified'])
folder_with_last_modified = fetch_last_modified()
update_report(report, folder_with_last_modified)

using dict.update is overwriting the string of the key itself AND the key's value

I am working on a CLI for AWS, and I'm trying to get all instances of all EC2 into a single dict, across all regions.
def get_all_ec2_instances_in_all_regions(self):
ec2_instance_list = {}
region_list = list_ec2_instance_regions() #this returns a list of regions to iterate on
print('Finding Regions')
for region in region_list:
region_name = region['RegionName']
ec2 = boto3.client('ec2', region_name=region_name)
regional_instance_list_return = ec2.describe_instances()['Reservations']
if len(regional_instance_list_return) == 0:
ec2_instance_list.update({
region_name: regional_instance_list_return
})
for reservation in regional_instance_list_return:
instance_id= reservation['Instances'][0]['InstanceId']
ec2_instance_list.update({
region_name: {
instance_id: reservation['Instances'][0]
}
})
print('Region search complete')
print(prettyPrintDict(ec2_instance_list)) #prettyPrintDict just console logs the dict in a nicer format for human readability
The resulting object only has one single dict object with the instance_id as the key, even though regional_instance_list_return actually has a list with multiple objects in it.
I would figure this code would add dictionaries with each InstanceID Like this
{ region1_name:
{ instance_id1: {instance1 data},
instance_id2: {instance2 data},
instance_id3: {instance3 data}
}
region2_name:
{ instance_id1: {instance1 data},
instance_id2: {instance2 data},
instance_id3: {instance3 data},
instance_id4: {instance4 data}
}
... and so on
}
but the resulting dict actually looks like this when it's finished:
{ region1_name:
{
instance_id3: {instance3 data}
}
region2_name:
{
instance_id4: {instance4 data}
}
... and so on
}
It doesn't actually add each instance, it just overwrites the instance_id key (Which is unique for each instance) and the key's values.
I was under the impression that if a key is unique, and you use dict.update() it'll just add them all without overwriting? What am I doing wrong?

dict.update takes keyword assignment arguments and converts the keyword into the dict key. You are telling it that the key is 'instance_id', not the value assigned to the variable instance_id. Instead of dict.update, try the syntax dict[key] = value, which appends a new key/value pair if key does not already exist.

Ok so in the original code, it was overwriting the region each time with the dict.update so I had to make a few small modifications to make sure that the update was happening in the right leaf of the object:
def get_all_ec2_instances_in_all_regions(self):
ec2_instance_list = {}
region_list = self.list_ec2_instance_regions()
print('Finding Regions')
for region in region_list:
region_name = region['RegionName']
self.ec2 = boto3.client('ec2', region_name=region_name)
regional_instance_list_return = self.ec2.describe_instances()['Reservations']
ec2_instance_list.update({
region_name: {}
}) #update the region name no matter if there are instances or not
for reservation in regional_instance_list_return:
instance_id = reservation['Instances'][0]['InstanceId']
ec2_instance_list[region_name].update({
# update the list starting at the
# root of the region_name, not one
# level up the leaf as I had before
instance_id: reservation['Instances'][0]
})

Get hyperlink from a cell in google Sheet api V4

I want to get the hyperlink of a cell (A1, for example) in Python. I have this code so far. Thanks
properties = {
"requests": [
{
"cell": {
"HyperlinkDisplayType": "LINKED"
},
"fields": "userEnteredFormat.HyperlinkDisplayType"
}
]
}
result = service.spreadsheets().values().get(
spreadsheetId=spreadsheet_id, range=rangeName, body=properties).execute()
values = result.get('values', [])

How about using sheets.spreadsheets.get? This sample script supposes that service of your script has already been able to be used for spreadsheets().values().get().
Sample script :
spreadsheetId = '### Spreadsheet ID ###'
range = ['sheet1!A1:A1'] # This is a sample.
result = service.spreadsheets().get(
spreadsheetId=spreadsheetId,
ranges=range,
fields="sheets/data/rowData/values/hyperlink"
).execute()
If this was not useful for you, I'm sorry.

It seems to me like this is the only way to actually get the link info (address as well as display text):
result = service.spreadsheets().values().get(
spreadsheetId=spreadsheetId, range=range_name,
valueRenderOption='FORMULA').execute()
values = results.get('values', [])
This returns the raw content of the cells which for hyperlinks look like this for each cell:
'=HYPERLINK("sample-link","http://www.sample.com")'
For my use I've parsed it with the following simple regex:
r'=HYPERLINK\("(.*?)","(.*?)"\)'

You can check the hyperlink if you add at the end:
print (values[0])

Example of update_item in dynamodb boto3

Following the documentation, I'm trying to create an update statement that will update or add if not exists only one attribute in a dynamodb table.
I'm trying this
response = table.update_item(
Key={'ReleaseNumber': '1.0.179'},
UpdateExpression='SET',
ConditionExpression='Attr(\'ReleaseNumber\').eq(\'1.0.179\')',
ExpressionAttributeNames={'attr1': 'val1'},
ExpressionAttributeValues={'val1': 'false'}
)
The error I'm getting is:
botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem operation: ExpressionAttributeNames contains invalid key: Syntax error; key: "attr1"
If anyone has done anything similar to what I'm trying to achieve please share example.

Found working example here, very important to list as Keys all the indexes of the table, this will require additional query before update, but it works.
response = table.update_item(
Key={
'ReleaseNumber': releaseNumber,
'Timestamp': result[0]['Timestamp']
},
UpdateExpression="set Sanity = :r",
ExpressionAttributeValues={
':r': 'false',
},
ReturnValues="UPDATED_NEW"
)

Details on dynamodb updates using boto3 seem incredibly sparse online, so I'm hoping these alternative solutions are useful.
get / put
import boto3
table = boto3.resource('dynamodb').Table('my_table')
# get item
response = table.get_item(Key={'pkey': 'asdf12345'})
item = response['Item']
# update
item['status'] = 'complete'
# put (idempotent)
table.put_item(Item=item)
actual update
import boto3
table = boto3.resource('dynamodb').Table('my_table')
table.update_item(
Key={'pkey': 'asdf12345'},
AttributeUpdates={
'status': 'complete',
},
)

If you don't want to check parameter by parameter for the update I wrote a cool function that would return the needed parameters to perform a update_item method using boto3.
def get_update_params(body):
"""Given a dictionary we generate an update expression and a dict of values
to update a dynamodb table.
Params:
body (dict): Parameters to use for formatting.
Returns:
update expression, dict of values.
"""
update_expression = ["set "]
update_values = dict()
for key, val in body.items():
update_expression.append(f" {key} = :{key},")
update_values[f":{key}"] = val
return "".join(update_expression)[:-1], update_values
Here is a quick example:
def update(body):
a, v = get_update_params(body)
response = table.update_item(
Key={'uuid':str(uuid)},
UpdateExpression=a,
ExpressionAttributeValues=dict(v)
)
return response

The original code example:
response = table.update_item(
Key={'ReleaseNumber': '1.0.179'},
UpdateExpression='SET',
ConditionExpression='Attr(\'ReleaseNumber\').eq(\'1.0.179\')',
ExpressionAttributeNames={'attr1': 'val1'},
ExpressionAttributeValues={'val1': 'false'}
)
Fixed:
response = table.update_item(
Key={'ReleaseNumber': '1.0.179'},
UpdateExpression='SET #attr1 = :val1',
ConditionExpression=Attr('ReleaseNumber').eq('1.0.179'),
ExpressionAttributeNames={'#attr1': 'val1'},
ExpressionAttributeValues={':val1': 'false'}
)
In the marked answer it was also revealed that there is a Range Key so that should also be included in the Key. The update_item method must seek to the exact record to be updated, there's no batch updates, and you can't update a range of values filtered to a condition to get to a single record. The ConditionExpression is there to be useful to make updates idempotent; i.e. don't update the value if it is already that value. It's not like a sql where clause.
Regarding the specific error seen.
ExpressionAttributeNames is a list of key placeholders for use in the UpdateExpression, useful if the key is a reserved word.
From the docs, "An expression attribute name must begin with a #, and be followed by one or more alphanumeric characters". The error is because the code hasn't used an ExpressionAttributeName that starts with a # and also not used it in the UpdateExpression.
ExpressionAttributeValues are placeholders for the values you want to update to, and they must start with :

Based on the official example, here's a simple and complete solution which could be used to manually update (not something I would recommend) a table used by a terraform S3 backend.
Let's say this is the table data as shown by the AWS CLI:
$ aws dynamodb scan --table-name terraform_lock --region us-east-1
{
"Items": [
{
"Digest": {
"S": "2f58b12ae16dfb5b037560a217ebd752"
},
"LockID": {
"S": "tf-aws.tfstate-md5"
}
}
],
"Count": 1,
"ScannedCount": 1,
"ConsumedCapacity": null
}
You could update it to a new digest (say you rolled back the state) as follows:
import boto3
dynamodb = boto3.resource('dynamodb', 'us-east-1')
try:
table = dynamodb.Table('terraform_lock')
response = table.update_item(
Key={
"LockID": "tf-aws.tfstate-md5"
},
UpdateExpression="set Digest=:newDigest",
ExpressionAttributeValues={
":newDigest": "50a488ee9bac09a50340c02b33beb24b"
},
ReturnValues="UPDATED_NEW"
)
except Exception as msg:
print(f"Oops, could not update: {msg}")
Note the : at the start of ":newDigest": "50a488ee9bac09a50340c02b33beb24b" they're easy to miss or forget.

Small update of Jam M. Hernandez Quiceno's answer, which includes ExpressionAttributeNames to prevent encoutering errors such as:
"errorMessage": "An error occurred (ValidationException) when calling the UpdateItem operation:
Invalid UpdateExpression: Attribute name is a reserved keyword; reserved keyword: timestamp",
def get_update_params(body):
"""
Given a dictionary of key-value pairs to update an item with in DynamoDB,
generate three objects to be passed to UpdateExpression, ExpressionAttributeValues,
and ExpressionAttributeNames respectively.
"""
update_expression = []
attribute_values = dict()
attribute_names = dict()
for key, val in body.items():
update_expression.append(f" #{key.lower()} = :{key.lower()}")
attribute_values[f":{key.lower()}"] = val
attribute_names[f"#{key.lower()}"] = key
return "set " + ", ".join(update_expression), attribute_values, attribute_names
Example use:
update_expression, attribute_values, attribute_names = get_update_params(
{"Status": "declined", "DeclinedBy": "username"}
)
response = table.update_item(
Key={"uuid": "12345"},
UpdateExpression=update_expression,
ExpressionAttributeValues=attribute_values,
ExpressionAttributeNames=attribute_names,
ReturnValues="UPDATED_NEW"
)
print(response)

An example to update any number of attributes given as a dict, and keep track of the number of updates. Works with reserved words (i.e name).
The following attribute names shouldn't be used as we will overwrite the value: _inc, _start.
from typing import Dict
from boto3 import Session
def getDynamoDBSession(region: str = "eu-west-1"):
"""Connect to DynamoDB resource from boto3."""
return Session().resource("dynamodb", region_name=region)
DYNAMODB = getDynamoDBSession()
def updateItemAndCounter(db_table: str, item_key: Dict, attributes: Dict) -> Dict:
"""
Update item or create new. If the item already exists, return the previous value and
increase the counter: update_counter.
"""
table = DYNAMODB.Table(db_table)
# Init update-expression
update_expression = "SET"
# Build expression-attribute-names, expression-attribute-values, and the update-expression
expression_attribute_names = {}
expression_attribute_values = {}
for key, value in attributes.items():
update_expression += f' #{key} = :{key},' # Notice the "#" to solve issue with reserved keywords
expression_attribute_names[f'#{key}'] = key
expression_attribute_values[f':{key}'] = value
# Add counter start and increment attributes
expression_attribute_values[':_start'] = 0
expression_attribute_values[':_inc'] = 1
# Finish update-expression with our counter
update_expression += " update_counter = if_not_exists(update_counter, :_start) + :_inc"
return table.update_item(
Key=item_key,
UpdateExpression=update_expression,
ExpressionAttributeNames=expression_attribute_names,
ExpressionAttributeValues=expression_attribute_values,
ReturnValues="ALL_OLD"
)
Hope it might be useful to someone!

In a simple way you can use below code to update item value with new one:
response = table.update_item(
Key={"my_id_name": "my_id_value"}, # to get record
UpdateExpression="set item_key_name=:item_key_value", # Operation action (set)
ExpressionAttributeValues={":value": "new_value"}, # item that you need to update
ReturnValues="UPDATED_NEW" # optional for declarative message
)

Simple example with multiple fields:
import boto3
dynamodb_client = boto3.client('dynamodb')
dynamodb_client.update_item(
TableName=table_name,
Key={
'PK1': {'S': 'PRIMARY_KEY_VALUE'},
'SK1': {'S': 'SECONDARY_KEY_VALUE'}
}
UpdateExpression='SET #field1 = :field1, #field2 = :field2',
ExpressionAttributeNames={
'#field1': 'FIELD_1_NAME',
'#field2': 'FIELD_2_NAME',
},
ExpressionAttributeValues={
':field1': {'S': 'FIELD_1_VALUE'},
':field2': {'S': 'FIELD_2_VALUE'},
}
)

using previous answer from eltbus , it worked for me , except for minor bug,
You have to delete the extra comma using update_expression[:-1]