Develop Reference

How to create an EMR cluster and submit a spark-submit step using Dagster?

I want to create a Dagster app that creates an EMR cluster and adds a spark-submit step, but due to a lack of documentation or examples I can't figure out how to do that (copilot also struggles with it :-)).
The idea is to create a scheduler with Dagster that creates an EMR cluster and runs scala-spark app as one of its steps.
Here's the code I have (it's not working correctly, but you may get a sense about what I was trying to do):
from dagster_shell import create_shell_command_op
from dagster_aws.emr.emr import EmrJobRunner
from dagster import graph, op
#op
def create_emr_cluster(context):
emr_job_runner = EmrJobRunner('us-east-1', aws_access_key_id='ACCESS_KEY', aws_secret_access='SECRET_KEY')
cluster_id = emr_job_runner.create_cluster()
step_dict = emr_job_runner.construct_step_dict_for_command('Spark Step', 'spark-submit --class org.apache.spark.examples.SparkPi --deploy-mode cluster s3://my-bucket/spark-examples.jar stage')
emr_job_runner.add_job_flow_steps(None, cluster_id, [step_dict])
#graph
def my_graph():
# a = create_shell_command_op('echo "hello, world!"', name="a") # this will invoke spark-submit on an existing cluster
# a()
create_emr_cluster()
my_job = my_graph.to_job()
How can I do it?

You had most of your components correctly setup. You were only missing EMR job flow settings which sets the application you want to use(on EMR), core/task node setup and so on..
More details here:
https://docs.aws.amazon.com/emr/latest/APIReference/API_RunJobFlow.html
Dagster api has a function run_job_flow which takes this input and creates a cluster.
Sharing a sample code snippet
from dagster_aws.emr import EmrJobRunner
REGION="us-east-1"
emr_cluster_config = {
"Applications": [
{
"Name": "Spark"
}
],
"JobFlowRole": "SomeRole",
"Instances": {
"Ec2SubnetId": "subnet-1",
"EmrManagedSlaveSecurityGroup": "sg-slave",
"EmrManagedMasterSecurityGroup": "sg-master",
"KeepJobFlowAliveWhenNoSteps": True,
"TerminationProtected": False,
"InstanceGroups": [
{
"InstanceCount": 1,
"EbsConfiguration": {
"EbsBlockDeviceConfigs": [
{
"VolumeSpecification": {
"SizeInGB": 32,
"VolumeType": "gp3"
},
"VolumesPerInstance": 2
}
]
},
"InstanceRole": "MASTER",
"InstanceType": "r6g.2xlarge",
"Name": "EMR Master"
},
{
"InstanceCount": 2,
"EbsConfiguration": {
"EbsBlockDeviceConfigs": [
{
"VolumeSpecification": {
"SizeInGB": 256,
"VolumeType": "gp3"
},
"VolumesPerInstance": 2
}
]
},
"InstanceRole": "CORE",
"InstanceType": "r6g.2xlarge",
"Name": "EMR Core"
},
{
"InstanceCount":2,
"EbsConfiguration": {
"EbsBlockDeviceConfigs": [
{
"VolumeSpecification": {
"SizeInGB": 256,
"VolumeType": "gp3"
},
"VolumesPerInstance": 2
}
]
},
"InstanceRole": "TASK",
"InstanceType": "r6g.2xlarge",
"Name": "EMR Task"
}
]
},
"StepConcurrencyLevel": 1,
"ReleaseLabel": "emr-5.36.0",
"LogUri": "s3n://<somebucket>/logs/",
"EbsRootVolumeSize": 32,
"ServiceRole": "emr-role",
"Name": "<cluster_name>"
}
emr = EmrJobRunner(region=REGION)
# This step create the cluster
cluster_id = emr.run_job_flow(emr_cluster_config)
step_name = 'test_step'
step_cmd = ['ls', '/']
step_ids = emr.add_job_flow_steps(
cluster_id, [emr.construct_step_dict_for_command(step_name, step_cmd)]
)
You can also look at the test cases on dagster repo, it does provide a very good examples for the same.

Resolve secret created with CDK in a Cfn L1 Construct

How can I use an L2 Secret created with Secrets Manager to resolve as an L1 Cfn Property value?
from aws_cdk import (
core,
aws_secretsmanager as secretsmanager,
aws_elasticache as elasticache
)
class MyStack(core.Stack):
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
redis_password = secretsmanager.Secret(
self, "RedisPassword",
description="Redis auth",
generate_secret_string=secretsmanager.SecretStringGenerator(
exclude_characters='/"#'
)
)
self.redis = elasticache.CfnReplicationGroup(self, 'RedisCluster',
auth_token=redis_password.secret_value,
# other properties
)
This gives the error
jsii.errors.JSIIError: Object of type #aws-cdk/aws-secretsmanager.Secret is not convertible to #aws-cdk/core.CfnElement
In Cloudformation to resolve a secret I'd use something like
AuthToken: !Sub '{{resolve:secretsmanager:${MySecret}::password}}'
But a L2 Secret doesn't output the Cfn Ref like L1 constructs do (that I know of)
What am I missing?

I was only missing the to_string() method
from aws_cdk import (
core,
aws_secretsmanager as secretsmanager,
aws_elasticache as elasticache
)
class MyStack(core.Stack):
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
redis_password = secretsmanager.Secret(
self, "RedisPassword",
description="Redis auth",
generate_secret_string=secretsmanager.SecretStringGenerator(
exclude_characters='/"#'
)
)
self.redis = elasticache.CfnReplicationGroup(self, 'RedisCluster',
auth_token=redis_password.secret_value.to_string(),
# other properties
)
This synthesizes to
{
"RedisPasswordED621C10": {
"Type": "AWS::SecretsManager::Secret",
"Properties": {
"Description": "Redis auth",
"GenerateSecretString": {
"ExcludeCharacters": "/\"#"
}
},
"Metadata": {
"aws:cdk:path": "my-cdk-stack/RedisPassword/Resource"
}
},
"RedisCluster": {
"Type": "AWS::ElastiCache::ReplicationGroup",
"Properties": {
"ReplicationGroupDescription": "RedisGroup",
"AtRestEncryptionEnabled": true,
"AuthToken": {
"Fn::Join": [
"",
[
"{{resolve:secretsmanager:",
{
"Ref": "RedisPasswordED621C10"
},
":SecretString:::}}"
]
]
},
"OtherProps": "..."
}
}
}

Flutter / Dart not returning a value from a locally hosted flask api

Using IOS, Visual Studio Code, Flask, VirtualEnv....
I've just started playing with Flask and Flutter. I've created a locally hosted flask server which returns users via the api and I'm trying to access the api with Flutter with my usb attached IOS device. When I try an api hosted on the internet, I get output on my phone, however when I point to the local API it does not look like it's hitting the server, I get no response in the flask server log stating a request was made. What am I doing wrong?
Via my browser I can navigate to 127.0.0.1:5000/users and get the following output. I expect this to appear on my phone:
[{"name": "Nick", "age": 42, "occupation": "Network Engineer"},
{"name": "Elvin", "age": 32, "occupation": "Business Analyst"},
{"name": "Nick", "age": 22, "occupation": "Test Analyst"}]
Here is my flask server code:
from flask import Flask
from flask_restful import Api, Resource, reqparse
from flask import jsonify
import json
app = Flask(__name__)
api = Api(app)
users = [
{
"name": "Nick",
"age": 42,
"occupation": "Network Engineer"
},
{
"name": "Elvin",
"age": 32,
"occupation": "Business Analyst"
},
{
"name": "Nick",
"age": 22,
"occupation": "Test Analyst"
}
]
class User(Resource):
def get(self, name):
for user in users:
if(name ==user["name"]):
return user, 200
return "User not found#, 404"
def post(self, name):
parser = reqparse.RequestParser()
parser.add_argument("age")
parser.add_argument("occupation")
args = parser.parse_args()
for user in users:
if(name == user["name"]):
return "User with name {} already exist".format(name), 400
user = {
"name": name,
"age": args["age"],
"occupation": args["occupation"]
}
users.append(user)
return user, 201
def put(self, name):
parser = reqparse.RequestParser()
parser.add_argument("age")
parser.add_argument("occupation")
args = parser.parse_args()
for user in users:
if(name == user["name"]):
user["age"] = args["age"]
user["occupation"] = args["occupation"]
return user, 200
user = {
"name": name,
"age": args["age"],
"occupation": args["occupation"]
}
users.append(user)
return user, 201
def delete(self, name):
global users
users = [user for user in users if user["name"] != name]
return "{} is deleted.".format(name), 200
#app.route('/users', methods=['GET'])
def getUsers():
return json.dumps(users)
api.add_resource(User, "/user/<string:name>")
app.run(debug=True)
Here is my pubspec.yaml
name: apitest2
description: A new Flutter project.
version: 1.0.0+1
environment:
sdk: ">=2.1.0 <3.0.0"
dependencies:
flutter:
sdk: flutter
http: ^0.12.0+2
cupertino_icons: ^0.1.2
dev_dependencies:
flutter_test:
sdk: flutter
flutter:
uses-material-design: true
Here is my API.dart - changing from 127... to my local IP address doesn't make a difference
import 'dart:async';
import 'package:http/http.dart' as http;
const baseUrl = "http://127.0.0.1:5000";
//const baseUrl = "http://192.168.86.24:5000";
//const baseUrl = "https://jsonplaceholder.typicode.com";
class API {
static Future getUsers() {
var url = baseUrl + "/users";
return http.get(url);
}
}
Here is my User.dart
class User {
String name;
int age;
String occupation;
User(String name, int age, String occupation) {
this.name = name;
this.age = age;
this.occupation = occupation;
}
User.fromJson(Map json)
: name = json['name'],
age = json['age'],
occupation = json['occupation'];
Map toJson() {
return {'name': name, 'age': age, 'occupation': occupation};
}
}
Here is my Main.dart
import 'dart:convert';
import 'package:apitest2/API.dart';
import 'package:apitest2/User.dart';
import 'package:flutter/material.dart';
void main() => runApp(MyApp());
class MyApp extends StatelessWidget {
#override
build(context) {
return MaterialApp(
debugShowCheckedModeBanner: true,
title: 'My Http App',
theme: ThemeData(
primarySwatch: Colors.blue,
),
home: MyListScreen(),
);
}
}
class MyListScreen extends StatefulWidget {
#override
createState() => _MyListScreenState();
}
class _MyListScreenState extends State {
var users = new List<User>();
_getUsers() {
API.getUsers().then((response) {
setState(() {
Iterable list = json.decode(response.body);
users = list.map((model) => User.fromJson(model)).toList();
});
});
}
initState() {
super.initState();
_getUsers();
}
dispose() {
super.dispose();
}
#override
build(context) {
return Scaffold(
appBar: AppBar(
title: Text("User List"),
),
body: ListView.builder(
itemCount: users.length,
itemBuilder: (context, index) {
return ListTile(title: Text(users[index].name));
},
));
}
}

Main Idea
If we want to make HTTP requests to our API from other computers or devices connected to our LAN, we should use the development computer IP address, 0.0.0.0 (for IPv4 configurations) or :: (for IPv6 configurations) as the desired IP address for our development server.
If we specify 0.0.0.0 as the desired IP address for IPv4 configurations, the development server will listen on every interface on port 5000.
Refer following solution, which also address a similar problem:
https://stackoverflow.com/a/62371660/10031056

Empty dictionary on AnnotationConsolidation lambda event for aws Sagemaker

I am starting to use aws sagemaker on the development of my machine learning model and I'm trying to build a lambda function to process the responses of a sagemaker labeling job. I already created my own lambda function but when I try to read the event contents I can see that the event dict is completely empty, so I'm not getting any data to read.
I have already given enough permissions to the role of the lambda function. Including:
- AmazonS3FullAccess.
- AmazonSagemakerFullAccess.
- AWSLambdaBasicExecutionRole
I've tried using this code for the Post-annotation Lambda (adapted for python 3.6):
https://docs.aws.amazon.com/sagemaker/latest/dg/sms-custom-templates-step2-demo1.html#sms-custom-templates-step2-demo1-post-annotation
As well as this one in this git repository:
https://github.com/aws-samples/aws-sagemaker-ground-truth-recipe/blob/master/aws_sagemaker_ground_truth_sample_lambda/annotation_consolidation_lambda.py
But none of them seemed to work.
For creating the labeling job I'm using boto3's functions for sagemaker:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sagemaker.html#SageMaker.Client.create_labeling_job
This is the code i have for creating the labeling job:
def create_labeling_job(client,bucket_name ,labeling_job_name, manifest_uri, output_path):
print("Creating labeling job with name: %s"%(labeling_job_name))
response = client.create_labeling_job(
LabelingJobName=labeling_job_name,
LabelAttributeName='annotations',
InputConfig={
'DataSource': {
'S3DataSource': {
'ManifestS3Uri': manifest_uri
}
},
'DataAttributes': {
'ContentClassifiers': [
'FreeOfAdultContent',
]
}
},
OutputConfig={
'S3OutputPath': output_path
},
RoleArn='arn:aws:myrolearn',
LabelCategoryConfigS3Uri='s3://'+bucket_name+'/config.json',
StoppingConditions={
'MaxPercentageOfInputDatasetLabeled': 100,
},
LabelingJobAlgorithmsConfig={
'LabelingJobAlgorithmSpecificationArn': 'arn:image-classification'
},
HumanTaskConfig={
'WorkteamArn': 'arn:my-private-workforce-arn',
'UiConfig': {
'UiTemplateS3Uri':'s3://'+bucket_name+'/templatefile'
},
'PreHumanTaskLambdaArn': 'arn:aws:lambda:us-east-1:432418664414:function:PRE-BoundingBox',
'TaskTitle': 'Title',
'TaskDescription': 'Description',
'NumberOfHumanWorkersPerDataObject': 1,
'TaskTimeLimitInSeconds': 600,
'AnnotationConsolidationConfig': {
'AnnotationConsolidationLambdaArn': 'arn:aws:my-custom-post-annotation-lambda'
}
}
)
return response
And this is the one i have for the lambda function:
print("Received event: " + json.dumps(event, indent=2))
print("event: %s"%(event))
print("context: %s"%(context))
print("event headers: %s"%(event["headers"]))
parsed_url = urlparse(event['payload']['s3Uri']);
print("parsed_url: ",parsed_url)
labeling_job_arn = event["labelingJobArn"]
label_attribute_name = event["labelAttributeName"]
label_categories = None
if "label_categories" in event:
label_categories = event["labelCategories"]
print(" Label Categories are : " + label_categories)
payload = event["payload"]
role_arn = event["roleArn"]
output_config = None # Output s3 location. You can choose to write your annotation to this location
if "outputConfig" in event:
output_config = event["outputConfig"]
# If you specified a KMS key in your labeling job, you can use the key to write
# consolidated_output to s3 location specified in outputConfig.
kms_key_id = None
if "kmsKeyId" in event:
kms_key_id = event["kmsKeyId"]
# Create s3 client object
s3_client = S3Client(role_arn, kms_key_id)
# Perform consolidation
return do_consolidation(labeling_job_arn, payload, label_attribute_name, s3_client)
I've tried debugging the event object with:
print("Received event: " + json.dumps(event, indent=2))
But it just prints an empty dictionary: Received event: {}
I expect the output to be something like:
#Content of an example event:
{
"version": "2018-10-16",
"labelingJobArn": <labelingJobArn>,
"labelCategories": [<string>], # If you created labeling job using aws console, labelCategories will be null
"labelAttributeName": <string>,
"roleArn" : "string",
"payload": {
"s3Uri": <string>
}
"outputConfig":"s3://<consolidated_output configured for labeling job>"
}
Lastly, when I try yo get the labeling job ARN with:
labeling_job_arn = event["labelingJobArn"]
I just get a KeyError (which makes sense because the dictionary is empty).

I am doing the same but in Labeled object section I am getting failed result and inside my output objects I am getting following error from Post Lambda function:
"annotation-case0-test3-metadata": {
"retry-count": 1,
"failure-reason": "ClientError: The JSON output from the AnnotationConsolidationLambda function could not be read. Check the output of the Lambda function and try your request again.",
"human-annotated": "true"
}
}

I found the problem, I needed to add the ARN of the role used by my Lamda function as a Trusted Entity on the Role used for the Sagemaker Labeling Job.
I just went to Roles > MySagemakerExecutionRole > Trust Relationships and added:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::xxxxxxxxx:role/My-Lambda-Role",
...
],
"Service": [
"lambda.amazonaws.com",
"sagemaker.amazonaws.com",
...
]
},
"Action": "sts:AssumeRole"
}
]
}
This made it work for me.

How to launch EC2 instance with Boto, specifying size of EBS?

I'm using boto/python to launch a new EC2 instance that boots from an EBS volume. At the time I launch the instance, I'd like to override the default size of the booting EBS volume.
I found no boto methods or parameters that might fit into my launch code:
ec2 = boto.connect_ec2( ACCESS_KEY, SECRET_KEY, region=region )
reservation = ec2.run_instances( image_id=AMI_ID,
key_name=EC2_KEY_HANDLE,
instance_type=INSTANCE_TYPE,
security_groups = [ SECGROUP_HANDLE, ] )
This web page shows how to increase the size of a running EC2-instance's EBS volume using command-line tools, but I'd like to use boto at the time the EC2 instance is specified:

You have to create a block device mapping first:
dev_sda1 = boto.ec2.blockdevicemapping.EBSBlockDeviceType()
dev_sda1.size = 50 # size in Gigabytes
bdm = boto.ec2.blockdevicemapping.BlockDeviceMapping()
bdm['/dev/sda1'] = dev_sda1
After this you can give the block device map in your run_instances call:
reservation = ec2.run_instances( image_id=AMI_ID,
key_name=EC2_KEY_HANDLE,
instance_type=INSTANCE_TYPE,
security_groups = [ SECGROUP_HANDLE, ],
block_device_mappings = [bdm])
Unfortunately this is not really well documented, but the example can be found in the source code.

You can also use CloudFormation, which is used to document and automate your environment.
You can check the template for the ESB definition at: https://s3.amazonaws.com/cloudformation-templates-us-east-1/EC2WithEBSSample.template
"Resources" : {
"Ec2Instance" : {
"Type" : "AWS::EC2::Instance",
"Properties" : {
"AvailabilityZone" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "TestAz" ]},
"SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ],
"KeyName" : { "Ref" : "KeyName" },
"ImageId" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "AMI" ]},
"Volumes" : [
{ "VolumeId" : { "Ref" : "NewVolume" },
"Device" : "/dev/sdk"
}
]
}
},
...
"NewVolume" : {
"Type" : "AWS::EC2::Volume",
"Properties" : {
"Size" : "100",
"AvailabilityZone" : { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "TestAz" ]}
}
}
You can then use Boto CloudFormation API to deploy your environment.

Here is a version of the code using the boto3 "everything is a resource" approach. Note also how to avoid hard-coded disk names:
import boto3
ec2 = boto3.resource('ec2')
def ec2_one_by_key_and_value(collection: str, key: str, value: str):
handler = getattr(ec2, collection)
return list(handler.filter(Filters=[{'Name': key, 'Values': [value]}]))[0]
image = ec2_one_by_key_and_value('images', 'name', 'ubuntu/images/hvm-ssd/...')
root_disk = None
for block_device in image.block_device_mappings:
if block_device['DeviceName'] == image.root_device_name:
root_disk = block_device
assert root_disk['Ebs']
root_disk['Ebs']['VolumeSize'] = 16 # New disk
break
assert root_disk
instances = ec2.create_instances(MinCount=1, ..., ImageId=image.id,
BlockDeviceMappings=image.block_device_mappings)