I am trying to create a Terraform module in CDKTF following the documentation at https://developer.hashicorp.com/terraform/cdktf/concepts/modules
I created the module as shown in the code:
#!/usr/bin/env python
from constructs import Construct
from cdktf import App, TerraformStack, TerraformOutput
from imports.aws.provider import AwsProvider
from imports.aws.s3_bucket import S3Bucket
class MyStack(TerraformStack):
def __init__(self, scope: Construct, id: str):
super().__init__(scope, id)
AwsProvider(self, "AWS", region="eu-west-1", profile="<MY-PROFILE>")
bucket_name = '<MY-BUCKET-NAME>'
# define resources here
s3_bucket = S3Bucket(
self, 'testBucket',
bucket=bucket_name
)
TerraformOutput(self, "my_output", value=s3_bucket.arn)
app = App()
MyStack(app, "s3_module")
app.synth()
Then, I created another folder for the main stack and added the module in the 'cdktf.json' file
{
"language": "python",
"app": "pipenv run python main.py",
"projectId": "e2b44a02-65b2-42de-ab52-3863d211c94c",
"sendCrashReports": "true",
"terraformProviders": [
"hashicorp/aws#~>4.0"
],
"terraformModules": [{
"name": "s3_module",
"source": "../s3_module"
}],
"codeMakerOutput": "imports",
"context": {
"excludeStackIdFromLogicalIds": "true",
"allowSepCharsInLogicalIds": "true"
}
}
I also called the module in main.py:
#!/usr/bin/env python
from constructs import Construct
from cdktf import App, TerraformStack
from imports.s3_module import S3Module
from imports.aws.provider import AwsProvider
class MyStack(TerraformStack):
def __init__(self, scope: Construct, id: str):
super().__init__(scope, id)
AwsProvider(self, "AWS", region="eu-west-1", profile="<MY-PROFILE>")
# define resources here
my_module = S3Module(self, 's3_module')
app = App()
MyStack(app, "service_v2")
app.synth()
However, when I run "cdktf deploy", Terraform does not add any resources:
Can you help me understand where the error is? Thank you
My understanding is that the terraformModules field in the cdktf.json file is for communicating which Terraform modules you want to use, i.e. directories that contain .tf files.
This is what allows the cdktf get command to know which Python bindings you need so that it can generate them into your imports/ directory. Enabling the import statements to work.
If instead you would like to factor out parts of your infrastructure configuration into separate Python modules, you could adapt a strategy where you subclass from Construct, as follows:
Example module: s3_bucket.py:
import constructs
import cdktf
from imports.aws.provider import AwsProvider
from imports.aws.s3_bucket import (
S3Bucket
)
class S3BucketInfra(constructs.Construct):
def __init__(self, scope: constructs.Construct,
construct_id: str,
bucket_name: str,
provider: AwsProvider):
super().__init__(scope=scope, id=construct_id)
self.bucket = S3Bucket( self,
'bucket',
bucket=bucket_name,
force_destroy=False,
provider=provider )
Example main module: main.py:
import constructs
import cdktf
from s3_bucket import (
S3BucketInfra
)
class SomeExampleStack(cdktf.TerraformStack):
def __init__(self, scope: constructs.Construct, construct_id: str):
super().__init__(scope=scope, id=construct_id)
self.some_bucket = S3BucketInfra( scope=self,
construct_id='some-bucket',
bucket_name='testBucket',
provider=AwsProvider(self, "AWS", region="eu-west-1", profile="<MY-PROFILE>") )
app = cdktf.App()
SomeExampleStack(scope=app, construct_id="service_v2")
app.synth()
I am pretty new to CDKTF myself and the documentation is a little thin, but this is what worked for me :)
Related
I came across this python class ResourcesMoveInfo for moving resources(Azure Images) from one subscription to another with Azure python SDK.
But it's failing when I use it like below:
Pattern 1
reference from https://buildmedia.readthedocs.org/media/pdf/azure-sdk-for-python/v1.0.3/azure-sdk-for-python.pdf
Usage:
metadata = azure.mgmt.resource.resourcemanagement.ResourcesMoveInfo(resources=rid,target_resource_group='/subscriptions/{0}/resourceGroups/{1}'.format(self.prod_subscription_id,self.resource_group))
Error:
AttributeError: module 'azure.mgmt.resource' has no attribute 'resourcemanagement'
Pattern 2
reference from - https://learn.microsoft.com/en-us/python/api/azure-mgmt-resource/azure.mgmt.resource.resources.v2019_07_01.models.resourcesmoveinfo?view=azure-python
Usage:
metadata = azure.mgmt.resource.resources.v2020_06_01.models.ResourcesMoveInfo(resources=rid,target_resource_group='/subscriptions/{0}/resourceGroups/{1}'.format(self.prod_subscription_id,self.resource_group))
Error:
AttributeError: module 'azure.mgmt.resource.resources' has no attribute 'v2020_06_01'
Any help on this requirement/issue would be helpful. Thanks!
Adding code snippet here:
import sys
import os
import time
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.resource import ResourceManagementClient
import azure.mgmt.resource
#from azure.mgmt.resource.resources.v2020_06_01.models import ResourcesMoveInfo
from azure.identity import ClientSecretCredential
from cred_wrapper import CredentialWrapper
class Move():
def __init__(self):
self.nonprod_subscription_id = "abc"
self.prod_subscription_id = "def"
self.credential = ClientSecretCredential(
client_id= os.environ["ARM_CLIENT_ID"],
client_secret= os.environ["ARM_CLIENT_SECRET"],
tenant_id= os.environ["ARM_TENANT_ID"]
)
#resource client for nonprod
self.sp = CredentialWrapper(self.credential)
self.resource_client = ResourceManagementClient(self.sp,self.nonprod_subscription_id)
self.resource_group = "imgs-rg"
def getresourceids(self):
resource_ids = list(resource.id for resource in self.resource_client.resources.list_by_resource_group("{0}".format(self.resource_group)) if resource.id.find("latest")>=0)
return resource_ids
def getresourcenames(self):
resource_names = list(resource.name for resource in self.resource_client.resources.list_by_resource_group("{0}".format(self.resource_group)) if resource.id.find("latest")>=0)
return resource_names
def deleteoldimages(self,name):
#resource client id for prod
rc = ResourceManagementClient(self.sp,self.prod_subscription_id)
for resource in rc.resources.list_by_resource_group("{0}".format(self.resource_group)):
if resource.name == name:
#2019-12-01 is the latest api_version supported for deleting the resource type "image"
rc.resources.begin_delete_by_id(resource.id,"2020-06-01")
print("deleted {0}".format(resource.name))
def moveimages(self):
rnames = self.getresourcenames()
for rname in rnames:
print(rname)
#self.deleteoldimages(rname)
time.sleep(10)
rids = self.getresourceids()
rid = list(rids[0:])
print(rid)
metadata = azure.mgmt.resource.resources.v2020_06_01.models.ResourcesMoveInfo(resources=rid,target_resource_group='/subscriptions/{0}/resourceGroups/{1}'.format(self.prod_subscription_id,self.resource_group))
#moving resources in the rid from nonprod subscription to prod subscription under the resource group avrc-imgs-rg
if rid != []:
print("moving {0}".format(rid))
print(self.resource_client.resources.move_resources(source_resource_group_name="{0}".format(self.resource_group),parameters=metadata))
#self.resource_client.resources.move_resources(source_resource_group_name="{0}".format(self.resource_group),resources=rid,target_resource_group='/subscriptions/{0}/resourceGroups/{1}'.format(self.prod_subscription_id,self.resource_group))
#self.resource_client.resources.begin_move_resources(source_resource_group_name="{0}".format(self.resource_group),parameters=metadata)
if __name__ == "__main__":
Move().moveimages()
From your inputs we can see that the code looks fine. From your error messages, the problem is with importing the modules.
Basically when we import a module few submodules will get installed along with and few will not. This will depend on the version of the package, to understand which modules are involved in a specific version we need to check for version-releases in official documentation.
In your case, looks like some resource modules were missing, if you could see the entire error-trace, there will be a path with sitepackages in our local. Try to find that package and its subfolder(Modules) and compare them with Azure SDK for Python under Resource module, you can find this here.
In such situation we need to explicitly add those sub modules under our package. In your case you can simple download the packaged code from Git link which I gave and can merge in your local folder.
I am trying to deploy python lambda function with serverless framework. This function need to run for 15 min (AWS Lambda Timeout). I want to simulate 100 IoT devices using AWS Lambda.
I have following code device_status.py
import os
import time
from uptime import uptime
import requests
from random import randrange
from configparser import ConfigParser, ExtendedInterpolation
class DeviceStatus:
def __init__(self):
self.config_file = 'config.ini'
self.config_dict = None
self.read_device_config()
self.dr_ins = DeviceRegistration(self.config_dict)
....
if __name__ == '__main__':
init_ds = DeviceStatus()
status_interval = init_ds.config_dict['status']['interval']
while True:
init_ds.send_device_status()
time.sleep(int(status_interval))
and serverless.yml
service: lambda-device
plugins:
- serverless-python-requirements
provider:
name: aws
runtime: python3.6
region: us-east-1
functions:
lambda-device:
handler: main.device_status
when I try to invoke it I get "errorMessage": "Unable to import module 'main'"
How to refer to main function in serverless.yml ?
The error message that you are receiving is saying that there is no main.py file in your serverless structure.
Referring to your serverless.yml:
functions:
lambda-device:
handler: main.device_status
The explanation from the above section is that you have a serverless-function that is named lambda-device which is having a structure with a filename main.py that in its definition would require to have a method:
def device_status(event, context):
# TODO
pass
So make sure you have main.py file with a method device_status(event, context)
My question is related to this one. I have a config_file consisting of dictionaries as shown below:
config_1 = {
'folder': 'raw1',
'start_date': '2019-07-01'
}
config_2 = {
'folder': 'raw2',
'start_date': '2019-08-01'
}
config_3 = {
'folder': 'raw3',
'start_date': '2019-09-01'
}
I then have a separate python file that imports each config and does some stuff:
from config_file import config_1 as cfg1
Do some stuff using 'folder' and 'start_date'
from config_file import config_2 as cfg2
Do some stuff using 'folder' and 'start_date'
from config_file import config_2 as cfg3
Do some stuff using 'folder' and 'start_date'
I would like to put this in a loop rather than have it listed 3 times in the python file. How can I do that?
If I understand your question correctly, just use importlib. In a nutshell, what in python you write like:
from package import module as alias_mod
in importlib it becomes:
alias_mod = importlib.import_module('module', 'package')
or, equivalentelly:
alias_mod = importlib.import_module('module.package')
for example:
from numpy import random as rm
in importlib:
rm = importlib.import_module('random', 'numpy')
Another interesting thing is this code proposed in this post, which allows you to import not only modules and packages but also directly functions and more:
def import_from(module, name):
module = __import__(module, fromlist=[name])
return getattr(module, name)
For your specific case, this code should work:
import importlib
n_conf = 3
for in range(1, n_conf)
conf = importlib.import_module('config_file.config_'+str(i))
# todo somethings with conf
However, if I can give you some advice I think the best thing for you is to build a json configuration file and read the file instead of importing modules. It's much more comfortable. For example in your case, you can create a config.json file like this:
{
"config_1": {
"folder": "raw1",
'start_date': '2019-07-01'
},
"config_2": {
'folder': 'raw2',
'start_date': '2019-08-01'
},
"config_3": {
'folder': 'raw3',
'start_date': '2019-09-01'
}
}
Read the json file as follows:
import json
with open('config.json') as json_data_file:
conf = json.load(json_data_file)
Now you have in memory a simple python dictionary with the configuration settings that interest you:
conf['config_1']
# output: {'folder': 'raw1', 'start_date': '2019-07-01'}
You can use inspect module to get all possible imports from config like following.
import config
import inspect
configs = [member[1] for member in inspect.getmembers(config) if 'config_' in member[0]]
configs
And then you can iterate over all configs, is this the behavior you wanted?
You can read more about inspect here
.
Based on #MikeMajara's comment, the following solution worked for me:
package = 'config_file'
configs = ['config_1', 'config_2', 'config_3']
for i in configs:
cfg = getattr(__import__(package, fromlist=[configs]), i)
Do some stuff using 'folder' and 'start_date'
I am new to AWS lambda function and i am trying to add my existing code to AWS lambda. My existing code looks like :
import boto3
import slack
import slack.chat
import time
import itertools
from slacker import Slacker
ACCESS_KEY = ""
SECRET_KEY = ""
slack.api_token = ""
slack_channel = "#my_test_channel"
def gather_info_ansible():
.
.
def call_snapshot_creater(data):
.
.
def call_snapshot_destroyer(data):
.
.
if __name__ == '__main__':
print "Calling Ansible Box Gather detail Method first!"
ansible_box_info = gather_info_ansible()
print "Now Calling the Destroyer of SNAPSHOT!! BEHOLD THIS IS HELL!!"
call_snapshot_destroyer(ansible_box_info)
#mapping = {i[0]: [i[1], i[2]] for i in data}
print "Now Calling the Snapshot Creater!"
call_snapshot_creater(ansible_box_info)
Now i try to create a lambda function from scratch on AWS Console as follows (a hello world)
from __future__ import print_function
import json
print('Loading function')
def lambda_handler(event, context):
#print("Received event: " + json.dumps(event, indent=2))
print("value1 = " + event['key1'])
print("value2 = " + event['key2'])
print("value3 = " + event['key3'])
print("test")
return event['key1'] # Echo back the first key value
#raise Exception('Something went wrong')
and the sample test event on AWS console is :
{
"key3": "value3",
"key2": "value2",
"key1": "value1"
}
I am really not sure how to put my code in AWS lambda coz if i even add the modules in lambda console and run it it throws me error :
Unable to import module 'lambda_function': No module named slack
How to solve this and import my code in lambda?
You have to make a zipped package consisting of your python script containing the lambda function and all the modules that you are importing in the python script. Upload the zipped package on aws.
Whatever module you want to import, you have to include that module in the zip package. Only then the import statements will work.
For example your zip package should consist of
test_package.zip
|-test.py (script containing the lambda_handler function)
|-boto3(module folder)
|-slack(module folder)
|-slacker(module folder)
You receive an error because AWS lambda does not have any information about a module called slack.
A module is a set of .py files that are stored somewhere on a computer.
In case of lambda, you should import all your libraries by creating a deployment package.
Here is an another question that describes similar case and provides several solutions:
AWS Lambda questions
We have an ETL data API repo. We do all etl processing inside of it, then spit the data out in API's. These API's are run one at a time from a single command passing the resource class through to the server to build an API. the resource class is in a web directory in an __init__.py.
This is a wonderful convention and quite simple to use, but the problem I am having is coming from trying to get one of the 3 API's available spun up for testing. Our directory stucture is like this (calling the project 'tomato')
tomato
- category_api
- web
- etl
- test
- here is where we are writing some tests (test_category_api.py)
- data
- article_api
- web
- etl
- test
- data
- recommendation_api
- web
- etl
- test
- data
- common
- common shit
Inside this test, I have the following test class. On the seventh line up from the bottom,
you will see a comment on where it breaks. It is the import_module method.
import unittest
import sys
import os
import sys
import json
from importlib import import_module
from flask import Flask
from flask_restful import Api, abort, wraps
from flask_restful.utils import cors
from flask.ext.testing import TestCase
#dir_above_top_level = os.path.join(os.path.abspath(__file__), '../../.. /')
#sys.path.append(os.path.abspath(dir_above_top_level))
_CATEGORY_ENDPOINT = '/v1/category/'
_PACKAGE_NAME = os.environ['RECO_API_PACKAGE_NAME']
_CORS = cors.crossdomain(origin='*',
headers=['Origin', 'X-Requested-With',
'Content-Type', 'Accept'],
methods=['GET', 'HEAD', 'OPTIONS'],
max_age=3600)
class CategoryTests(TestCase):
def __init__(self):
self.app = Flask(__name__)
self._configure_app()
for resource in self.resource_classes:
self.api.add_resource(self.resource,
self.resource.URI_TEMPLATE)
def test_status_code(self):
self.response = self.client.post(_CATEGORY_ENDPOINT,
data=json.dumps(
{'title': 'Enjoy this delicious food'}),
headers=json.dumps(
{'content-type':'application/json'}))
self.assertEqual(self.response.status_code, 200)
def test_version(self):
self.response = self.client.post(_CATEGORY_ENDPOINT,
data=json.dumps(
{"title": "eat some delicious stuff"}),
headers=json.dumps(
{'content-type':'application/json'}))
self.assertEqual(json.dumps(self.response['version']), '1')
def _configure_app(self):
self.app = Flask(__name__)
self.app.config['TESTING'] = True
self.app.debug = True
self.decorators = [_CORS]
self.app.Threaded = True
self.web_package = 'tomato.category.web'
self.package = import_module('.__init__', self.web_package) # WE BREAK HERE
self.resources = package.RESOURCE_NAMES
self.resource_classes = [ getattr(package, resource) for resource in resources ]
self.api = Api(self.app, catch_all_404s=True, decorators=self.decorators)
if __name__ == '__main__':
unittest.main()
we are given an exception when running these tests:
ImportError: No module named tomato.category.web.__init__
yet cd into the main top dir, and ls tomato/category/web gets us __init__.py and its right there with the resource class.
How do I import this class so that I can instantiate the API to run the tests in this class?
Or if I'm completely on the wrong track what should I be doing instead?
You don't need to import __init__, just like you probably wouldn't do from tomato.category.web import __init__. You should be able to import the web package directly.
self.web_package = 'tomato.category.web'
self.package = import_module(self.web_package)
The problem here lies in the directory structure. In the current path, I am not at the top level. It is a module. So What was needed was to uncomment the line two lines at the top, and change the structure to append the path like this.
dir_above_top_level = os.path.join(os.path.abspath(__file__), '../../../..')
sys.path.append(os.path.abspath(dir_above_top_level))
and now, I can import it using
self.web_package = 'tomato.category.web'
self.package = import_module('.__init__', self.web_package)
and now it will import fine and I can grab the resource class to set up the testing API