Develop Reference

Does my lambda function go inside my main python script?

I don't know how to write a Lambda. Here is my main_script.py that executes 2 stored procedures. It inserts records every day then finds the difference between yesterday's and today's records and writes them to a table.
import logging
import pymysql as pm
import os
import json
class className:
env=None
config=None
def __init__(self, env_filename):
self.env=env_filename
self.config=self.get_config()
def get_config(self):
with open(self.env) as file_in:
return json.load(file_in)
def DB_connection(self):
config=className.get_config(self)
username=config["exceptions"]["database-secrets"]["aws_secret_username"]
password=config["exceptions"]["database-secrets"]["aws_secret_password"]
host=config["exceptions"]["database-secrets"]["aws_secret_host"]
port=config["exceptions"]["database-secrets"]["aws_secret_port"]
database=config["exceptions"]["database-secrets"]["aws_secret_db"]
return pm.connect(
user=username,
password=password,
host=host,
port=port,
database=database
)
def run_all(self):
def test_function(self):
test_function_INSERT_QUERY = "CALL sp_test_insert();"
test_function_EXCEPTIONS_QUERY = "CALL sp_test_exceptions();"
test = self.config["exceptions"]["functions"]["test_function"]
if test:
with self.DB_connection() as cnxn:
with cnxn.cursor() as cur:
try:
cur.execute(test_function_INSERT_QUERY)
print("test_function_INSERT_QUERY insertion query ran successfully, {} records updated.".format(cur.rowcount))
cur.execute(test_function_EXCEPTIONS_QUERY)
print("test_function_EXCEPTIONS_QUERY exceptions query ran successfully, {} exceptions updated.".format(cur.rowcount))
except pm.Error as e:
print(f"Error: {e}")
except Exception as e:
logging.exception(e)
else:
cnxn.commit()
test_function(self)
def main():
cwd=os.getcwd()
vfc=(cwd+"\_config"+".json")
ve=className(vfc)
ve.run_all()
if __name__ == "__main__":
main()
Would I write my lambda_handler function inside my script above or have it as a separate script?
def lambda_handler(event, context):
#some code

I would treat lambda_handler(event, context) as the equivalent of main() with the exception that you do not need if __name__ ... clause because you never run a lambda function from the console.
You would also need to use boto3 library to abstract away AWS services and their functions. Have a look at the tutorial to get started.
As the first order of business, I would put the DB credentials out of the file system and into a secure datastore. You can of course configure Lambda environment variables, but Systems Manager Parameter Store is more secure and super-easy to call from the code, e.g.:
import boto3
ssm = boto3.client('ssm', region_name='us-east-1')
def lambda_handler(event, context):
password = ssm.get_parameters(Names=['/pathto/password'], WithDecryption=True)['Parameters'][0]['Value']
return {"password": password}
There is a more advanced option, the Secrets Manager, which for a little money will even rotate passwords for you (because it is fully integrated with Relational Database Service).

Why open the same one database with sqlalchemy, but get different, how can I update it?

I write some tests with pytest, I want to test create user and email with post method.
With some debug, I know the issue is I open two databases in memory, but they are same database SessionLocal().
So how can I fix this, I try db.flush(), but it doesn't work.
this is the post method code
#router.post("/", response_model=schemas.User)
def create_user(
*,
db: Session = Depends(deps.get_db), #the get_db is SessionLocal()
user_in: schemas.UserCreate,
current_user: models.User = Depends(deps.get_current_active_superuser),
) -> Any:
"""
Create new user.
"""
user = crud.user.get_by_email(db, email=user_in.email)
if user:
raise HTTPException(
status_code=400,
detail="The user with this username already exists in the system.",
)
user = crud.user.create(db, obj_in=user_in)
print("====post====")
print(db.query(models.User).count())
print(db)
if settings.EMAILS_ENABLED and user_in.email:
send_new_account_email(
email_to=user_in.email, username=user_in.email, password=user_in.password
)
return user
and the test code is:
def test_create_user_new_email(
client: TestClient, superuser_token_headers: dict, db: Session # db is SessionLocal()
) -> None:
username = random_email()
password = random_lower_string()
data = {"email": username, "password": password}
r = client.post(
f"{settings.API_V1_STR}/users/", headers=superuser_token_headers, json=data,
)
assert 200 <= r.status_code < 300
created_user = r.json()
print("====test====")
print(db.query(User).count())
print(db)
user = crud.user.get_by_email(db, email=username)
assert user
assert user.email == created_user["email"]
and the test result is
> assert user
E assert None
====post====
320
<sqlalchemy.orm.session.Session object at 0x7f0a9f660910>
====test====
319
<sqlalchemy.orm.session.Session object at 0x7f0aa09c4d60>

Your code does not provide enough information to help you, the key issues are probably in what is hidden and explained by your comments.
And it seems like you are confusing sqlalchemy session and databases. If you are not familiar with these concepts, I highly recommend you to have a look at SQLAlchemy documentation.
But, looking at your code structure, it seems like you are using FastAPI.
Then, if you want to test SQLAlchemy with pytest, I recommend you to use pytest fixture with SQL transactions.
Here is my suggestion on how to implement such a test. I'll suppose that you want to run the test on your actual database and not create a new database especially for the tests. This implementation is heavily based on this github gist (the author made a "feel free to use statement", so I suppose he is ok with me copying his code here):
# test.py
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from fastapi.testclient import TestClient
from myapp.models import BaseModel
from myapp.main import app # import your fastapi app
from myapp.database import get_db # import the dependency
client = TestClient(app)
# scope="session" mean that the engine will last for the whole test session
#pytest.fixture(scope="session")
def engine():
return create_engine("postgresql://localhost/test_database")
# at the end of the test session drops the created metadata using fixture with yield
#pytest.fixture(scope="session")
def tables(engine):
BaseModel.metadata.create_all(engine)
yield
BaseModel.metadata.drop_all(engine)
# here scope="function" (by default) so each time a test finished, the database is cleaned
#pytest.fixture
def dbsession(engine, tables):
"""Returns an sqlalchemy session, and after the test tears down everything properly."""
connection = engine.connect()
# begin the nested transaction
transaction = connection.begin()
# use the connection with the already started transaction
session = Session(bind=connection)
yield session
session.close()
# roll back the broader transaction
transaction.rollback()
# put back the connection to the connection pool
connection.close()
## end of the gist.github code
#pytest.fixture
def db_fastapi(dbsession):
def override_get_db():
db = dbsession
try:
yield db
finally:
db.close()
client.app.dependency_overrides[get_db] = override_get_db
yield db
# Now you can run your test
def test_create_user_new_email(db_fastapi):
username = random_email()
# ...

SNS mocking with moto is not working correctly

In my unit test:
def test_my_function_that_publishes_to_sns():
conn = boto3.client("sns", region_name="us-east-1")
mock_topic = conn.create_topic(Name="mock-topic")
topic_arn = mock_topic.get("TopicArn")
os.environ["SNS_TOPIC"] = topic_arn
# call my_function
my_module.my_method()
The the function being tested
# inside my_module, my_function...
sns_client.publish(
TopicArn=os.environ["SNS_TOPIC"], Message="my message",
)
I get the error: botocore.errorfactory.NotFoundException: An error occurred (NotFound) when calling the Publish operation: Endpoint with arn arn:aws:sns:us-east-1:123456789012:mock-topic not found
Doesn't make sense, that's the topic moto is suppose to have created and mocked. Why is it saying it doesn't exist? If I call conn.publish(TopicArn=topic_arn, Message="sdfsdsdf") inside of the unit test itself it seems to mock it, but it doesn't mock it for my_module.my_method() which the unit test executes. Maybe it's destroying the mocked topic too soon?
EDIT I tried this every which way and I get the exact same error:
# Using context manager
def test_my_function_that_publishes_to_sns():
with mock_sns():
conn = boto3.client("sns", region_name="us-east-1")
mock_topic = conn.create_topic(Name="mocktopic")
topic_arn = mock_topic.get("TopicArn")
os.environ["SNS_TOPIC"] = topic_arn
# call my_function
my_module.my_method()
# Using decorator
#mock_sns
def test_my_function_that_publishes_to_sns():
conn = boto3.client("sns", region_name="us-east-1")
mock_topic = conn.create_topic(Name="mocktopic")
topic_arn = mock_topic.get("TopicArn")
os.environ["SNS_TOPIC"] = topic_arn
# call my_function
my_module.my_method()
# Using decorator and context manager
#mock_sns
def test_my_function_that_publishes_to_sns():
with mock_sns():
conn = boto3.client("sns", region_name="us-east-1")
mock_topic = conn.create_topic(Name="mocktopic")
topic_arn = mock_topic.get("TopicArn")
os.environ["SNS_TOPIC"] = topic_arn
# call my_function
my_module.my_method()
Opened GitHub issue as well: https://github.com/spulec/moto/issues/3027

issue was my_module.my_method() wasn't setting a region just doing client = boto3.client("sns")
It could not find it because it was defaulting to a diff region than us-east-1 which was hard coded into the unit test

maybe it will help you
keep all modules in a single class and put a decorator #mock_sns on the class too for mocking the sns, also put decorator #mock_sns on the function where you are initializing you connection to sns.
Example:
#mock_sns
class TestSnsMock(unittest.TestCase):
#classmethod
#mock_sns
def setUpClass(cls):
cls.conn = boto3.client("sns", region_name="us-east-1")
cls.conn.create_topic(Name="some-topic")
cls.response = cls.conn.list_topics()
cls.topic_arn = cls.response["Topics"][0]["TopicArn"]
def test_publish_sns(self):
message = "here is same message"
self.sns_client.publish(TopicArn=self.topic_arn, Message=message)
if __name__ == "__main__":
unittest.main()

Sample code below. I hope it helps somebody. The suggested fix about setting the Region was not my issue. If you are still stuck, this video is great.
Approach:
Create a mocked Boto3 Resource ( not a Boto3 Client ).
Set mock SNS Topic ARN in this new resource.
Overwrite the SNS Topic ARN environment var for the test.
Get a Boto3 Client that calls Publish to the mocked SNS Topic ARN.
I hit the below error because I set the Topic ARN to mock_topic and not arn:aws:sns:eu-west-1:123456789012:mock_topic:
botocore.errorfactory.NotFoundException: An error occurred (NotFound) when calling the Publish operation: Endpoint does not exist
"""
import main
import boto3
import pytest
import botocore
from moto import mock_sns
# http://docs.getmoto.org/en/latest/docs/getting_started.html
#####################################################################
# test_main.py
#####################################################################
#pytest.fixture()
def mock_message():
return {
"foo": "1st wonderful message.",
"bar": "2nd wonderful message.",
"baz": "3rd wonderful message.",
}
#pytest.fixture()
def mock_sns_client():
return sns_publish.get_sns_client()
def test_get_mocked_sns_client(mock_sns_client):
assert isinstance(mock_sns_client, botocore.client.BaseClient)
mock_topic_name = "mock_topic"
#mock_sns
def test_mock_send_sns(mock_message, monkeypatch, mock_sns_client):
"""
1. Create a mocked Boto3 Resource ( not a Boto3 Client ).
2. Set mock SNS Topic ARN in this new resource.
3. Overwrite the SNS Topic ARN environment var for the test.
"""
sns_resource = boto3.resource(
"sns",
region_name=os.environ.get("AWS_REGION")
)
topic = sns_resource.create_topic(
Name=mock_topic_name
)
assert mock_topic_name in topic.arn
monkeypatch.setenv('SNS_TOPIC_ARN', topic.arn)
assert os.environ.get("SNS_TOPIC_ARN") == topic.arn
response = sns_publish.send_sns(mock_sns_client, mock_message)
assert isinstance(response, dict)
message_id = response.get("MessageId", None)
assert isinstance(message_id, str)
#####################################################################
# main.py
# split the get Client and Publish for simpler testing
#####################################################################
import boto3
import json
import botocore
import os
from conf.base_logger import logger
# split the get Client and Publish for simpler testing
def get_sns_client():
return boto3.client("sns", region_name=os.environ.get("AWS_REGION"))
def send_sns(sns_client, message: dict) -> dict:
if not isinstance(message, dict):
logger.info("message to send Slack is not in expected format")
return None
if not isinstance(sns_client, botocore.client.BaseClient):
logger.info("something wrong with the SNS client")
return None
return sns_client.publish(
TargetArn=os.environ.get("SNS_TOPIC_ARN"),
Message=json.dumps({'default': json.dumps(message, indent=4, sort_keys=True)}),
Subject='Foo\'s stats',
MessageStructure='json'
)

Trouble using bottle-jwt decorator (not works)

Having some trouble using this plugin https://github.com/agile4you/bottle-jwt/
It seems to not work as I expected, down below my code:
import bottle
from Py.engine import *
from bottle_jwt import (JWTProviderPlugin, jwt_auth_required)
class AuthBackend(object):
user = {'id': 1237832, 'username': 'pav', 'password': '123', 'data': {'sex': 'male', 'active': True}}
def authenticate_user(self, username, password):
"""Authenticate User by username and password.
Returns:
A dict representing User Record or None.
"""
if username == self.user['username'] and password == self.user['password']:
return self.user
return None
def get_user(self, user_id):
"""Retrieve User By ID.
Returns:
A dict representing User Record or None.
"""
if user_id == self.user['id']:
return {k: self.user[k] for k in self.user if k != 'password'}
return None
app = bottle.Bottle()
server_secret = 'secret'
provider_plugin = JWTProviderPlugin(
keyword='jwt',
auth_endpoint='/login',
backend=AuthBackend(),
fields=('username', 'password'),
secret=server_secret,
ttl=30
)
app.install(provider_plugin)
#app.route('/')
#jwt_auth_required
def index():
return open('Html/index.html', 'r').read()
#app.post('/login')
def login():
return open('Html/login.html', 'r').read()
#app.get('/login')
def login():
return open('Html/login.html', 'r').read()
def run_server():
bottle.run(app=app, host='localhost', port=8080, debug=True, reloader=True)
# Main
if __name__ == '__main__':
run_server()
Once running, if I open browser On 127.0.0.1/8080 i get back a blank page with the string "{"AuthError": ["Cannot access this resource!"]}"
Which is Fine, it means that I'm not allowed to open index.html file (Cool: #jwt_auth_required worked)
Digging in source file I found a function named validate_token() with:
if not token:
logger.debug("Forbidden access")
raise JWTForbiddenError('Cannot access this resource!')
Here is the exception
except JWTForbiddenError as error:
bottle.response.content_type = b('application/json')
bottle.response._status_line = b('403 Forbidden')
return {"AuthError": error.args}
So, is there any way to redirect me on my login.html page if token does not match or is absent?
Plugin includes some way to do that or is just an API pckg?

That's not how JWT concept is supposed to be used. JWT are for RESTFul.
You need to make the server as REST API and on the client use JS
libraries such as AngularJs / Vue.js etc.,
Coming to the question about the plugin:
provider_plugin = JWTProviderPlugin(
keyword='jwt',
auth_endpoint='/login',
backend=AuthBackend(),
fields=('username', 'password'),
secret=server_secret,
ttl=30
)
auth_endpoint='/login' is to give a custom endpoint for authorization where the Bottle_JWT methods are looking for credentials to validate and generate JWT for.
I created a mock just to construct a response and this is how it should be used.
Once you pass the correct credential, the plugin responds with the JWT and expire which you have to intercept in authorized calls and add as request headers
Hope this helps.

How to create a SECRET_HASH for AWS Cognito using boto3?

I want to create/calculate a SECRET_HASH for AWS Cognito using boto3 and python. This will be incorporated in to my fork of warrant.
I configured my cognito app client to use an app client secret. However, this broke the following code.
def renew_access_token(self):
"""
Sets a new access token on the User using the refresh token.
NOTE:
Does not work if "App client secret" is enabled. 'SECRET_HASH' is needed in AuthParameters.
'SECRET_HASH' requires HMAC calculations.
Does not work if "Device Tracking" is turned on.
https://stackoverflow.com/a/40875783/1783439
'DEVICE_KEY' is needed in AuthParameters. See AuthParameters section.
https://docs.aws.amazon.com/cognito-user-identity-pools/latest/APIReference/API_InitiateAuth.html
"""
refresh_response = self.client.initiate_auth(
ClientId=self.client_id,
AuthFlow='REFRESH_TOKEN',
AuthParameters={
'REFRESH_TOKEN': self.refresh_token
# 'SECRET_HASH': How to generate this?
},
)
self._set_attributes(
refresh_response,
{
'access_token': refresh_response['AuthenticationResult']['AccessToken'],
'id_token': refresh_response['AuthenticationResult']['IdToken'],
'token_type': refresh_response['AuthenticationResult']['TokenType']
}
)
When I run this I receive the following exception:
botocore.errorfactory.NotAuthorizedException:
An error occurred (NotAuthorizedException) when calling the InitiateAuth operation:
Unable to verify secret hash for client <client id echoed here>.
This answer informed me that a SECRET_HASH is required to use the cognito client secret.
The aws API reference docs AuthParameters section states the following:
For REFRESH_TOKEN_AUTH/REFRESH_TOKEN: USERNAME (required), SECRET_HASH
(required if the app client is configured with a client secret),
REFRESH_TOKEN (required), DEVICE_KEY
The boto3 docs state that a SECRET_HASH is
A keyed-hash message authentication code (HMAC) calculated using the
secret key of a user pool client and username plus the client ID in
the message.
The docs explain what is needed, but not how to achieve this.

The below get_secret_hash method is a solution that I wrote in Python for a Cognito User Pool implementation, with example usage:
import boto3
import botocore
import hmac
import hashlib
import base64
class Cognito:
client_id = app.config.get('AWS_CLIENT_ID')
user_pool_id = app.config.get('AWS_USER_POOL_ID')
identity_pool_id = app.config.get('AWS_IDENTITY_POOL_ID')
client_secret = app.config.get('AWS_APP_CLIENT_SECRET')
# Public Keys used to verify tokens returned by Cognito:
# http://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-with-identity-providers.html#amazon-cognito-identity-user-pools-using-id-and-access-tokens-in-web-api
id_token_public_key = app.config.get('JWT_ID_TOKEN_PUB_KEY')
access_token_public_key = app.config.get('JWT_ACCESS_TOKEN_PUB_KEY')
def __get_client(self):
return boto3.client('cognito-idp')
def get_secret_hash(self, username):
# A keyed-hash message authentication code (HMAC) calculated using
# the secret key of a user pool client and username plus the client
# ID in the message.
message = username + self.client_id
dig = hmac.new(self.client_secret, msg=message.encode('UTF-8'),
digestmod=hashlib.sha256).digest()
return base64.b64encode(dig).decode()
# REQUIRES that `ADMIN_NO_SRP_AUTH` be enabled on Client App for User Pool
def login_user(self, username_or_alias, password):
try:
return self.__get_client().admin_initiate_auth(
UserPoolId=self.user_pool_id,
ClientId=self.client_id,
AuthFlow='ADMIN_NO_SRP_AUTH',
AuthParameters={
'USERNAME': username_or_alias,
'PASSWORD': password,
'SECRET_HASH': self.get_secret_hash(username_or_alias)
}
)
except botocore.exceptions.ClientError as e:
return e.response

I also got a TypeError when I tried the above solution. Here is the solution that worked for me:
import hmac
import hashlib
import base64
# Function used to calculate SecretHash value for a given client
def calculateSecretHash(client_id, client_secret, username):
key = bytes(client_secret, 'utf-8')
message = bytes(f'{username}{client_id}', 'utf-8')
return base64.b64encode(hmac.new(key, message, digestmod=hashlib.sha256).digest()).decode()
# Usage example
calculateSecretHash(client_id, client_secret, username)