I'm new to Python unit testing, and I want to mock calls to the boto3 3rd party library. Here's my stripped down code:
real_code.py:
import boto3
def temp_get_variable(var_name):
return boto3.client('ssm').get_parameter(Name=var_name)['Parameter']['Value']
test_real_code.py:
import unittest
from datetime import datetime
from unittest.mock import patch
import real_code
class TestRealCode(unittest.TestCase):
#patch('patching_config.boto3.client')
def test_get_variable(self, mock_boto_client):
response = {
'Parameter': {
'Name': 'MyTestParameterName',
'Type': 'String',
'Value': 'myValue',
'Version': 123,
'Selector': 'asdf',
'SourceResult': 'asdf',
'LastModifiedDate': datetime(2019, 7, 16),
'ARN': 'asdf'
}
}
mock_boto_client.get_variable.return_value = response
result_value = real_code.get_variable("MyTestParameterName")
self.assertEqual("myValue", result_value)
When I run it the test fails with
Expected :myValue
Actual :<MagicMock name='client().get_parameter().__getitem__().__getitem__()' id='2040071816528'>
What am I doing wrong? I thought by setting mock_boto_client.get_variable.return_value = response it would mock out the call and return my canned response instead. I don't understand why I am getting a MagicMock object instead of the return value I tried to set. I'd like to set up my test so that when the call to get_parameter is made with specific parameters, the mock returns the canned response I specified in the test.
There are two issues with your test code. The first is that when your mock object mock_boto_client called, it returns a new mock object. This means that the object that get_parameter() is being called on is different than the one you are attempting to set a return value on. You can have it return itself with the following:
mock_boto_client.return_value = mock_boto_client
You can also use a different mock object:
foo = MagicMock()
mock_boto_client.return_value = foo
The second issue that you have is that you are mocking the wrong method call. mock_boto_client.get_variable.return_value should be mock_boto_client.get_parameter.return_value. Here is the test updated and working:
import unittest
from datetime import datetime
from unittest.mock import patch
import real_code
class TestRealCode(unittest.TestCase):
#patch('boto3.client')
def test_get_variable(self, mock_boto_client):
response = {
'Parameter': {
'Name': 'MyTestParameterName',
'Type': 'String',
'Value': 'myValue',
'Version': 123,
'Selector': 'asdf',
'SourceResult': 'asdf',
'LastModifiedDate': datetime(2019, 7, 16),
'ARN': 'asdf'
}
}
mock_boto_client.return_value = mock_boto_client
mock_boto_client.get_parameter.return_value = response
result_value = real_code.get_variable("MyTestParameterName")
self.assertEqual("myValue", result_value)
Related
I am Getting this error. I am executing code of aws lambda function using python 3.7 to know quicksight dashboard version. Thanks in advance!
errorMessage: "Unable to marshal response: Object of type datetime is not JSON serializable",
errorType : "Runtime.MarshalError"
Code-
import boto3
import time
import sys
client = boto3.client('quicksight')
def lambda_handler(event, context):
response = client.list_dashboard_versions(AwsAccountId='11111', DashboardId='2222',MaxResults=10)
return response
I quick fix could be:
import boto3
import time
import sys
import json
client = boto3.client('quicksight')
def lambda_handler(event, context):
response = client.list_dashboard_versions(AwsAccountId='11111', DashboardId='2222',MaxResults=10)
return json.dumps(response, default=str)
Looking at https://boto3.amazonaws.com/v1/documentation/api/1.14.8/reference/services/quicksight.html#QuickSight.Client.list_dashboard_versions the return looks like this -
{
'DashboardVersionSummaryList': [
{
'Arn': 'string',
'CreatedTime': datetime(2015, 1, 1),
'VersionNumber': 123,
'Status': 'CREATION_IN_PROGRESS'|'CREATION_SUCCESSFUL'|'CREATION_FAILED'|'UPDATE_IN_PROGRESS'|'UPDATE_SUCCESSFUL'|'UPDATE_FAILED',
'SourceEntityArn': 'string',
'Description': 'string'
},
],
'NextToken': 'string',
'Status': 123,
'RequestId': 'string'
}
As you can see, CreatedTime is returned as datetime. If you want to return this as a JSON, you should transform this value.
I was struggling with this today with a method that also returns a datetime.
In my example 'JoinedTimestamp': datetime(2015, 1, 1) resulting in the same Unable to marshal response.
If you don't need the CreatedTime value you might as well remove it from the response as:
for account in list_accounts_response["Accounts"]:
if "JoinedTimestamp" in account:
del account["JoinedTimestamp"]
To follow up on Joseph Lane's answer, transforming this value could be something along the lines of:
for account in list_accounts_response["Accounts"]:
if "JoinedTimestamp" in account:
account["JoinedTimestamp"] = str(account["JoinedTimestamp"])
I have a function that verifies if a given input string is a proper GCP zone:
def validate_zone(compute, project_id, zone):
try:
zone_response = compute.zones().get(project=project_id, zone=zone).execute()
print(zone_response)
print(zone_response.return_value)
if ['status'] in zone_response:
zone_details = {
'status': zone_response['status'],
'region': zone_response['region'],
'name': zone_response['name']
}
return zone_details
else:
return "Zone {} not found for project {}".format(zone, project_id)
except HttpError as error:
print("Error calling zone {}: \n {}".format(zone, error))
I am trying to write a test to verify that but I can't mock the output of the compute method correctly.
#patch('googleapiclient.discovery')
def test_validate_zone(self, mock_response):
compute = mock_response.build(serviceName='compute', version='v1')
compute.zones().get(project_id=self.project_id, zone=self.zone).execute().return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
This results in the zone_response output being a MagicMock object with its return_value being correct as developed in the test.
zone_response = MagicMock name='discovery.build().zones().get().execute()' id='139870134525456'
zone_response.return_value = {'status': 'status', 'region': 'region', 'name': 'name'}
Any ideas on what I'm doing wrong? I've been trying to write tests for this for quite a while so maybe my approach is just off.
Turns out the issue was the () on the execute method in the test. So the correct test should be:
#patch('inventory.discovery.build', serviceName='compute', version='v1')
def test_validate_zone(self, compute):
print(compute)
compute.zones().get(project_id=self.project_id, zone=self.zone).execute.return_value = {
'status': 'status',
'region': 'region',
'name': 'name'
}
zone_response = inventory.validate_zone(compute, self.project_id, self.zone)
print(zone_response)
Source can be found at: https://realpython.com/python-mock-library/#managing-a-mocks-return-value
I want to create a key that's managed by AWS. So far this is what I have
#mock_kms
def test_mocking_getting_keys(self):
session = boto3.Session(profile_name=profile)
client = session.client('kms', 'us-east-2')
key = client.create_key(
Policy='string',
Description='string',
KeyUsage='SIGN_VERIFY',
CustomerMasterKeySpec='RSA_2048',
Origin='AWS_KMS',
CustomKeyStoreId='string',
BypassPolicyLockoutSafetyCheck=True,
Tags=[
{
'TagKey': 'string',
'TagValue': 'string'
},
]
)
print(key)
But the key doesn't seem to have KeyManager field:
{'KeyMetadata': {'AWSAccountId': '012345678912', 'KeyId': '7fc3e676-0d1c-4526-9161-41b27a776033', 'Arn': 'arn:aws:kms:us-east-2:012345678912:key/7fc3e676-0d1c-4526-9161-41b27a776033', 'CreationDate': datetime.datetime(2020, 1, 3, 13, 31, 17, tzinfo=tzutc()), 'Enabled': True, 'Description': 'string', 'KeyUsage': 'SIGN_VERIFY', 'KeyState': 'Enabled'}, 'ResponseMetadata': {'HTTPStatusCode': 200, 'HTTPHeaders': {'server': 'amazon.com'}, 'RetryAttempts': 0}}
I tried adding KeyManager as a param during create_key call but that didn't work either.
Seems like moto doens't return the KeyManager field. Is there a way to mock that return value specifically but not change the behavior of the dictionary.get method for the rest of the params?
i.e.
key['KeyMetadata']['AWSAccountId'] would return the mocked value and then
key['KeyMetadata']['KeyManager'] would return a another mocked value that I could specify.
The KeyManager attribute is currently not returned by Moto, you can either open an Issue on the Moto GitHub, or add it yourself (either locally, or PR'ed to upstream)
I am having a problem understanding how mock works and how to write unittests with mock objects. I wanted to mock an external api call every time when my model calls save() method.
My code:
models.py
from . import utils
class Book(Titleable, Isactiveable, Timestampable, IsVoidable, models.Model):
title
orig_author
orig_title
isbn
def save(self, *args, **kwargs):
if self.isbn:
google_data = utils.get_original_title_and_name(self.isbn)
if google_data:
self.original_author = google_data['author']
self.original_title = google_data['title']
super().save(*args, **kwargs)
utils.py
def get_original_title_and_name(isbn, **kawargs):
isbn_search_string = 'isbn:{}'.format(isbn)
payload = {
'key': GOOGLE_API_KEY,
'q': isbn_search_string,
'printType': 'books',
}
r = requests.get(GOOGLE_API_URL, params=payload)
response = r.json()
if 'items' in response.keys():
title = response['items'][THE_FIRST_INDEX]['volumeInfo']['title']
author = response['items'][THE_FIRST_INDEX]['volumeInfo']['authors'][THE_FIRST_INDEX]
return {
'title': title,
'author': author
}
else:
return None
I began read docs and write test:
test.py:
from unittest import mock
from django.test import TestCase
from rest_framework import status
from .constants import THE_FIRST_INDEX, GOOGLE_API_URL, GOOGLE_API_KEY
class BookModelTestCase(TestCase):
#mock.patch('requests.get')
def test_get_original_title_and_name_from_google_api(self, mock_get):
# Define new Mock object
mock_response = mock.Mock()
# Define response data from Google API
expected_dict = {
'kind': 'books#volumes',
'totalItems': 1,
'items': [
{
'kind': 'books#volume',
'id': 'IHxXBAAAQBAJ',
'etag': 'B3N9X8vAMWg',
'selfLink': 'https://www.googleapis.com/books/v1/volumes/IHxXBAAAQBAJ',
'volumeInfo': {
'title': "Alice's Adventures in Wonderland",
'authors': [
'Lewis Carroll'
]
}
}
]
}
# Define response data for my Mock object
mock_response.json.return_value = expected_dict
mock_response.status_code = 200
# Define response for the fake API
mock_get.return_value = mock_response
The first of all, I can't write target for the #mock.patch correct. If a define target as utuls.get_original_title_and_name.requests.get, I get ModuleNotFoundError. Also I can't understand how to make fake-call to external API and verify recieved data (whether necessarly its, if I've already define mock_response.json.return_value = expected_dict?) and verify that my save() method work well?
How do I write test for this cases? Could anyone explain me this case?
You should mock the direct collaborators of the code under test. For Book that would be utils. For utils that would be requests.
So for the BookModelTestCase:
class BookModelTestCase(TestCase):
#mock.patch('app.models.utils')
def test_save_book_calls_google_api(self, mock_utils):
mock_utils.get_original_title_and_name.return_value = {
'title': 'Google title',
'author': 'Google author'
}
book = Book(
title='Some title',
isbn='12345'
)
book.save()
self.assertEqual(book.title, 'Google title')
self.assertEqual(book.author, 'Google author')
mock_utils.get_original_title_and_name.assert_called_once_with('12345')
And then you can create a separate test case to test get_original_title_and_name:
class GetOriginalTitleAndNameTestCase(TestCase):
#mock.patch('app.utils.requests.get')
def test_get_original_title_and_name_from_google_api(self, mock_get):
mock_response = mock.Mock()
# Define response data from Google API
expected_dict = {
'kind': 'books#volumes',
'totalItems': 1,
'items': [
{
'kind': 'books#volume',
'id': 'IHxXBAAAQBAJ',
'etag': 'B3N9X8vAMWg',
'selfLink': 'https://www.googleapis.com/books/v1/volumes/IHxXBAAAQBAJ',
'volumeInfo': {
'title': "Alice's Adventures in Wonderland",
'authors': [
'Lewis Carroll'
]
}
}
]
}
# Define response data for my Mock object
mock_response.json.return_value = expected_dict
mock_response.status_code = 200
# Define response for the fake API
mock_get.return_value = mock_response
# Call the function
result = get_original_title_and_name(12345)
self.assertEqual(result, {
'title': "Alice's Adventures in Wonderland",
'author': 'Lewis Carroll'
})
mock_get.assert_called_once_with(GOOGLE_API_URL, params={
'key': GOOGLE_API_KEY,
'q': 'isbn:12345',
'printType': 'books',
})
I have clien-server app.
I localized trouble and there logic of this:
Client:
# -*- coding: utf-8 -*-
import requests
def fixing:
response = requests.post('http://url_for_auth/', data={'client_id': 'client_id',
'client_secret':'its_secret', 'grant_type': 'password',
'username': 'user', 'password': 'password'})
f = response.json()
data = {'coordinate_x': 12.3, 'coordinate_y': 8.4, 'address': u'\u041c, 12',
'products': [{'count': 1, 'id': 's123'},{'count': 2, 'id': 's124'}]}
data.update(f)
response = requests.post('http://url_for_working/, data=data)
response.text #There I have an Error about which I will say later
oAuth2 working well. But in server-side I have no products in request.data
<QueryDict: {u'token_type': [u'type_is_ok'], u'access_token': [u'token_is_ok'],
u'expires_in': [u'36000'], u'coordinate_y': [u'8.4'],
u'coordinate_x': [u'12.3'], u'products': [u'count', u'id', u'count',
u'id'], u'address': [u'\u041c, 12'], u'scope': [u'read write'],
u'refresh_token': [u'token_is_ok']}>
This part of QueryDict make me sad...
'products': [u'count', u'id', u'count', u'id']
And when I tried to make python dict:
request.data.dict()
... u'products': u'id', ...
And for sure other fields working well with Django serializer's validation. But not that, because there I have wrong values.
Looks like request (because it have x-www-encoded-form default) cant include list of dicts as value for key in dict so... I should use json in this case.
Finally I maked this func:
import requests
import json
def fixing:
response = requests.post('http://url_for_auth/', data={'client_id': 'client_id',
'client_secret':'its_secret', 'grant_type': 'password',
'username': 'user', 'password': 'password'})
f = response.json()
headers = {'authorization': f['token_type'].encode('utf-8')+' '+f['access_token'].encode('utf-8'),
'Content-Type': 'application/json'}
data = {'coordinate_x': 12.3, 'coordinate_y': 8.4, 'address': u'\u041c, 12',
'products': [{'count': 1, 'id': 's123'},{'count': 2, 'id': 's124'}]}
response = requests.post('http://url_for_working/', data=json.dumps(data),
headers=headers)
response.text
There I got right response.
Solved!
Hello i would like to refresh this topic, cause i have similar problem to this and above solution doesn`t work for me.
import requests
import urllib.request
import pprint
import json
from requests import auth
from requests.models import HTTPBasicAuth
payload = {
'description': 'zxcy',
'tags':[{
'id': 22,
'label': 'Card'}]
}
files = {'file': open('JAM5.pdf','rb')}
client_id = 32590
response = requests.post('https://system...+str(client_id)' , files=files ,data=payload, auth=HTTPBasicAuth(...)
Above code succesfully add file to CRM system and description to added file, but i have to add label to this too, and its seems doesnt work at all
When i try it with data=json.dumps(payload) i got this:
raise ValueError("Data must not be a string.")
ValueError: Data must not be a string.