Develop Reference

How do I Mock a method that makes multiple POST and GET request all requiring different response data?

I have looked at How to mock REST API and I have read the answers but I still can't seem to get my head around how I would go about dealing with a method that executes multiple GET and POST requests. Here is some of my code below.
I have a class, UserAliasGroups(). Its __init__() method executes requests.post() to login into the external REST API. I have in my unit test this code to handling the mocking of the login and it works as expected.
#mock.patch('aliases.user_alias_groups.requests.get')
#mock.patch('aliases.user_alias_groups.requests.post')
def test_user_alias_groups_class(self, mock_post, mock_get):
init_response = {
'HID-SessionData': 'token==',
'errmsg': '',
'success': True
}
mock_response = Mock()
mock_response.json.return_value = init_response
mock_response.status_code = status.HTTP_201_CREATED
mock_post.return_value = mock_response
uag = UserAliasGroups(auth_user='TEST_USER.gen',
auth_pass='FakePass',
groups_api_url='https://example.com')
self.assertEqual(uag.headers, {'HID-SessionData': 'token=='})
I also have defined several methods like obtain_request_id(), has_group_been_deleted(), does_group_already_exists() and others. I also define a method called create_user_alias_group() that calls obtain_request_id(), has_group_been_deleted(), does_group_already_exists() and others.
I also have code in my unit test to mock a GET request to the REST API to test my has_group_been_deleted() method that looks like this:
has_group_been_deleted_response = {
'error_code': 404,
'error_message': 'A group with this ID does not exist'
}
mock_response = Mock()
mock_response.json.return_value = has_group_been_deleted_response
mock_response.status_code = status.HTTP_404_NOT_FOUND
mock_get.return_value = mock_response
Now I can get to my question. Below is the pertinent part of my code.
class UserAliasGroups:
def __init__(
self,
auth_user=settings.GENERIC_USER,
auth_pass=settings.GENERIC_PASS,
groups_api_url=settings.GROUPS_API_URL
):
""" __init__() does the login to groups. """
self.auth_user = auth_user
self.auth_pass = auth_pass
self.headers = None
self.groups_api_url = groups_api_url
# Initializes a session with the REST API service. Each login session times out after 5 minutes of inactivity.
self.login_url = f'{self.groups_api_url}/api/login'
response = requests.post(self.login_url, json={}, headers={'Content-type': 'application/json'},
auth=(auth_user, auth_pass))
if response.status_code is not 201:
try:
json = response.json()
except:
json = "Could not decode json."
raise self.UserAliasGroupsException(f"Error: User {self.auth_user}, failed to login into "
f"{self.login_url} {json}")
response_json = response.json()
self.headers = {'HID-SessionData': response_json['HID-SessionData']}
def obtain_request_id(self, request_reason):
payload = {'request_reason': request_reason}
url = f'{self.groups_api_url}/api/v1/session/requests'
response = requests.post(url=url, json=payload, headers=self.headers)
if response.status_code is not status.HTTP_200_OK:
try:
json = response.json()
except:
json = "Could not decode json."
msg = f'obtain_request_id() Error url={url} {response.status_code} {json}.'
raise self.UserAliasGroupsException(msg)
request_id = response.json().get('request_id')
return request_id
def has_group_been_deleted(self, group_name):
url = f'{self.groups_api_url}/api/v1/groups/{group_name}/attributes/RESATTR_GROUP_DELETED_ON'
response = requests.get(url=url, headers=self.headers)
return response.status_code == status.HTTP_200_OK
def does_group_already_exists(self, group_name):
url = f'{self.groups_api_url}/api/v1/groups/{group_name}'
response = requests.get(url=url, headers=self.headers)
if response.status_code is status.HTTP_200_OK:
# check if the group has been "deleted".
return not self.has_group_been_deleted(group_name=group_name)
return False
def create_user_alias_group(
self,
... long list of params omitted for brevity ...
):
if check_exists:
# Check if group already exists or not.
if self.does_group_already_exists(group_name):
msg = f'Cannot create group {group_name}. Group already exists.'
raise self.UserAliasGroupsException(msg)
... more code omitted for brevity ...
My question is how do I write my unit test to deal with multiple calls to requests.post() and request.get() all resulting in different responses in my create_user_alias_group() method?
I want to call create_user_alias_group() in my unit test so I have to figure out how to mock multiple requests.get() and requests.post() calls.
Do I have use multiple decorators like this:
#mock.patch('aliases.user_alias_groups.obtain_request_id.requests.post')
#mock.patch('aliases.user_alias_groups.does_group_already_exists.requests.get')
#mock.patch('aliases.user_alias_groups.has_group_been_deleted.requests.get')
def test_user_alias_groups_class(self, mock_post, mock_get):
...
?
Thanks for looking my long question :)

You can use mock.side_effect which takes an iterable. Then different calls will return different values:
mock = Mock()
mock.side_effect = ['a', 'b', 'c']
This way the first call to mock returns "a", then the next one "b" and so on. (In your case, you'll set mock_get.side_effect).

Error: Timestamp for this request is outside of revcWindow

I'm trying to send a request to the binance servers which require an api-key and a signature but the console is saying that the timestamp is outside of the revcWindow
I've looked up the problem and found that I need sync my computer's time to the Binance's. I'm not quite sure how to do that though (pretty newbie in Python)
def test(self):
self.url += self.url_list['test']
params = {'symbol': 'BTCETH', "timestamp": 0, "side": "BUY", "type": "LIMIT", "quantity": 0.0005, "recvWindow": 500 }
data = parammanger.encode_params(params)
data += "&signature=" + self.hash(data)
headers = {'X-MBX-APIKEY': self.a_key}
print(data)
r_body = {'signature': self.hash(data)}
r = requests.post(self.url, data, headers=headers)
print(r.request.headers)
print(r.json())
def hash(self, data):
return hashmanager.create_hash(self.s_key.encode("utf-8"), data.encode("utf-8"))
{'code': -1021, 'msg': 'Timestamp for this request is outside of the recvWindow.'}

The syncing has to do more with you SO that with Python.
Those guys here: have some solutions enter link description here
This one seems to be bulletproof (so far)
import time
from binance.client import Client
class Binance:
def __init__(self, public_key = '', secret_key = '', sync = False):
self.time_offset = 0
self.b = Client(public_key, secret_key)
if sync:
self.time_offset = self._get_time_offset()
def _get_time_offset(self):
res = self.b.get_server_time()
return res['serverTime'] - int(time.time() * 1000)
def synced(self, fn_name, **args):
args['timestamp'] = int(time.time() - self.time_offset)
return getattr(self.b, fn_name)(**args)
#I then instantiate it and call my private account functions with synced
binance = Binance(public_key = 'my_pub_key', secret_key = 'my_secret_key', sync=True)
binance.synced('order_market_buy', symbol='BNBBTC', quantity=10)
Edit:
I found an easier solution here, and the other one does not seems to be so bulletproof: enter link description here
from binance.client import Client
client = Client('api_key', 'api_secret')
import time
import win32api
gt = client.get_server_time()
tt=time.gmtime(int((gt["serverTime"])/1000))
win32api.SetSystemTime(tt[0],tt[1],0,tt[2],tt[3],tt[4],tt[5],0)

Python/Pyramid - Wrapping Cornice response with extra data when raising HTTP Exception

I'm using Pyramid and Cornice to write some RESTful Python app and I made a simple Cornice resource:
#resource(collection_path='/users/', path='/users/{id}')
class UsersResource(object):
def __init__(self, request):
self.request = request
#view(renderer='json', content_type=content_type)
#my_wrapper
def get(self):
return {'user_id': self.request.matchdict['id']}
As you may have noticed, along with Cornice's view decorator, I also added an extra decorator here (my_decorator), which I intended to use as an wrapper to add some extra information to the response:
def my_wrapper(method):
def wrapper(*args, **kw):
time_start = time()
profiler = sqltap.start()
fn_result = method(*args, **kw)
stats = profiler.collect()
time_end = time()
result = {
'info': {
'api_version': args[0].request.registry.settings.api_version,
'request_path': args[0].request.path_info,
'request_method': args[0].request.method,
'current_time': datetime.datetime.now(pytz.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
'execution_time': time_end - time_start,
'total_queries': len(stats),
'query_time': stats[0].duration if len(stats) > 0 else 0,
},
}
result.update(fn_result)
return result
return wrapper
This works fine unless I define Cornice validators in my view decorator:
from validators import validate_int
#resource(collection_path='/users/', path='/users/{id}')
class UsersResource(object):
def __init__(self, request):
self.request = request
#view(renderer='json', content_type=content_type, validators=validate_int) # added validators
#my_wrapper
def get(self):
return {'user_id': self.request.matchdict['id']}
validators.py
import responses
def validate_int(request):
should_be_int = request.matchdict['id']
try:
int(should_be_int)
except:
raise responses._400('This doesn\'t look like a valid ID.')
responses.py
class _400(exc.HTTPError):
def __init__(self, desc):
body = {'status': 400, 'message': 'Bad Request', 'description': desc}
Response.__init__(self, json.dumps(body))
With a code like this, my_wrapper wraps a response only if the validation passes (which is completely understandable), but I wonder how can I still wrap the response with some extra information when the default HTTPException is raised (because in that case the code never reaches my_wrapper at all)?

In a short discussion with a brilliant Pyramid IRC community, I decided to do this with Pyramid's tweens, rather than using the wrapper.

Where to mock to test a Django view that calls a service?

This is Django 1.6.8, Python 2.7, and the mock library.
I have a view that calls a remote service using suds for sales tax information (this is a simplified version):
def sales_tax(self, bundle_slug):
bundle = get_object_or_404(Bundle, slug=bundle_slug,
active=True)
cloud = TaxCloud(TAXCLOUD_API_ID, TAXCLOUD_API_KEY)
origin = Address('origin address')
destination = Address('customer address')
cart_item = CartItem(bundle.sku, TAXCLOUD_TIC_ONLINE_GAMES,
bundle.price, 1)
try:
rate_info = cloud.get_rate(origin, destination,
[cart_item],
str(customer.id))
sales_tax = Decimal(rate_info['sales_tax'])
response = {'salesTax': locale.currency(sales_tax),
'total': locale.currency(bundle.price + sales_tax)}
except TaxCloudException as tce:
response = {
'error': str(tce)
}
Here's relevant code from the TaxCloud class:
from suds.client import Client
from suds import WebFault
class TaxCloud(object):
def __init__(self, api_login_id, api_key):
self.api_login_id = api_login_id
self.api_key = api_key
self.soap_url = 'https://api.taxcloud.net/1.0/TaxCloud.asmx'
self.wsdl_url = 'https://api.taxcloud.net/1.0/?wsdl'
self.client = Client(url=self.wsdl_url, location=self.soap_url, faults=False)
def get_rate(self, billing_address, origin_address, raw_cart_items, customer_id):
address = self.convert_to_address(billing_address)
origin = self.convert_to_address(origin_address)
cart_items = self.convert_to_cart_list(raw_cart_items)
response = self.client.service.Lookup(self.api_login_id,
self.api_key, customer_id,
self.cart_id(customer_id), cart_items,
address, origin, True, None)
if( response[1].ResponseType == 'Error' ):
raise TaxCloudException(response[1].Messages[0][0].Message)
return {
'sales_tax': str(response[1].CartItemsResponse[0][0].TaxAmount),
'cart_id': response[1].CartID
}
In my test for the view, I don't want to call the remote service. Using this example of a mocked client, I built out a dumb mock class (my ClientMock matches exactly the example in that answer):
class TaxCloudServiceClientMock(ClientMock):
"""
Mock object that implements remote side services.
"""
def Lookup(cls, api_id, api, customer_id, cart_id, cart_items,
address, origin, flag, setting):
"""
Stub for remote service.
"""
return """(200, (LookupRsp){
ResponseType = "OK"
Messages = ""
CartID = "82cabf35faf66d8b197c7040a9f7382b3f61573fc043d73717"
CartItemsResponse =
(ArrayOfCartItemResponse){
CartItemResponse[] =
(CartItemResponse){
CartItemIndex = 1
TaxAmount = 0.10875
},
}
})"""
In my test, I'm trying to #patch the Client used in the TaxCloud class:
#patch('sales.TaxCloud.Client', new=TaxCloudServiceClientMock)
def test_can_retrieve_sales_tax(self):
from django.core.urlresolvers import reverse
tax_url = reverse('sales_tax', kwargs={'bundle_slug': self.bundle.slug})
self.client.login(username=self.user.username, password='testpassword')
response = self.client.get(tax_url, {'address1': '1234 Blah St',
'city': 'Some City',
'state': 'OH',
'zipcode': '12345',
'country': 'US'},
HTTP_X_REQUESTED_WITH='XMLHttpRequest')
self.assertEqual(response.status_code, 200)
The remote call is still being made, however. Based on the "Where to mock" documentation, I'm correctly targeting sales.TaxCloud.Client instead of suds.client.Client.
What could be causing the patch to be ignored/bypassed?

Python input validation and edge case handling

Is there a better pattern for input validation than I'm using in this function?
https://github.com/nathancahill/clearbit-intercom/blob/133e4df0cfd1a146cedb3c749fc1b4fac85a6e1b/server.py#L71
Here's the same function without any validation. It's much more readable, it's short and to the point (9 LoC vs 53 LoC).
def webhook(clearbitkey, appid, intercomkey):
event = request.get_json()
id = event['data']['item']['id']
email = event['data']['item']['email']
person = requests.get(CLEARBIT_USER_ENDPOINT.format(email=email), auth=(clearbitkey, '')).json()
domain = person['employment']['domain']
company = requests.get(CLEARBIT_COMPANY_ENDPOINT.format(domain=domain), auth=(clearbitkey, '')).json()
note = create_note(person, company)
res = requests.post(INTERCOM_ENDPOINT,
json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey))
return jsonify(note=res.json())
However, it doesn't handle any of these errors:
dict KeyError's (especially nested dicts)
HTTP errors
Invalid JSON
Unexpected responses
Is there a better pattern to follow? I looked into using a data validation library like voluptous but it seems like I'd still have the same problem of verbosity.

Your original code on github seems fine to me. It's a little over complicated, but also handle all cases of error. You can try to improve readability by abstract things.
Just for demonstration, I may write code like this:
class ValidationError(Exception):
"Raises when data validation fails"
pass
class CallExternalApiError(Exception):
"Raises when calling external api fails"
pass
def get_user_from_event(event):
"""Get user profile from event
:param dict event: request.get_json() result
:returns: A dict of user profile
"""
try:
event_type = event['data']['item']['type']
except KeyError:
raise ValidationError('Unexpected JSON format.')
if event_type != 'user':
return ValidationError('Event type is not supported.')
try:
id = event['data']['item']['id']
email = event['data']['item']['email']
except KeyError:
return ValidationError('User object missing fields.')
return {'id': id, 'email': email}
def call_json_api(request_function, api_name, *args, **kwargs):
"""An simple wrapper for sending request
:param request_function: function used for sending request
:param str api_name: name for this api call
"""
try:
res = request_function(*args, **kwargs)
except:
raise CallExternalApiError('API call failed to %s.' % api_name)
try:
return res.json()
except:
raise CallExternalApiError('Invalid response from %s.' % api_name)
#app.route('/<clearbitkey>+<appid>:<intercomkey>', methods=['POST'])
def webhook(clearbitkey, appid, intercomkey):
"""
Webhook endpoint for Intercom.io events. Uses this format for Clearbit and
Intercom.io keys:
/<clearbitkey>+<appid>:<intercomkey>
:clearbitkey: Clearbit API key.
:appid: Intercom.io app id.
:intercomkey: Intercom.io API key.
Supports User events, specifically designed for the User Created event.
Adds a note to the user with their employment and company metrics.
"""
event = request.get_json()
try:
return handle_event(event, clearbitkey, appid, intercomkey)
except (ValidationError, CallExternalApiError) as e:
# TODO: include **res_objs in response
return jsonify(error=str(e))
def handle_event(event):
"""Handle the incoming event
"""
user = get_user_from_event(event)
res_objs = dict(event=event)
person = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_USER_ENDPOINT.format(email=user['email']),
auth=(clearbitkey, '')
)
res_objs['person'] = person
if 'error' in person:
raise CallExternalApiError('Error response from Clearbit.')
domain = person['employment']['domain']
company = None
if domain:
try:
company = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_COMPANY_ENDPOINT.format(domain=domain),
auth=(clearbitkey, ''))
)
if 'error' in company:
company = None
except:
company = None
res_objs['company'] = company
try:
note = create_note(person, company)
except:
return jsonify(error='Failed to generate note for user.', **res_objs)
result = call_json_api(
requests.post,
'Intercom',
(INTERCOM_ENDPOINT, json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey)
)
return jsonify(note=result, **res_objs)
I hope it helps.