Im trying to write a REST Client for a closed API. I call a specific function twice but it only works the first time. It's the same command. If i run the script twice with batch It works. Im suspecting Requests keeps the connection alive or caches data. How do i "reset" Requests ?
base_headers = {"Connection":"keep-alive",
"User-Agent":user_agent,
"Accept-Encoding":"gzip",
"Host":"xxxxxxxxxxx",
"Content-Type":"application/json; charset=UTF-8"}
global auth
auth = 'xxxxxx'
def create_header(doAuth = True):
headers = base_headers
if doAuth:
headers['Authorization'] = auth
return headers
def do_get(url):
return requests.get(url, headers=create_header());
def do_put(url, payload):
if payload is None:
return requests.put(url, headers=create_header())
return requests.put(url, data=payload, headers=create_header())
def upvote(postid):
return do_put("xxxxxxxxxxxxx" % postid, None).content
def set_pos(longtitude, latitude, place, country="DE"):
payload = {'xxxxx':'xxxx'}
json_payload = json.dumps(payload)
return do_put("xxxxxxxxxxxxxxx",json_payload).content
def do_post(url, payload, doAuth=True):
return requests.post(url, data=payload, headers=create_header(doAuth=doAuth))
def new_acc(place, latitude, longtitude):
access = get_access_token(place, latitude, longtitude)
print access
global auth
auth = "Bearer %s" % access['access_token']
set_pos(longtitude, latitude, place) # Does a POST
for i in range(1,30):
posts = new_acc(uni['city'], uni['latitude'], uni['longtitude'])
upvote('xxxxxxxxxxxxx')
Basically the first upvote() goes through every time but every succeding does not.
I'm almost positive that keep-alive is not doing this. I would suggest writing some handlers to debug the response if you don't get the expected response code after the request.
Also, might I suggest a slightly different design that avoids global?
class RESTClient(object):
BASE_HEADERS = {"Connection":"keep-alive",
"Accept-Encoding":"gzip",
"Host":"xxxxxxxxxxx",
"Content-Type":"application/json; charset=UTF-8"}
def __init__(self, user_agent, auth = None):
self.headers = dict(self.BASE_HEADERS)
self.headers['User-Agent'] = user_agent
self.auth = auth
def create_header(self):
headers = dict(self.headers)
if auth:
headers['Authorization'] = auth
return headers
def do_get(self, url):
return requests.get(url, headers=self.create_header());
def do_put(self, url, payload = None):
if not payload:
return requests.put(url, headers=self.create_header())
return requests.put(url, data=payload, headers=self.create_header())
def upvote(self, postid):
return do_put("xxxxxxxxxxxxx" % postid).content
def set_pos(self, longtitude, latitude, place, country="DE"):
payload = {'xxxxx':'xxxx'}
json_payload = json.dumps(payload)
return do_put("xxxxxxxxxxxxxxx",json_payload).content
def do_post(self, url, payload, do_auth = None):
return requests.post(url, data=payload, headers=self.create_header())
def new_acc(self, place, latitude, longtitude):
access = get_access_token(place, latitude, longtitude)
print access
self.auth = "Bearer %s" % access['access_token']
set_pos(longtitude, latitude, place)
rc = RESTClient('Mozilla/5.0 ... Firefox/38.0', 'my-auth-token')
rc.upvote(post_id)
etc...
It could be that your use of globals is breaking things. Having not run your code, I can't check, but I would avoid using globals and favor tighter control of your state via objects.
EDIT:
Given that this answer was accepted by the OP, I am assuming the defect was caused by mutation of globals.
Related
I am using Django Rest API on open stack version 2.1 Unable to update Server Tags it is giving response text:'{"itemNotFound": {"code": 404, "message": "The resource could not be found."}}'
the code explanation is there is an array of object contains multiple server objects with updated tags within a loop it will request and append responses in an array for front-end
#classmethod
def update_tags(self, params):
check(self, params)
responses = []
try:
headers = {
"X-Auth-Token": params.headers['x-auth-token'],
"Content-type": 'application/json'
}
for server in params.data['payload']:
replace_tags = {"tags": server["tags"]}
response=requests.put(os.environ.get('PROTOCOL')+'://'+os.environ.get('OPENSTACK_HOST')+':'+os.environ.get('COMPUTE_PORT')+'/v2.1/servers/' +server['id']+'/tags', data=json.dumps(replace_tags), headers=headers, verify=False)
responses.append(response)
return responses
except Exception as e:
raise e
parameters which are being passed from check method which is written below
def check(self, params):
self.connection = get_openstack_conn(params)
the get_openstack_conn method
def get_openstack_conn(params):
token = params.META.get("HTTP_X_AUTH_TOKEN")
scope = params.META.get("HTTP_X_SCOPE")
organization = params.META.get("HTTP_X_ORG")
print("environment variables")
print("PROTOCOL = ",os.environ.get('PROTOCOL'))
print("OPENSTACK_HOST = ",os.environ.get('OPENSTACK_HOST'))
print("KEY_STONE_PORT = ",os.environ.get('KEY_STONE_PORT'))
if(scope == 'projectScope'):
connector = openstack.connect(auth_url=os.environ.get('PROTOCOL')+'://'+os.environ.get('OPENSTACK_HOST') +
':'+os.environ.get('KEY_STONE_PORT')+'/v3', verify=False, auth_type="token", token=token, project_id=organization)
elif(scope == 'domainScope'):
connector = openstack.connect(auth_url=os.environ.get('PROTOCOL')+'://'+os.environ.get('OPENSTACK_HOST') +
':'+os.environ.get('KEY_STONE_PORT')+'/v3', verify=False, auth_type="token", token=token, domain_id=organization)
return connector
I'm trying to figure out how to mock my request for a bearer token in python.
I have a class:
class grab_apitokens(object):
def __init__(self, consumer_key, first_api_url, second_api_user, second_api_password, second_api_url):
self.consumer_key = consumer_key
self.second_api_user = second_api_user
self.second_api_password = second_api_password
self.first_api_url = first_api_url
self.second_api_url = second_api_url
def logintofirstsite(self):
b64val = base64.b64encode(self.consumer_key.encode()).decode()
headers = {"Authorization": "Basic %s" % b64val}
data = {'grant_type': 'client_credentials', 'validity_period': '3600'}
try:
response = requests.post(self.first_api_url, headers=headers, data=data)
decodedresponse = json.loads(response.content.decode())
access_token = decodedresponse['access_token']
return access_token
except:
return None
def logintosecondsite(self):
header = {"accept": "application/json", "Content-Type": "application/x-www-form-urlencoded"}
logindata = {'grant_type': 'password',
'username': "" + self.second_api_user + "", 'password': "" + self.second_api_password + ""
}
try:
returnedfromsite = requests.post(self.second_api_url + '/api/V1/token',
headers=header, data=logindata)
return returnedfromsite.json()['access_token']
except:
return None
What I can't figure out is how to mock that requests call and what it would look like in Python.
My test currently looks like:
class MyTestCase(unittest.TestCase):
def setUp(self) -> None: # PROBLEM BEGINS HERE
self.grab_apitokens = grab_apitokens(actual_key, actual_site1, actual_user, actual_pass, actual_2nd_site)
#patch('grab_apitokens.requests.get')
def test_login(self, mock_get):
mock_get.return_value.ok = True
response = self.grab_apitokens.logintosite()
assert_is_not_none(response)
# self.assertEqual(True, False)
if __name__ == '__main__':
unittest.main()
How would I mock the requests.post functionality?
With the help of a good mentor I figured out that my approach was all wrong. Here's what I ended up with for the unit test:
class MyTestCase(unittest.TestCase):
def setUp(self) -> None:
self.grab_apitokens = grab_apitokens("complete","gibberish","it really doesnt","matter","what is","in","here")
#patch('grab_apitokens.requests.posts')
def test_login(self, mock_get):
mock_json = {'token': 'foo'}
mock_get.return_value = Mock(ok=True)
mock_get.return_value.json.return_value = mock_json
mock_get.return_value.content = b'{"token": "foo"}'
response = self.grab_apitokens.logintofirstsite()
assert_equal(response, "foo")
if __name__ == '__main_
To understand what this does, I needed to know that what we're really mocking isn't the method logintofirstsite(), we're mocking the response that requests.post is making in the method. With mock_get, we're inject requests.posts to always be: {'token': 'foo'} . So everything after
response = requests.post(self.first_api_url, headers=headers, data=data)
in logintofirstsite() is what I'm really testing. Which is:
decodedresponse = json.loads(response.content.decode())
access_token = decodedresponse['token']
return access_token
The setup before the requests.post call doesn't matter one bit. Since with {'token': 'foo'} is what my requests.post call returns, the returned value after that bit of logic is 'foo', and so the assert_equal back in MyTestCase passes.
I have looked at How to mock REST API and I have read the answers but I still can't seem to get my head around how I would go about dealing with a method that executes multiple GET and POST requests. Here is some of my code below.
I have a class, UserAliasGroups(). Its __init__() method executes requests.post() to login into the external REST API. I have in my unit test this code to handling the mocking of the login and it works as expected.
#mock.patch('aliases.user_alias_groups.requests.get')
#mock.patch('aliases.user_alias_groups.requests.post')
def test_user_alias_groups_class(self, mock_post, mock_get):
init_response = {
'HID-SessionData': 'token==',
'errmsg': '',
'success': True
}
mock_response = Mock()
mock_response.json.return_value = init_response
mock_response.status_code = status.HTTP_201_CREATED
mock_post.return_value = mock_response
uag = UserAliasGroups(auth_user='TEST_USER.gen',
auth_pass='FakePass',
groups_api_url='https://example.com')
self.assertEqual(uag.headers, {'HID-SessionData': 'token=='})
I also have defined several methods like obtain_request_id(), has_group_been_deleted(), does_group_already_exists() and others. I also define a method called create_user_alias_group() that calls obtain_request_id(), has_group_been_deleted(), does_group_already_exists() and others.
I also have code in my unit test to mock a GET request to the REST API to test my has_group_been_deleted() method that looks like this:
has_group_been_deleted_response = {
'error_code': 404,
'error_message': 'A group with this ID does not exist'
}
mock_response = Mock()
mock_response.json.return_value = has_group_been_deleted_response
mock_response.status_code = status.HTTP_404_NOT_FOUND
mock_get.return_value = mock_response
Now I can get to my question. Below is the pertinent part of my code.
class UserAliasGroups:
def __init__(
self,
auth_user=settings.GENERIC_USER,
auth_pass=settings.GENERIC_PASS,
groups_api_url=settings.GROUPS_API_URL
):
""" __init__() does the login to groups. """
self.auth_user = auth_user
self.auth_pass = auth_pass
self.headers = None
self.groups_api_url = groups_api_url
# Initializes a session with the REST API service. Each login session times out after 5 minutes of inactivity.
self.login_url = f'{self.groups_api_url}/api/login'
response = requests.post(self.login_url, json={}, headers={'Content-type': 'application/json'},
auth=(auth_user, auth_pass))
if response.status_code is not 201:
try:
json = response.json()
except:
json = "Could not decode json."
raise self.UserAliasGroupsException(f"Error: User {self.auth_user}, failed to login into "
f"{self.login_url} {json}")
response_json = response.json()
self.headers = {'HID-SessionData': response_json['HID-SessionData']}
def obtain_request_id(self, request_reason):
payload = {'request_reason': request_reason}
url = f'{self.groups_api_url}/api/v1/session/requests'
response = requests.post(url=url, json=payload, headers=self.headers)
if response.status_code is not status.HTTP_200_OK:
try:
json = response.json()
except:
json = "Could not decode json."
msg = f'obtain_request_id() Error url={url} {response.status_code} {json}.'
raise self.UserAliasGroupsException(msg)
request_id = response.json().get('request_id')
return request_id
def has_group_been_deleted(self, group_name):
url = f'{self.groups_api_url}/api/v1/groups/{group_name}/attributes/RESATTR_GROUP_DELETED_ON'
response = requests.get(url=url, headers=self.headers)
return response.status_code == status.HTTP_200_OK
def does_group_already_exists(self, group_name):
url = f'{self.groups_api_url}/api/v1/groups/{group_name}'
response = requests.get(url=url, headers=self.headers)
if response.status_code is status.HTTP_200_OK:
# check if the group has been "deleted".
return not self.has_group_been_deleted(group_name=group_name)
return False
def create_user_alias_group(
self,
... long list of params omitted for brevity ...
):
if check_exists:
# Check if group already exists or not.
if self.does_group_already_exists(group_name):
msg = f'Cannot create group {group_name}. Group already exists.'
raise self.UserAliasGroupsException(msg)
... more code omitted for brevity ...
My question is how do I write my unit test to deal with multiple calls to requests.post() and request.get() all resulting in different responses in my create_user_alias_group() method?
I want to call create_user_alias_group() in my unit test so I have to figure out how to mock multiple requests.get() and requests.post() calls.
Do I have use multiple decorators like this:
#mock.patch('aliases.user_alias_groups.obtain_request_id.requests.post')
#mock.patch('aliases.user_alias_groups.does_group_already_exists.requests.get')
#mock.patch('aliases.user_alias_groups.has_group_been_deleted.requests.get')
def test_user_alias_groups_class(self, mock_post, mock_get):
...
?
Thanks for looking my long question :)
You can use mock.side_effect which takes an iterable. Then different calls will return different values:
mock = Mock()
mock.side_effect = ['a', 'b', 'c']
This way the first call to mock returns "a", then the next one "b" and so on. (In your case, you'll set mock_get.side_effect).
Currently my API requires to use Token authentication.
POST /api/authorize HTTP/1.1
"version": "20151130", // The version of the REST API you are using
"client_id": "01234567890123456789", // Your 20 char public client id
"client_secret": "0123456789..." // Your 40 char client secret
I get the response:
{
"auth_token": "abcdef...",
"auth_expires": "20180524T062442Z"
}
My current pattern is like this:
I have a list of items I need to pass to the API via POST method.
This is my main function: ProcessProducts which receives a Pandas Dataframe which includes a product per row.
def ProcessProducts(products):
all_results = []
for _, product in products.iterrows():
results = GetProductData(product)
if results:
all_results.extend(results)
return all_results
def GetAuthorizationToken():
payload = {
'version': api_version,
'client_id': api_client_id,
'client_secret': api_client_secret
}
request_url = '%s%s' % (api_host, '/api/authorize')
r = requests.post(request_url, data=payload)
if r.status_code != 200:
raise Exception('Failed to authorize: ' + r.text)
token_data = json.loads(r.text)
api_auth_token = token_data['auth_token']
api_auth_expires = token_data['auth_expires']
return {
"X-API-Version": api_version,
"Authorization": "token %s" % api_auth_token
}
Client function...
def GetProductData(self, product):
"""Gets Product information from API."""
url = '%s%s' % (self.api_url, _COMPANY_DATA)
request = json.dumps({'products': [product]})
form_data = {'request': request, 'start': 1, 'limit': 1000}
logging.info('Looking up: %s', url)
auth_headers = GetAuthorizationToken()
response = _SendApiRequest(url, auth_headers, form_data)
return _HandleResponse(response)
def _SendApiRequest(self, url, auth_headers, form_data):
session = requests.Session()
try:
response = session.post(
url,
timeout=(_CONNECT_TIMEOUT_SECONDS, _READ_TIMEOUT_SECONDS),
headers=auth_headers,
data=form_data,
verify=True) # Check for valid public/signed HTTPS certificate.
response.raise_for_status()
return response
except requests.exceptions.HTTPError as err:
logging.exception(err)
Questions:
API returns the code expiry field "auth_expires", Where may be the best way to check in code when token expires so I can request a new one?
Is there a better Pattern to call API, so I can control the QPS rate as well (Use RateLimiter). Right now I'm creating a Session per request, which may not be ideal.
Not sure where my problem is. I would use the Try It Out option in the docs page but there doesn't appear to be any way of providing a token/secret.
I keep getting an error 500 when trying to send a message from a Jupyter notebook to my bot. Here's my code:
class DirectLineConversation:
"""A conversation with a bot via the Direct Line API."""
def __init__(self, direct_line_secret):
self._direct_line_secret = direct_line_secret
self._base_url = 'https://directline.botframework.com/api'
self._initiate_conversation()
def _initiate_conversation(self):
self._init_headers()
requests.post("https://directline.botframework.com/api/tokens/conversation", headers=self._headers)
r = requests.post("https://directline.botframework.com/api/conversations", headers=self._headers)
json_ = r.json()
self.conversationId = json_['conversationId']
self.token = json_['token']
def _init_headers(self):
headers = {'Content-Type': 'application/json'}
headers.update(self._get_authorization_pair())
self._headers = headers
def _get_authorization_pair(self):
value = ' '.join(['BotConnector', self._direct_line_secret])
return {'Authorization': value}
def send_message(self, text):
url = '/'.join([self._base_url, 'conversations', self.conversationId, 'messages'])
print(url)
return requests.post(url, headers=self._headers, json={'conversationId': self.conversationId,
'text': text})
convo = DirectLineConversation(_DIRECT_LINE_SECRET)
response = convo.send_message("Hello")
print(response.status_code)
print(response.json())
According to the documentation, everything in the schema is optional so I assumed passing json={'text': text} would work just fine however I have also tried it with all other values being null and also with the conversationId as I have seen in several other SO answers.
Here is the output:
https://directline.botframework.com/api/conversations/3BpMMKssV6P/messages
500
{'message': 'failed to send message'}