What I've Done
I've written an authentication class for obtaining an application's bearer token from Twitter using the application's API Key and its API key secret as demonstrated in the Twitter developer docs.
I've mocked the appropriate endpoint using requests_mock this way:
#pytest.fixture
def mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
requests_mock.post(
"https://api.twitter.com/oauth2/token",
request_headers={
"Authorization": f"Basic {basic_auth_string}",
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
},
json={"token_type": "bearer", "access_token": f"{bearer_token}"},
)
And my test method is :
#pytest.mark.usefixtures("mock_post_bearer_token_endpoint")
def test_basic_auth(api_key, api_key_secret, bearer_token):
response = requests.post(
'https://api.twitter.com/oauth2/token',
data={"grant_type": "client_credentials"},
auth=TwitterBasicAuth(api_key, api_key_secret),
)
assert response.json()['access_token'] == bearer_token
(Where TwitterBasicAuth is the authentication class I wrote, and the fixture basic_auth_string is a hardcoded string that would be obtained from transforming the fixtures api_key and api_key_secret appropriately).
And it works.
The Problem
But I'm really bothered by the fact that the mocked endpoint doesn't check the payload. In this particular case, the payload is vital to obtain a bearer token.
I've combed through the documentation for requests_mock (and responses, too) but haven't figured out how to make the endpoint respond with a bearer token only when the correct payload is POSTed.
Please help.
I think the misconception here is that you need to put everything in the matcher and let NoMatchException be the thing to tell you if you got it right.
The matcher can be the simplest thing it needs to be in order to return the right response and then you can do all the request/response checking as part of your normal unit test handling.
additional_matchers is useful if you need to switch the response value based on the body of the request for example, and typically true/false is sufficient there.
eg, and i made no attempt to look up twitter auth for this:
import requests
import requests_mock
class TwitterBasicAuth(requests.auth.AuthBase):
def __init__(self, api_key, api_key_secret):
self.api_key = api_key
self.api_key_secret = api_key_secret
def __call__(self, r):
r.headers['x-api-key'] = self.api_key
r.headers['x-api-key-secret'] = self.api_key_secret
return r
with requests_mock.mock() as m:
api_key = 'test'
api_key_secret = 'val'
m.post(
"https://api.twitter.com/oauth2/token",
json={"token_type": "bearer", "access_token": "token"},
)
response = requests.post(
'https://api.twitter.com/oauth2/token',
data={"grant_type": "client_credentials"},
auth=TwitterBasicAuth(api_key, api_key_secret),
)
assert response.json()['token_type'] == "bearer"
assert response.json()['access_token'] == "token"
assert m.last_request.headers['x-api-key'] == api_key
assert m.last_request.headers['x-api-key-secret'] == api_key_secret
https://requests-mock.readthedocs.io/en/latest/history.html
Updated Answer
I went with gold_cy's comment and wrote a custom matcher that takes a request and returns an appropriately crafted OK response if the request has the correct url path, headers and json payload. It returns a 403 response otherwise, as I'd expect from the Twitter API.
#pytest.fixture
def mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
def matcher(req):
if req.path != "/oauth2/token":
# no mock address
return None
if req.headers.get("Authorization") != f"Basic {basic_auth_string}":
return create_forbidden_response()
if (
req.headers.get("Content-Type")
!= "application/x-www-form-urlencoded;charset=UTF-8"
):
return create_forbidden_response()
if req.json().get("grant_type") != "client_credentials":
return create_forbidden_response()
resp = requests.Response()
resp._content = json.dumps(
{"token_type": "bearer", "access_token": f"{bearer_token}"}
).encode()
resp.status_code = 200
return resp
requests_mock._adapter.add_matcher(matcher)
yield
def create_forbidden_response():
resp = requests.Response()
resp.status_code = 403
return resp
Older Answer
I went with gold_cy's comment and wrote an additional matcher that takes the request and checks for the presence of the data of interest in the payload.
#pytest.fixture(name="mock_post_bearer_token_endpoint")
def fixture_mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
def match_grant_type_in_payload(request):
if request.json().get("grant_type") == "client_credentials":
return True
resp = Response()
resp.status_code = 403
resp.raise_for_status()
requests_mock.post(
"https://api.twitter.com/oauth2/token",
request_headers={
"Authorization": f"Basic {basic_auth_string}",
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
},
json={"token_type": "bearer", "access_token": f"{bearer_token}"},
additional_matcher=match_grant_type_in_payload,
)
I opted to raise an Http403 error (instead of just returning False) in order to reduce the cognitive load of determining the reason exceptions are raised — returning False would lead to a requests_mock.exceptions.NoMockAddress being raised, which I don't think is descriptive enough in this case.
I still think there's a better way around this, and I'll keep searching for it.
Related
This is the code I wrote to get user balance from BingX API.
I think I do everything correct but it doesn't work properly.
import urllib.request
import json
import base64
import hmac
import time
APIURL = "https://open-api.bingx.com"
APIKEY = "MyApiKey"
SECRETKEY = "MySecretKey"
def genSignature(paramsStr):
return hmac.new(SECRETKEY.encode("utf-8"),
paramsStr.encode("utf-8"), digestmod="sha256").digest()
def post(url, body):
req = urllib.request.Request(url, headers={
'User-Agent': 'Mozilla/5.0',
'X-BX-APIKEY': APIKEY,
}, method="GET")
return urllib.request.urlopen(req).read()
def getBalance():
paramsMap = {
"timestamp": int(time.time()*1000)
}
paramsStr = "&".join(["%s=%s" % (k, paramsMap[k]) for k in paramsMap])
paramsStr += "&signature=" + genSignature(paramsStr).hex()
url = "%s/openApi/swap/v2/user/balance?%s" % (APIURL, paramsStr)
return post(url, paramsStr)
def main():
print(getBalance())
if __name__ == "__main__":
main()
But when I run it I get this:
b'{"code":100001,"msg":"","success":false,"timestamp":1675069039381}'
This is the doc link
The response from the API is indicating that the request was unsuccessful and returned a code of 100001 with a success value of false. This means that there was some sort of signature authentication error in the request that was made.
The 100001 error code means that the signature authentication has failed. The signature is used to verify the authenticity of the request, so if the signature is incorrect, the request will fail.
There are a few things that could be causing the signature to fail:
Incorrect calculation of the signature: Make sure the code for generating the signature is correct and follows the requirements of the BingX API.
Incorrect encoding: Make sure the signature is properly encoded before being added to the request as a query parameter.
Incorrect secret key: Make sure the secret key used to generate the signature is correct and up-to-date.
Incorrect time stamp: Make sure the time stamp included in the request is correct and in the correct format.
You should carefully review the code and the API documentation to ensure that the signature is being generated correctly and that all required information is included in the request. If the issue persists, you may also want to reach out to the BingX API support team for additional assistance.
import requests
# Replace YOUR_API_KEY with your actual Binance API key
headers = {'X-MBX-APIKEY': 'YOUR_API_KEY'}
# Make a GET request to the Binance account endpoint
response = requests.get('https://api.binance.com/api/v3/account', headers=headers)
# Check if the request was successful
if response.status_code == 200:
# Parse the JSON response
data = response.json()
# Get the user's available balance for the specified asset
asset_balance = [balance for balance in data['balances'] if balance['asset'] == 'BTC'][0]['free']
print('User balance:', asset_balance)
else:
# Handle the error
print('Error:', response.text)
I want to test an authenticated post request on an API using Pytest. This is what I am doing so far:
def test_auth_user_can_create(self, client):
url = api_reverse('crud-simulation_api')
data = {
"project": "testproject",
....
}
response = client.post(url, json=data)
assert response.status_code == 200
This doesn't work because it gives me back a 401 (Unauthorized) instead of a 200. That makes sense since the fixture is a client and not an admin client.
Yet if I pass in admin_client instead of client it gives me a Bad Request. The data that I send should be fine though.
I also tried to pass in the headers like so (since I use JWT authorization):
token = "bigassstringwhichismytoken"
headers = {
"Authorization": "JWT " + token
}
Finally I tried to log in before which gives me a 403 (Forbidden):
def test_auth_user_can_create_simulation_api(self, client, django_user_model):
username = "Jack"
password = "password"
django_user_model.objects.create_user(username=username, password=password)
client.login(username=username, password=password)
url = api_reverse('crud-simulation_api')
data = {
"project": "testproject",
...
}
response = client.post(url, json=data)
assert response.status_code == 200
If someone could point me into the right direction that would be fantastic! Thanks a lot in advance
To provide headers for client.{request} pass them individually as keyword agruments:
client.post(url, data, HTTP_FIRST_HEADER='...', HTTP_SECOND_HEADER='...')
Although you're unlikely to collide with any reserved parameter names in post call chain, better collect all headers you need in a dictionary:
headers = {
'HTTP_FIRST_HEADER': '...',
'HTTP_SECOND_HEADER': '...',
}
And pass them to request as arbitrary number of keyword arguments:
client.post(url, data, **headers)
In this case ** arguments are treated as extra information and are automatically added as headers.
You can hit the login url with username and password and get the token.
creade a header dictionary like headers = {'Authorization': 'JWT <token>'}
and use the header when using post.
client.post(url, json=data, headers=headers)
I would suggest installing the pytest-django package. Based on its docs, the easiest answer would be just using the admin_client fixture. As admin_client has the type of django.test.Client, it can be used for both get and post requests.
def test_sth_with_auth(admin_client):
response = admin_client.get('/private')
assert response.status_code == 200
Also if you want to use a specific user, you can try sth like this:
#pytest.fixture
def my_user(django_user_model):
return django_user_model.objects.create_user(username=username, password=password)
#pytest.fixture
def logged_in_client(client, my_user):
return client.force_login(my_user)
def test_sth_with_auth(logged_in_client):
response = logged_in_client.get('/private')
assert response.status_code == 200
this part of the doc can be helpful to write your desired logged_in_client().
I have gone through number of similar posts related to firing GET requests with Basic Auth (eg: Python, HTTPS GET with basic authentication), still can't figure out the problem. I keep getting the error requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url
With the same credentials, headers tried the same in postman it works as expected. Verified that base64encoded value for the api_key, password is exactly same as the value used in postman, so I don't think its encoding or resource access permission problem.
python -V
Python 3.6.4 :: Anaconda, Inc.
Approach 1
api_key = 'some_api_key'
password = 'some_password'
headers = {'accept': 'application/json'}
url = 'https://test.access.com/this/url'
api_key_password = "%s:%s" % (api_key, password)
b64_encoded = b64encode(bytes(api_key_password, 'utf-8')).decode("ascii")
headers['authorization'] = 'Basic %s' % b64_encoded
response = requests.get(url,
headers=headers)
if (response.ok):
json_data = json.loads(response.content)
print (json_data)
else:
print (response)
response.raise_for_status()
Approach 2
api_key = 'some_api_key'
password = 'some_password'
url = 'https://test.access.com/this/url'
headers = {
'accept': 'application/json',
}
response = requests.get(url, headers=headers, auth=(api_key, password))
print (response.ok)
if (response.ok):
json_data = json.loads(response.content)
print (json_data)
else:
print (response)
response.raise_for_status()
Can you please provide some pointers?
I had a similar issue (although in .NET Framework).
In my case the reason was that I was using the url without a forward slash in the end and the API apparently does not support that.
So https://test.access.com/this/url
Throws 401 error Unauthorized
but
https://test.access.com/this/url/
Returns 200 OK.
Older post but I had a similar issue. Postman will cache your JSESSIONID. Be sure you are clearing out that cookie while testing. If you are hitting an API that requires a login API call to establish a session before you can make subsequent API calls, this Postman behavior can produce a false sense of security.
In this situation with Python requests, it can be handled with code similar to what I've provided below:
import requests,json
loginAPI = "https://myapi.myco.comv/someuri/someuri/users/login"
someHTTPGetAPI = "https://myapi.myco.com/someuri/someuri/someservice"
username = "myuser"
password = "mypass"
headers = {
"Content-Type": "application/json",
"login": username,
"password": password
}
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
verify=False
session = requests.Session()
sessionResponse = session.get(url=loginURL,headers=headers, verify=verify)
if sessionResponse.status_code == 200:
getResponse = session.get(url=someHTTPGetAPI)
if getResponse.status_code == 200:
responseJSON = agentStatus.json()
I'm building a website + backend with the FLask Framework in which I use Flask-OAuthlib to authenticate with google. After authentication, the backend needs to regularly scan the user his Gmail. So currently users can authenticate my app and I store the access_token and the refresh_token. The access_token expires after one hour, so within that one hour I can get the userinfo like so:
google = oauthManager.remote_app(
'google',
consumer_key='xxxxxxxxx.apps.googleusercontent.com',
consumer_secret='xxxxxxxxx',
request_token_params={
'scope': ['https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/gmail.readonly'],
'access_type': 'offline'
},
base_url='https://www.googleapis.com/oauth2/v1/',
request_token_url=None,
access_token_method='POST',
access_token_url='https://accounts.google.com/o/oauth2/token',
authorize_url='https://accounts.google.com/o/oauth2/auth'
)
token = (the_stored_access_token, '')
userinfoObj = google.get('userinfo', token=token).data
userinfoObj['id'] # Prints out my google id
Once the hour is over, I need to use the refresh_token (which I've got stored in my database) to request a new access_token. I tried replacing the_stored_access_token with the_stored_refresh_token, but this simply gives me an Invalid Credentials-error.
In this github issue I read the following:
regardless of how you obtained the access token / refresh token (whether through an authorization code grant or resource owner password credentials), you exchange them the same way, by passing the refresh token as refresh_token and grant_type set to 'refresh_token'.
From this I understood I had to create a remote app like so:
google = oauthManager.remote_app(
'google',
# also the consumer_key, secret, request_token_params, etc..
grant_type='refresh_token',
refresh_token=u'1/xK_ZIeFn9quwvk4t5VRtE2oYe5yxkRDbP9BQ99NcJT0'
)
But this leads to a TypeError: __init__() got an unexpected keyword argument 'refresh_token'. So from here I'm kinda lost.
Does anybody know how I can use the refresh_token to get a new access_token? All tips are welcome!
This is how I get a new access_token for google:
from urllib2 import Request, urlopen, URLError
from webapp2_extras import json
import mimetools
BOUNDARY = mimetools.choose_boundary()
def refresh_token()
url = google_config['access_token_url']
headers = [
("grant_type", "refresh_token"),
("client_id", <client_id>),
("client_secret", <client_secret>),
("refresh_token", <refresh_token>),
]
files = []
edata = EncodeMultiPart(headers, files, file_type='text/plain')
headers = {}
request = Request(url, headers=headers)
request.add_data(edata)
request.add_header('Content-Length', str(len(edata)))
request.add_header('Content-Type', 'multipart/form-data;boundary=%s' % BOUNDARY)
try:
response = urlopen(request).read()
response = json.decode(response)
except URLError, e:
...
EncodeMultipart function is taken from here:
https://developers.google.com/cloud-print/docs/pythonCode
Be sure to use the same BOUNDARY
Looking at the source code for OAuthRemoteApp. The constructor does not take a keyword argument called refresh_token. It does however take an argument called access_token_params which is an optional dictionary of parameters to forward to the access token url.
Since the url is the same, but the grant type is different. I imagine a call like this should work:
google = oauthManager.remote_app(
'google',
# also the consumer_key, secret, request_token_params, etc..
grant_type='refresh_token',
access_token_params = {
refresh_token=u'1/xK_ZIeFn9quwvk4t5VRtE2oYe5yxkRDbP9BQ99NcJT0'
}
)
flask-oauthlib.contrib contains an parameter named auto_refresh_url / refresh_token_url in the remote_app which does exactely what you wanted to wanted to do. An example how to use it looks like this:
app= oauth.remote_app(
[...]
refresh_token_url='https://www.douban.com/service/auth2/token',
authorization_url='https://www.douban.com/service/auth2/auth',
[...]
)
However I did not manage to get it running this way. Nevertheless this is possible without the contrib package. My solution was to catch 401 API calls and redirect to a refresh page if a refresh_token is available.
My code for the refresh endpoint looks as follows:
#app.route('/refresh/')
def refresh():
data = {}
data['grant_type'] = 'refresh_token'
data['refresh_token'] = session['refresh_token'][0]
data['client_id'] = CLIENT_ID
data['client_secret'] = CLIENT_SECRET
# make custom POST request to get the new token pair
resp = remote.post(remote.access_token_url, data=data)
# checks the response status and parses the new tokens
# if refresh failed will redirect to login
parse_authorized_response(resp)
return redirect('/')
def parse_authorized_response(resp):
if resp is None:
return 'Access denied: reason=%s error=%s' % (
request.args['error_reason'],
request.args['error_description']
)
if isinstance(resp, dict):
session['access_token'] = (resp['access_token'], '')
session['refresh_token'] = (resp['refresh_token'], '')
elif isinstance(resp, OAuthResponse):
print(resp.status)
if resp.status != 200:
session['access_token'] = None
session['refresh_token'] = None
return redirect(url_for('login'))
else:
session['access_token'] = (resp.data['access_token'], '')
session['refresh_token'] = (resp.data['refresh_token'], '')
else:
raise Exception()
return redirect('/')
Hope this will help. The code can be enhanced of course and there surely is a more elegant way than catching 401ers but it's a start ;)
One other thing: Do not store the tokens in the Flask Session Cookie. Rather use Server Side Sessions from "Flask Session" which I did in my code!
This is how i got my new access token.
from urllib2 import Request, urlopen, URLError
import json
import mimetools
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
def EncodeMultiPart(fields, files, file_type='application/xml'):
"""Encodes list of parameters and files for HTTP multipart format.
Args:
fields: list of tuples containing name and value of parameters.
files: list of tuples containing param name, filename, and file contents.
file_type: string if file type different than application/xml.
Returns:
A string to be sent as data for the HTTP post request.
"""
lines = []
for (key, value) in fields:
lines.append('--' + BOUNDARY)
lines.append('Content-Disposition: form-data; name="%s"' % key)
lines.append('') # blank line
lines.append(value)
for (key, filename, value) in files:
lines.append('--' + BOUNDARY)
lines.append(
'Content-Disposition: form-data; name="%s"; filename="%s"'
% (key, filename))
lines.append('Content-Type: %s' % file_type)
lines.append('') # blank line
lines.append(value)
lines.append('--' + BOUNDARY + '--')
lines.append('') # blank line
return CRLF.join(lines)
def refresh_token():
url = "https://oauth2.googleapis.com/token"
headers = [
("grant_type", "refresh_token"),
("client_id", "xxxxxx"),
("client_secret", "xxxxxx"),
("refresh_token", "xxxxx"),
]
files = []
edata = EncodeMultiPart(headers, files, file_type='text/plain')
#print(EncodeMultiPart(headers, files, file_type='text/plain'))
headers = {}
request = Request(url, headers=headers)
request.add_data(edata)
request.add_header('Content-Length', str(len(edata)))
request.add_header('Content-Type', 'multipart/form-data;boundary=%s' % BOUNDARY)
response = urlopen(request).read()
print(response)
refresh_token()
#response = json.decode(response)
#print(refresh_token())
With your refresh_token, you can get a new access_token like:
from google.oauth2.credentials import Credentials
from google.auth.transport import requests
creds = {"refresh_token": "<goes here>",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"client_id": "<YOUR_CLIENT_ID>.apps.googleusercontent.com",
"client_secret": "<goes here>",
"scopes": ["https://www.googleapis.com/auth/userinfo.email"]}
cred = Credentials.from_authorized_user_info(creds)
cred.refresh(requests.Request())
my_new_access_token = cred.token
What I want to do is GET from a site and if that request returns a 401, then redo my authentication wiggle (which may be out of date) and try again. But I don't want to try a third time, since that would be my authentication wiggle having the wrong credentials. Does anyone have a nice way of doing this that doesn't involve properly ugly code, ideally in python requests library, but I don't mind changing.
It doesn't get any less ugly than this, I think:
import requests
from requests.auth import HTTPBasicAuth
response = requests.get('http://your_url')
if response.status_code == 401:
response = requests.get('http://your_url', auth=HTTPBasicAuth('user', 'pass'))
if response.status_code != 200:
# Definitely something's wrong
You could have wrapped this in a function and used a decorator to evaluate the response and retry the auth on 401. Then you only need to decorate any function that requires this re-auth logic....
Update:
As requested, a code example. I'm afraid this one is an old piece of code, Python 2 based, but you'll get the idea. This one will retry an http call a number of times as defined in settings.NUM_PLATFORM_RETRIES and will call a refresh_token on auth failures. you can adjust the use case and result to whatever.
You can then use this decorator around methods:
#retry_on_read_error
def some_func():
do_something()
def retry_on_read_error(fn):
"""
Retry Feed reads on failures
If a token refresh is required it is performed before retry.
This decorator relies on the model to have a refresh_token method defined, othewise it will fail
"""
#wraps(fn)
def _wrapper(self, *args, **kwargs):
for i in range(settings.NUM_PLATFORM_RETRIES):
try:
res = fn(self, *args, **kwargs)
try:
_res = json.loads(res)
except ValueError:
# not a json response (could be local file read or non json data)
return res
if 'error' in _res and _res['error']['status'] in (401, 400):
raise AccessRefusedException(_res['error']['message'])
return res
except (urllib2.URLError, IOError, AccessRefusedException) as e:
if isinstance(e, AccessRefusedException):
self.refresh_token()
continue
raise ApiRequestFailed(
"Api failing, after %s retries: %s" % (settings.NUM_PLATFORM_RETRIES, e), args, kwargs
)
return _wrapper
You can use something like this
# 401 retry strategy
import requests
from requests import Request, Session, RequestException
class PreparedRequest:
"""
Class to make Http request with 401 retry
"""
failedRequests = []
defaultBaseUrl = "https://jsonplaceholder.typicode.com"
MAX_RETRY_COUNT = 0
def __init__(self, method, endpoint,
baseurl=defaultBaseUrl, headers=None, data=None, params=None):
"""
Constructor for PreparedRequest class
#param method: Http Request Method
#param endpoint: endpoint of the request
#param headers: headers of the request
#param data: data of request
#param params: params of the request
"""
self.method = method
self.url = baseurl + endpoint
self.headers = headers
self.data = data
self.params = params
self.response = None
def send(self):
"""
To send http request to the server
#return: response of the request
"""
req = Request(method=self.method, url=self.url, data=self.data,
headers=self.headers,params=self.params)
session = Session()
prepared = session.prepare_request(req)
response = session.send(prepared)
if response.status_code == 200:
PreparedRequest.failedRequests.append(self)
PreparedRequest.refresh_token()
elif response.status_code == 502:
raise Exception(response.raise_for_status())
else:
self.response = session.send(prepared)
#staticmethod
def refresh_token():
if PreparedRequest.MAX_RETRY_COUNT > 3:
return
print("Refreshing the token")
# Write your refresh token strategy here
PreparedRequest.MAX_RETRY_COUNT += 1
total_failed = len(PreparedRequest.failedRequests)
for i in range(total_failed):
item = PreparedRequest.failedRequests.pop()
item.send()
r = PreparedRequest(method="GET", endpoint="/todos/")
r.send()
print(r.response.json())
You need to send in the header of the request the authentication param
import requests
from requests.auth import HTTPBasicAuth
auth = HTTPBasicAuth("username", "password")
response = requests.get("http://serverIpOrName/html", auth=auth)
if response.status_code == 401 :
print("Authentication required")
if response.status_code == 200:
print(response.content)