What I want to do is GET from a site and if that request returns a 401, then redo my authentication wiggle (which may be out of date) and try again. But I don't want to try a third time, since that would be my authentication wiggle having the wrong credentials. Does anyone have a nice way of doing this that doesn't involve properly ugly code, ideally in python requests library, but I don't mind changing.
It doesn't get any less ugly than this, I think:
import requests
from requests.auth import HTTPBasicAuth
response = requests.get('http://your_url')
if response.status_code == 401:
response = requests.get('http://your_url', auth=HTTPBasicAuth('user', 'pass'))
if response.status_code != 200:
# Definitely something's wrong
You could have wrapped this in a function and used a decorator to evaluate the response and retry the auth on 401. Then you only need to decorate any function that requires this re-auth logic....
Update:
As requested, a code example. I'm afraid this one is an old piece of code, Python 2 based, but you'll get the idea. This one will retry an http call a number of times as defined in settings.NUM_PLATFORM_RETRIES and will call a refresh_token on auth failures. you can adjust the use case and result to whatever.
You can then use this decorator around methods:
#retry_on_read_error
def some_func():
do_something()
def retry_on_read_error(fn):
"""
Retry Feed reads on failures
If a token refresh is required it is performed before retry.
This decorator relies on the model to have a refresh_token method defined, othewise it will fail
"""
#wraps(fn)
def _wrapper(self, *args, **kwargs):
for i in range(settings.NUM_PLATFORM_RETRIES):
try:
res = fn(self, *args, **kwargs)
try:
_res = json.loads(res)
except ValueError:
# not a json response (could be local file read or non json data)
return res
if 'error' in _res and _res['error']['status'] in (401, 400):
raise AccessRefusedException(_res['error']['message'])
return res
except (urllib2.URLError, IOError, AccessRefusedException) as e:
if isinstance(e, AccessRefusedException):
self.refresh_token()
continue
raise ApiRequestFailed(
"Api failing, after %s retries: %s" % (settings.NUM_PLATFORM_RETRIES, e), args, kwargs
)
return _wrapper
You can use something like this
# 401 retry strategy
import requests
from requests import Request, Session, RequestException
class PreparedRequest:
"""
Class to make Http request with 401 retry
"""
failedRequests = []
defaultBaseUrl = "https://jsonplaceholder.typicode.com"
MAX_RETRY_COUNT = 0
def __init__(self, method, endpoint,
baseurl=defaultBaseUrl, headers=None, data=None, params=None):
"""
Constructor for PreparedRequest class
#param method: Http Request Method
#param endpoint: endpoint of the request
#param headers: headers of the request
#param data: data of request
#param params: params of the request
"""
self.method = method
self.url = baseurl + endpoint
self.headers = headers
self.data = data
self.params = params
self.response = None
def send(self):
"""
To send http request to the server
#return: response of the request
"""
req = Request(method=self.method, url=self.url, data=self.data,
headers=self.headers,params=self.params)
session = Session()
prepared = session.prepare_request(req)
response = session.send(prepared)
if response.status_code == 200:
PreparedRequest.failedRequests.append(self)
PreparedRequest.refresh_token()
elif response.status_code == 502:
raise Exception(response.raise_for_status())
else:
self.response = session.send(prepared)
#staticmethod
def refresh_token():
if PreparedRequest.MAX_RETRY_COUNT > 3:
return
print("Refreshing the token")
# Write your refresh token strategy here
PreparedRequest.MAX_RETRY_COUNT += 1
total_failed = len(PreparedRequest.failedRequests)
for i in range(total_failed):
item = PreparedRequest.failedRequests.pop()
item.send()
r = PreparedRequest(method="GET", endpoint="/todos/")
r.send()
print(r.response.json())
You need to send in the header of the request the authentication param
import requests
from requests.auth import HTTPBasicAuth
auth = HTTPBasicAuth("username", "password")
response = requests.get("http://serverIpOrName/html", auth=auth)
if response.status_code == 401 :
print("Authentication required")
if response.status_code == 200:
print(response.content)
Related
What I've Done
I've written an authentication class for obtaining an application's bearer token from Twitter using the application's API Key and its API key secret as demonstrated in the Twitter developer docs.
I've mocked the appropriate endpoint using requests_mock this way:
#pytest.fixture
def mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
requests_mock.post(
"https://api.twitter.com/oauth2/token",
request_headers={
"Authorization": f"Basic {basic_auth_string}",
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
},
json={"token_type": "bearer", "access_token": f"{bearer_token}"},
)
And my test method is :
#pytest.mark.usefixtures("mock_post_bearer_token_endpoint")
def test_basic_auth(api_key, api_key_secret, bearer_token):
response = requests.post(
'https://api.twitter.com/oauth2/token',
data={"grant_type": "client_credentials"},
auth=TwitterBasicAuth(api_key, api_key_secret),
)
assert response.json()['access_token'] == bearer_token
(Where TwitterBasicAuth is the authentication class I wrote, and the fixture basic_auth_string is a hardcoded string that would be obtained from transforming the fixtures api_key and api_key_secret appropriately).
And it works.
The Problem
But I'm really bothered by the fact that the mocked endpoint doesn't check the payload. In this particular case, the payload is vital to obtain a bearer token.
I've combed through the documentation for requests_mock (and responses, too) but haven't figured out how to make the endpoint respond with a bearer token only when the correct payload is POSTed.
Please help.
I think the misconception here is that you need to put everything in the matcher and let NoMatchException be the thing to tell you if you got it right.
The matcher can be the simplest thing it needs to be in order to return the right response and then you can do all the request/response checking as part of your normal unit test handling.
additional_matchers is useful if you need to switch the response value based on the body of the request for example, and typically true/false is sufficient there.
eg, and i made no attempt to look up twitter auth for this:
import requests
import requests_mock
class TwitterBasicAuth(requests.auth.AuthBase):
def __init__(self, api_key, api_key_secret):
self.api_key = api_key
self.api_key_secret = api_key_secret
def __call__(self, r):
r.headers['x-api-key'] = self.api_key
r.headers['x-api-key-secret'] = self.api_key_secret
return r
with requests_mock.mock() as m:
api_key = 'test'
api_key_secret = 'val'
m.post(
"https://api.twitter.com/oauth2/token",
json={"token_type": "bearer", "access_token": "token"},
)
response = requests.post(
'https://api.twitter.com/oauth2/token',
data={"grant_type": "client_credentials"},
auth=TwitterBasicAuth(api_key, api_key_secret),
)
assert response.json()['token_type'] == "bearer"
assert response.json()['access_token'] == "token"
assert m.last_request.headers['x-api-key'] == api_key
assert m.last_request.headers['x-api-key-secret'] == api_key_secret
https://requests-mock.readthedocs.io/en/latest/history.html
Updated Answer
I went with gold_cy's comment and wrote a custom matcher that takes a request and returns an appropriately crafted OK response if the request has the correct url path, headers and json payload. It returns a 403 response otherwise, as I'd expect from the Twitter API.
#pytest.fixture
def mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
def matcher(req):
if req.path != "/oauth2/token":
# no mock address
return None
if req.headers.get("Authorization") != f"Basic {basic_auth_string}":
return create_forbidden_response()
if (
req.headers.get("Content-Type")
!= "application/x-www-form-urlencoded;charset=UTF-8"
):
return create_forbidden_response()
if req.json().get("grant_type") != "client_credentials":
return create_forbidden_response()
resp = requests.Response()
resp._content = json.dumps(
{"token_type": "bearer", "access_token": f"{bearer_token}"}
).encode()
resp.status_code = 200
return resp
requests_mock._adapter.add_matcher(matcher)
yield
def create_forbidden_response():
resp = requests.Response()
resp.status_code = 403
return resp
Older Answer
I went with gold_cy's comment and wrote an additional matcher that takes the request and checks for the presence of the data of interest in the payload.
#pytest.fixture(name="mock_post_bearer_token_endpoint")
def fixture_mock_post_bearer_token_endpoint(
requests_mock, basic_auth_string, bearer_token
):
def match_grant_type_in_payload(request):
if request.json().get("grant_type") == "client_credentials":
return True
resp = Response()
resp.status_code = 403
resp.raise_for_status()
requests_mock.post(
"https://api.twitter.com/oauth2/token",
request_headers={
"Authorization": f"Basic {basic_auth_string}",
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
},
json={"token_type": "bearer", "access_token": f"{bearer_token}"},
additional_matcher=match_grant_type_in_payload,
)
I opted to raise an Http403 error (instead of just returning False) in order to reduce the cognitive load of determining the reason exceptions are raised — returning False would lead to a requests_mock.exceptions.NoMockAddress being raised, which I don't think is descriptive enough in this case.
I still think there's a better way around this, and I'll keep searching for it.
I have the below requests for GET & POST and the GET's work fine and the post shows a 200 response but when i check the external program it has not received any data from the post.
import requests
import json
class BearerAuth(requests.auth.AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, r):
r.headers["authorization"] = "Bearer " + self.token
return r
class BearerAuth2(requests.auth.AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, s):
s.headers["authorization"] = "api " + self.token
return s
headers={'content-type': 'application/json'}
response = requests.get('https://api', auth=BearerAuth('123'))
response2 = requests.get('https://api2', auth=BearerAuth('123'))
splunktok = requests.post('https://http-inputs', data={response, respons2}, headers=headers, auth=BearerAuth2('456'))
print(response.json(), response2.json())
What i want to do is get all the response body data from response & response 2 and use it in the POST to my external program. im not sure if i need to store the response as raw in variables first?
You are sending the requests.Response objects as a set:
data={response, respons2}
Assuming you intend to send the JSON responses, you can do something like:
data={**response.json(), **respons2.json()}
Ive worked this out. I added the below
after the first request i add
payload=response.json()
and then on the post i added
data=json.dumps(payload)
I would like to know how to set the useragent in all SOAP request with suds in Python, including WSDL get.
Indeed, on the following code :
Client('http://...')
The WSDL is get with the default Python useragent.
The WSDL is available on the server only for specific useragent.
Thank you
I don't know whether that's the easiest way to do it, but it is certainly possible to do using httplib2 (this trick also gives you keep-alive connections) :
from suds.transport import Transport
import httplib2, StringIO
class Httplib2Response:
pass
class Httplib2Transport(Transport):
def __init__(self, **kwargs):
Transport.__init__(self)
self.http = httplib2.Http()
def send(self, request):
url = request.url
message = request.message
headers = request.headers
headers['User-Agent']='XYZ'
response = Httplib2Response()
response.headers, response.message = self.http.request(url,
"PUT", body=message, headers=headers)
return response
def open(self, request):
response = Httplib2Response()
request.headers['User-Agent']='XYZ'
response.headers, response.message = self.http.request(request.url, "GET",
body=request.message, headers=request.headers)
return StringIO.StringIO(response.message)
And then you need to pass the transport class to the suds.client:
http = Httplib2Transport()
client = Client(url,transport=http)
You can override the u2opener method of Transport class to set your own addheaders attribute:
class HttpTransportCustomUserAgent(HttpTransport):
def __init__(self, **kwargs):
self.user_agent = kwargs.get('user_agent', 'Python-urllib/%s' % urllib2.__version__)
if 'user_agent' in kwargs:
del(kwargs['user_agent'])
HttpTransport.__init__(self, **kwargs)
def u2opener(self):
"""
Create a urllib opener.
#return: An opener.
#rtype: I{OpenerDirector}
"""
if self.urlopener is None:
result = urllib2.build_opener(*self.u2handlers())
result.addheaders = [('User-agent', self.user_agent)]
return result
else:
return self.urlopener
Now you can use this new transporter class for suds.client:
http = HttpTransportCustomUserAgent(user_agent='My custom User Agent')
client = Client(url, transport=http)
I am trying to accomplish the Oauth authentication on Identi.ca. Basically, I am trying to use the very same code that works for Twitter, but, of course, changing the urls. I am not even being able to get the request token, because of a HTTP Error 400: Bad Request, as shown bellow. I really would like to know what I am doing wrong, or what differs from the Twitter way of doing it. What I have so far is:
from oauth import oauth
import urllib2
class IdenticaOauth:
def __init__(self):
self.request_token_url = 'https://identi.ca/api/oauth/request_token'
self.access_token_url = 'https://identi.ca/api/oauth/access_token'
self.authorize_url = 'https://identi.ca/api/oauth/authorize'
self.consumer_key = '8024d4db70d9e49d22728f25b4c1458b'
self.consumer_secret = '4eb762cfe3c0a55950375dad795cf20e'
self.consumer = oauth.OAuthConsumer(self.consumer_key, self.consumer_secret)
self.signature_method = oauth.OAuthSignatureMethod_HMAC_SHA1()
def get_unauthorized_request_token(self):
oauth_request = oauth.OAuthRequest.from_consumer_and_token(self.consumer, http_url = self.request_token_url)
oauth_request.sign_request(self.signature_method, self.consumer, None)
url = oauth_request.to_url()
response = self.get(url)
token = oauth.OAuthToken.from_string(response)
return token
def get(self, url):
request = urllib2.Request(url)
# urllib2.HTTPError: HTTP Error 400: Bad Request
response = urllib2.urlopen(request)
return response.read()
identica_oauth = IdenticaOauth()
request_token = identica_oauth.get_unauthorized_request_token()
I have been trying to make an API request to Twilio using the httplib2 Http class and no matter how I try to setup the request, it doesn't send my post DATA. I know this, because I posted to a local URL and the post arguments are empty. Here is my code:
_TWILIO_URL = 'https://api.twilio.com/2010-04-01/Accounts/%s/%s'
class Api(object):
'''
A Python interface to the Twilio API
'''
def __init__(self, AccountSid, AuthToken, From):
self.AccountSid = AccountSid
self.AuthToken = AuthToken
self.From = From
def _get_from(self, From):
"""Use the provided From number or the one defined on initialization"""
if From:
return From
else:
return self.From
def call(self, To, Url, From=None):
"""Sends a request to Twilio having it call a number; the provided URL will indicate how to handle the call"""
url = _TWILIO_URL % (self.AccountSid, 'Calls')
data = dict(From=self._get_from(From), To=To, Url=Url)
return self.request(url, body=urlencode(data))
def request(self, url, method='POST', body=None, headers={'content-type':'text/plain'}):
"""Send the actual request"""
h = Http()
h.add_credentials(self.AccountSid, self.AuthToken)
resp, content = h.request(url, method=method, body=body, headers=headers)
print content
if resp['status'] == '200':
return loads(content)
else:
raise TwilioError(resp['status'], content)
def sms(self, To, Body, From=None):
"""Sends a request to Twilio having it call a number; the provided URL will indicate how to handle the call"""
url = _TWILIO_URL % (self.AccountSid, 'SMS/Messages')
data = dict(From=self._get_from(From), To=To, Body=Body)
return self.request(url, body=urlencode(data))
I can't find anything on google talking about troubleshooting
Twilio mentions this requirement in their API docs concerning POST requests:
But be sure to set the HTTP
Content-Type header to
"application/x-www-form-urlencoded"
for your requests if you are
writing your own client.
It turns out that the 'content-type' has to be set to 'application/x-www-form-urlencoded'. If anyone knows why, please let me know.