I am trying access a REST API.
I can get it working in Curl/REST Client (the UI tool), with preemptive authentication enabled.
But, using urllib2, it doesn't seem to support this by default and I can't find a way to turn it on.
Thanks :)
Here's a simple Preemptive HTTP basic auth handler, based on the code from urllib2.HTTPBasicAuthHandler. It can be used in the exact same manner, except an Authorization header will be added to every request with a matching URL. Note that this handler should be used with a HTTPPasswordMgrWithDefaultRealm. That's because there is no realm coming back in a WWW-Authenticate challenge since you're being preemptive.
class PreemptiveBasicAuthHandler(urllib2.BaseHandler):
def __init__(self, password_mgr=None):
if password_mgr is None:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
self.passwd = password_mgr
self.add_password = self.passwd.add_password
def http_request(self,req):
uri = req.get_full_url()
user, pw = self.passwd.find_user_password(None,uri)
#logging.debug('ADDING REQUEST HEADER for uri (%s): %s:%s',uri,user,pw)
if pw is None: return req
raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip()
req.add_unredirected_header('Authorization', auth)
return req
similar to #thom-nichols's answer; but subclassing HTTPBasicAuthHandler also handling HTTPS requests.
import urllib2
import base64
class PreemptiveBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
'''Preemptive basic auth.
Instead of waiting for a 403 to then retry with the credentials,
send the credentials if the url is handled by the password manager.
Note: please use realm=None when calling add_password.'''
def http_request(self, req):
url = req.get_full_url()
realm = None
# this is very similar to the code from retry_http_basic_auth()
# but returns a request object.
user, pw = self.passwd.find_user_password(realm, url)
if pw:
raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip()
req.add_unredirected_header(self.auth_header, auth)
return req
https_request = http_request
here is an example for dealing with a jenkins server which does not send you 401 http errors (retry with auth). I'm using urllib2.install_opener to make things easy.
jenkins_url = "https://jenkins.example.com"
username = "johndoe"
api_token = "some-cryptic-value"
auth_handler = PreemptiveBasicAuthHandler()
auth_handler.add_password(
realm=None, # default realm.
uri=jenkins_url,
user=username,
passwd=api_token)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
Depending on what kind of authentication is required, you can send the Authorization headers manually by adding them to your request before you send out a body.
Related
I'm trying to log into the vRealize Operations Rest API using basic auth method from this question:
HTTP Basic Authentication not working in Python 3.4
So I use the second code sample:
import urllib.parse
import urllib.request
import urllib.response
userName = "username"
passWord = "password"
top_level_url = "URL"
# create an authorization handler
p = urllib.request.HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, top_level_url, userName, passWord);
auth_handler = urllib.request.HTTPBasicAuthHandler(p)
opener = urllib.request.build_opener(auth_handler)
urllib.request.install_opener(opener)
try:
result = opener.open(top_level_url)
messages = result.read()
print (messages)
except IOError as e:
print (e)
and I get a blue font: HTTP Error 401: Unauthorized
So I think this is an issue with trusting the certificate or it has something to do with the headers. How can I work around this? Any advice would be appreciated.
I'm trying to issue a basic UbuntuOne API call.
As explained on https://one.ubuntu.com/developer/account_admin/auth/otherplatforms, I'm getting the OAUTH token and then passing it to the UbuntuOne service.
I get the token and consumer info alright
I'm then trying to issue a /api/file_storage/v1 API call (see: https://one.ubuntu.com/developer/files/store_files/cloud.) The request is signed using the OAUTH token.
The code snippet below is the exact code I'm executing (minus the email.password/description fields.) The token and consumer data is returned properly. I'm getting a '401 UNAUTHORIZED' from the server when issuing the /api/file_storage/v1 request... any idea why?
import base64
import json
import urllib
import urllib2
import oauth2
email = 'bla'
password = 'foo'
description = 'bar'
class Unauthorized(Exception):
"""The provided email address and password were incorrect."""
def acquire_token(email_address, password, description):
"""Aquire an OAuth access token for the given user."""
# Issue a new access token for the user.
request = urllib2.Request(
'https://login.ubuntu.com/api/1.0/authentications?' +
urllib.urlencode({'ws.op': 'authenticate', 'token_name': description}))
request.add_header('Accept', 'application/json')
request.add_header('Authorization', 'Basic %s' % base64.b64encode('%s:%s' % (email_address, password)))
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError, exc:
if exc.code == 401: # Unauthorized
raise Unauthorized("Bad email address or password")
else:
raise
data = json.load(response)
consumer = oauth2.Consumer(data['consumer_key'], data['consumer_secret'])
token = oauth2.Token(data['token'], data['token_secret'])
# Tell Ubuntu One about the new token.
get_tokens_url = ('https://one.ubuntu.com/oauth/sso-finished-so-get-tokens/')
oauth_request = oauth2.Request.from_consumer_and_token(consumer, token, 'GET', get_tokens_url)
oauth_request.sign_request(oauth2.SignatureMethod_PLAINTEXT(), consumer, token)
request = urllib2.Request(get_tokens_url)
for header, value in oauth_request.to_header().items():
request.add_header(header, value)
response = urllib2.urlopen(request)
return consumer, token
if __name__ == '__main__':
consumer, token = acquire_token(email, password, description)
print 'Consumer:', consumer
print 'Token:', token
url = 'https://one.ubuntu.com/api/file_storage/v1'
oauth_request = oauth2.Request.from_consumer_and_token(consumer, token, 'GET', url)
oauth_request.sign_request(oauth2.SignatureMethod_PLAINTEXT(), consumer, token)
request = urllib2.Request(url)
request.add_header('Accept', 'application/json')
for header, value in oauth_request.to_header().items():
request.add_header(header, value)
response = urllib2.urlopen(request)
The issue was with the 'description' field. It must be in the following format:
Ubuntu One # $hostname [$application]
Else, the UbuntuOne service returns a "ok 0/1" and does not register the token.
I was using Mechanize module a while ago, and now try to use Requests module.
(Python mechanize doesn't work when HTTPS and Proxy Authentication required)
I have to go through proxy-server when I access the Internet.
The proxy-server requires authentication. I wrote the following codes.
import requests
from requests.auth import HTTPProxyAuth
proxies = {"http":"192.168.20.130:8080"}
auth = HTTPProxyAuth("username", "password")
r = requests.get("http://www.google.co.jp/", proxies=proxies, auth=auth)
The above codes work well when proxy-server requires basic authentication.
Now I want to know what I have to do when proxy-server requires digest authentication.
HTTPProxyAuth seems not to be effective in digest authentication (r.status_code returns 407).
No need to implement your own! in most cases
Requests has built in support for proxies, for basic authentication:
proxies = { 'https' : 'https://user:password#proxyip:port' }
r = requests.get('https://url', proxies=proxies)
see more on the docs
Or in case you need digest authentication HTTPDigestAuth may help.
Or you might need try to extend it like yutaka2487 did bellow.
Note: must use ip of proxy server not its name!
I wrote the class that can be used in proxy authentication (based on digest auth).
I borrowed almost all codes from requests.auth.HTTPDigestAuth.
import requests
import requests.auth
class HTTPProxyDigestAuth(requests.auth.HTTPDigestAuth):
def handle_407(self, r):
"""Takes the given response and tries digest-auth, if needed."""
num_407_calls = r.request.hooks['response'].count(self.handle_407)
s_auth = r.headers.get('Proxy-authenticate', '')
if 'digest' in s_auth.lower() and num_407_calls < 2:
self.chal = requests.auth.parse_dict_header(s_auth.replace('Digest ', ''))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.raw.release_conn()
r.request.headers['Authorization'] = self.build_digest_header(r.request.method, r.request.url)
r.request.send(anyway=True)
_r = r.request.response
_r.history.append(r)
return _r
return r
def __call__(self, r):
if self.last_nonce:
r.headers['Proxy-Authorization'] = self.build_digest_header(r.method, r.url)
r.register_hook('response', self.handle_407)
return r
Usage:
proxies = {
"http" :"192.168.20.130:8080",
"https":"192.168.20.130:8080",
}
auth = HTTPProxyDigestAuth("username", "password")
# HTTP
r = requests.get("http://www.google.co.jp/", proxies=proxies, auth=auth)
r.status_code # 200 OK
# HTTPS
r = requests.get("https://www.google.co.jp/", proxies=proxies, auth=auth)
r.status_code # 200 OK
I've written a Python module (available here) which makes it possible to authenticate with a HTTP proxy using the digest scheme. It works when connecting to HTTPS websites (through monkey patching) and allows to authenticate with the website as well. This should work with latest requests library for both Python 2 and 3.
The following example fetches the webpage https://httpbin.org/ip through HTTP proxy 1.2.3.4:8080 which requires HTTP digest authentication using user name user1 and password password1:
import requests
from requests_digest_proxy import HTTPProxyDigestAuth
s = requests.Session()
s.proxies = {
'http': 'http://1.2.3.4:8080/',
'https': 'http://1.2.3.4:8080/'
}
s.auth = HTTPProxyDigestAuth('user1', 'password1')
print(s.get('https://httpbin.org/ip').text)
Should the website requires some kind of HTTP authentication, this can be specified to HTTPProxyDigestAuth constructor this way:
# HTTP Basic authentication for website
s.auth = HTTPProxyDigestAuth(('user1', 'password1'),
auth=requests.auth.HTTPBasicAuth('user1', 'password0'))
print(s.get('https://httpbin.org/basic-auth/user1/password0').text))
# HTTP Digest authentication for website
s.auth = HTTPProxyDigestAuth(('user1', 'password1'),,
auth=requests.auth.HTTPDigestAuth('user1', 'password0'))
print(s.get('https://httpbin.org/digest-auth/auth/user1/password0').text)
This snippet works for both types of requests (http and https). Tested on the current version of requests (2.23.0).
import re
import requests
from requests.utils import get_auth_from_url
from requests.auth import HTTPDigestAuth
from requests.utils import parse_dict_header
from urllib3.util import parse_url
def get_proxy_autorization_header(proxy, method):
username, password = get_auth_from_url(proxy)
auth = HTTPProxyDigestAuth(username, password)
proxy_url = parse_url(proxy)
proxy_response = requests.request(method, proxy_url, auth=auth)
return proxy_response.request.headers['Proxy-Authorization']
class HTTPSAdapterWithProxyDigestAuth(requests.adapters.HTTPAdapter):
def proxy_headers(self, proxy):
headers = {}
proxy_auth_header = get_proxy_autorization_header(proxy, 'CONNECT')
headers['Proxy-Authorization'] = proxy_auth_header
return headers
class HTTPAdapterWithProxyDigestAuth(requests.adapters.HTTPAdapter):
def proxy_headers(self, proxy):
return {}
def add_headers(self, request, **kwargs):
proxy = kwargs['proxies'].get('http', '')
if proxy:
proxy_auth_header = get_proxy_autorization_header(proxy, request.method)
request.headers['Proxy-Authorization'] = proxy_auth_header
class HTTPProxyDigestAuth(requests.auth.HTTPDigestAuth):
def init_per_thread_state(self):
# Ensure state is initialized just once per-thread
if not hasattr(self._thread_local, 'init'):
self._thread_local.init = True
self._thread_local.last_nonce = ''
self._thread_local.nonce_count = 0
self._thread_local.chal = {}
self._thread_local.pos = None
self._thread_local.num_407_calls = None
def handle_407(self, r, **kwargs):
"""
Takes the given response and tries digest-auth, if needed.
:rtype: requests.Response
"""
# If response is not 407, do not auth
if r.status_code != 407:
self._thread_local.num_407_calls = 1
return r
s_auth = r.headers.get('proxy-authenticate', '')
if 'digest' in s_auth.lower() and self._thread_local.num_407_calls < 2:
self._thread_local.num_407_calls += 1
pat = re.compile(r'digest ', flags=re.IGNORECASE)
self._thread_local.chal = requests.utils.parse_dict_header(
pat.sub('', s_auth, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
r.content
r.close()
prep = r.request.copy()
requests.cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Proxy-Authorization'] = self.build_digest_header(prep.method, prep.url)
_r = r.connection.send(prep, **kwargs)
_r.history.append(r)
_r.request = prep
return _r
self._thread_local.num_407_calls = 1
return r
def __call__(self, r):
# Initialize per-thread state, if needed
self.init_per_thread_state()
# If we have a saved nonce, skip the 407
if self._thread_local.last_nonce:
r.headers['Proxy-Authorization'] = self.build_digest_header(r.method, r.url)
r.register_hook('response', self.handle_407)
self._thread_local.num_407_calls = 1
return r
session = requests.Session()
session.proxies = {
'http': 'http://username:password#proxyhost:proxyport',
'https': 'http://username:password#proxyhost:proxyport'
}
session.trust_env = False
session.mount('http://', HTTPAdapterWithProxyDigestAuth())
session.mount('https://', HTTPSAdapterWithProxyDigestAuth())
response_http = session.get("http://ww3.safestyle-windows.co.uk/the-secret-door/")
print(response_http.status_code)
response_https = session.get("https://stackoverflow.com/questions/13506455/how-to-pass-proxy-authentication-requires-digest-auth-by-using-python-requests")
print(response_https.status_code)
Generally, the problem of proxy autorization is also relevant for other types of authentication (ntlm, kerberos) when connecting using the protocol HTTPS. And despite the large number of issues (since 2013, and maybe there are earlier ones that I did not find):
in requests: Digest Proxy Auth, NTLM Proxy Auth, Kerberos Proxy Auth
in urlib3: NTLM Proxy Auth, NTLM Proxy Auth
and many many others,the problem is still not resolved.
The root of the problem in the function _tunnel of the module httplib(python2)/http.client(python3). In case of unsuccessful connection attempt, it raises an OSError without returning a response code (407 in our case) and additional data needed to build the autorization header. Lukasa gave a explanation here.
As long as there is no solution from maintainers of urllib3 (or requests), we can only use various workarounds (for example, use the approach of #Tey' or do something like this).In my version of workaround, we pre-prepare the necessary authorization data by sending a request to the proxy server and processing the received response.
You can use digest authentication by using requests.auth.HTTPDigestAuth instead of requests.auth.HTTPProxyAuth
For those of you that still end up here, there appears to be a project called requests-toolbelt that has this plus other common but not built in functionality of requests.
https://toolbelt.readthedocs.org/en/latest/authentication.html#httpproxydigestauth
This works for me. Actually, don't know about security of user:password in this soulution:
import requests
import os
http_proxyf = 'http://user:password#proxyip:port'
os.environ["http_proxy"] = http_proxyf
os.environ["https_proxy"] = http_proxyf
sess = requests.Session()
# maybe need sess.trust_env = True
print(sess.get('https://some.org').text)
import requests
import os
# in my case I had to add my local domain
proxies = {
'http': 'proxy.myagency.com:8080',
'https': 'user#localdomain:password#proxy.myagency.com:8080',
}
r=requests.get('https://api.github.com/events', proxies=proxies)
print(r.text)
Here is an answer that is not for http Basic Authentication - for example a transperant proxy within organization.
import requests
url = 'https://someaddress-behindproxy.com'
params = {'apikey': '123456789'} #if you need params
proxies = {'https': 'https://proxyaddress.com:3128'} #or some other port
response = requests.get(url, proxies=proxies, params=params)
I hope this helps someone.
I am using Web2Py to create a simple app which sends Push notifications through UrbanAirship. For some reason, I am getting a 400 response when I try to send it through my code. It UA API works fine using REST client. This is my code:
url = 'https://go.urbanairship.com/api/push/'
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
# this creates a password manager
passman.add_password(None, url, username, password)
# because we have put None at the start it will always
# use this username/password combination for urls
# for which `theurl` is a super-url
authhandler = urllib2.HTTPBasicAuthHandler(passman)
# create the AuthHandler
opener = urllib2.build_opener(authhandler)
urllib2.install_opener(opener)
# All calls to urllib2.urlopen will now use our handler
# Make sure not to include the protocol in with the URL, or
# HTTPPasswordMgrWithDefaultRealm will be very confused.
# You must (of course) use it when fetching the page though.
values = {"device_tokens": ["<DEVICE TOKEN>"], "aps": {"alert": "Hello!"}}
data = urllib.urlencode(values)
headers = {'Content-Type': 'application/json'}
req = urllib2.Request(url, data, headers)
try:
response = urllib2.urlopen(req)
return response
except IOError, e:
if e.code == 200:
return "Push sent!"
else:
return 'The server couldn\'t fulfill the request. Error: %d' % e.code
As far as I can understand, the problem is in the format of data being sent. Where am I going wrong?
The urllib.urlencode function is for making a URL-encoded parameter body (Content-Type: application/x-www-form-urlencoded). For JSON, which is apparently what you want, use json.dumps instead.
Update: based on Lee's comment I decided to condense my code to a really simple script and run it from the command line:
import urllib2
import sys
username = sys.argv[1]
password = sys.argv[2]
url = sys.argv[3]
print("calling %s with %s:%s\n" % (url, username, password))
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, username, password)
urllib2.install_opener(urllib2.build_opener(urllib2.HTTPBasicAuthHandler(passman)))
req = urllib2.Request(url)
f = urllib2.urlopen(req)
data = f.read()
print(data)
Unfortunately it still won't generate the Authorization header (per Wireshark) :(
I'm having a problem sending basic AUTH over urllib2. I took a look at this article, and followed the example. My code:
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, "api.foursquare.com", username, password)
urllib2.install_opener(urllib2.build_opener(urllib2.HTTPBasicAuthHandler(passman)))
req = urllib2.Request("http://api.foursquare.com/v1/user")
f = urllib2.urlopen(req)
data = f.read()
I'm seeing the following on the Wire via wireshark:
GET /v1/user HTTP/1.1
Host: api.foursquare.com
Connection: close
Accept-Encoding: gzip
User-Agent: Python-urllib/2.5
You can see the Authorization is not sent, vs. when I send a request via curl: curl -u user:password http://api.foursquare.com/v1/user
GET /v1/user HTTP/1.1
Authorization: Basic =SNIP=
User-Agent: curl/7.19.4 (universal-apple-darwin10.0) libcurl/7.19.4 OpenSSL/0.9.8k zlib/1.2.3
Host: api.foursquare.com
Accept: */*
For some reason my code seems to not send the authentication - anyone see what I'm missing?
thanks
-simon
The problem could be that the Python libraries, per HTTP-Standard, first send an unauthenticated request, and then only if it's answered with a 401 retry, are the correct credentials sent. If the Foursquare servers don't do "totally standard authentication" then the libraries won't work.
Try using headers to do authentication:
import urllib2, base64
request = urllib2.Request("http://api.foursquare.com/v1/user")
base64string = base64.b64encode('%s:%s' % (username, password))
request.add_header("Authorization", "Basic %s" % base64string)
result = urllib2.urlopen(request)
Had the same problem as you and found the solution from this thread: http://forums.shopify.com/categories/9/posts/27662
(copy-paste/adapted from https://stackoverflow.com/a/24048772/1733117).
First you can subclass urllib2.BaseHandler or urllib2.HTTPBasicAuthHandler, and implement http_request so that each request has the appropriate Authorization header.
import urllib2
import base64
class PreemptiveBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
'''Preemptive basic auth.
Instead of waiting for a 403 to then retry with the credentials,
send the credentials if the url is handled by the password manager.
Note: please use realm=None when calling add_password.'''
def http_request(self, req):
url = req.get_full_url()
realm = None
# this is very similar to the code from retry_http_basic_auth()
# but returns a request object.
user, pw = self.passwd.find_user_password(realm, url)
if pw:
raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip()
req.add_unredirected_header(self.auth_header, auth)
return req
https_request = http_request
Then if you are lazy like me, install the handler globally
api_url = "http://api.foursquare.com/"
api_username = "johndoe"
api_password = "some-cryptic-value"
auth_handler = PreemptiveBasicAuthHandler()
auth_handler.add_password(
realm=None, # default realm.
uri=api_url,
user=api_username,
passwd=api_password)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
Here's what I'm using to deal with a similar problem I encountered while trying to access MailChimp's API. This does the same thing, just formatted nicer.
import urllib2
import base64
chimpConfig = {
"headers" : {
"Content-Type": "application/json",
"Authorization": "Basic " + base64.encodestring("hayden:MYSECRETAPIKEY").replace('\n', '')
},
"url": 'https://us12.api.mailchimp.com/3.0/'}
#perform authentication
datas = None
request = urllib2.Request(chimpConfig["url"], datas, chimpConfig["headers"])
result = urllib2.urlopen(request)
The second parameter must be a URI, not a domain name. i.e.
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, "http://api.foursquare.com/", username, password)
I would suggest that the current solution is to use my package urllib2_prior_auth which solves this pretty nicely (I work on inclusion to the standard lib.