Develop Reference

requests in Python not sending data to external program

I have the below requests for GET & POST and the GET's work fine and the post shows a 200 response but when i check the external program it has not received any data from the post.
import requests
import json
class BearerAuth(requests.auth.AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, r):
r.headers["authorization"] = "Bearer " + self.token
return r
class BearerAuth2(requests.auth.AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, s):
s.headers["authorization"] = "api " + self.token
return s
headers={'content-type': 'application/json'}
response = requests.get('https://api', auth=BearerAuth('123'))
response2 = requests.get('https://api2', auth=BearerAuth('123'))
splunktok = requests.post('https://http-inputs', data={response, respons2}, headers=headers, auth=BearerAuth2('456'))
print(response.json(), response2.json())
What i want to do is get all the response body data from response & response 2 and use it in the POST to my external program. im not sure if i need to store the response as raw in variables first?

You are sending the requests.Response objects as a set:
data={response, respons2}
Assuming you intend to send the JSON responses, you can do something like:
data={**response.json(), **respons2.json()}

Ive worked this out. I added the below
after the first request i add
payload=response.json()
and then on the post i added
data=json.dumps(payload)

oauth2 POST - twitter

i created a script that will get the users friend list (GET request) and i was successful. Now i am attempting to make a script that will follow a particular user (POST request) and i've been unsuccessful.
here is my oauth function (where the problem lies):
def augment_POST(url,**kwargs) :
secrets = hidden.oauth()
consumer = oauth2.Consumer(secrets['consumer_key'], secrets['consumer_secret'])
token = oauth2.Token(secrets['token_key'],secrets['token_secret'])
oauth_request = oauth2.Request.from_consumer_and_token(consumer, token= token, http_method='POST', http_url=url, parameters=kwargs)
oauth_request.to_postdata() # this returns post data, where should i put it?
oauth_request.sign_request(oauth2.SignatureMethod_HMAC_SHA1(), consumer, token)
return oauth_request.to_url()
my augment_GET function is the exact same thing except http_mehtod='GET'
for clarity:
def follow_user(id):
seedurl="https://api.twitter.com/1.1/friendships/create.json"
print 'Attempting to follow: %d' % (id,)
url = augment_POST(seedurl,user_id=id)
connection = urllib.urlopen(url)
data = connection.read()
headers = connection.info().dict
any help will be greatly appreciated.

First it seems you need to import urllib2 to make a POST request.
You have to send the POST data that you get from the to_postdata method
using the data argument of urlopen:
def augment_POST(url, **kwargs) :
secrets = hidden.oauth()
consumer = oauth2.Consumer(secrets['consumer_key'],
secrets['consumer_secret'])
token = oauth2.Token(secrets['token_key'],
secrets['token_secret'])
oauth_request = oauth2.Request.from_consumer_and_token(
consumer,
token= token,
http_method='POST',
http_url=url,
parameters=kwargs
)
oauth_request.sign_request(oauth2.SignatureMethod_HMAC_SHA1(),
consumer, token)
# this is the data that must be sent with you POST request
return oauth_request.to_postdata()
def follow_user(id):
url = "https://api.twitter.com/1.1/friendships/create.json"
print 'Attempting to follow: %d' % id
postdata = augment(url, method='GET', user_id=id)
# Send the POST request with the data argument
# The url is the same as the data is sent in the body of the request
connection = urllib2.urlopen(url, data=postdata)
data = connection.read()
headers = connection.info().dict
I would recommend to use the requests_oauthlib module which makes all this really easy:
from requests_oauthlib import OAuth1Session
tokens = hidden.oauth()
client = OAuth1Session(tokens['consumer_key'],
tokens['consumer_secret'],
tokens['token_key'],
tokens['token_secret'])
def follow_user(id):
url = "https://api.twitter.com/1.1/friendships/create.json"
print 'Attempting to follow: %d' % id
# for GET requests use client.get and the `params` argument
# instead of the `data` argument
response = client.post(url, data={'user_id': id})
data = response.text
# or even `data = response.json()` to decode the data
headers = response.headers

How to deal with 401 (unauthorised) in python requests

What I want to do is GET from a site and if that request returns a 401, then redo my authentication wiggle (which may be out of date) and try again. But I don't want to try a third time, since that would be my authentication wiggle having the wrong credentials. Does anyone have a nice way of doing this that doesn't involve properly ugly code, ideally in python requests library, but I don't mind changing.

It doesn't get any less ugly than this, I think:
import requests
from requests.auth import HTTPBasicAuth
response = requests.get('http://your_url')
if response.status_code == 401:
response = requests.get('http://your_url', auth=HTTPBasicAuth('user', 'pass'))
if response.status_code != 200:
# Definitely something's wrong

You could have wrapped this in a function and used a decorator to evaluate the response and retry the auth on 401. Then you only need to decorate any function that requires this re-auth logic....
Update:
As requested, a code example. I'm afraid this one is an old piece of code, Python 2 based, but you'll get the idea. This one will retry an http call a number of times as defined in settings.NUM_PLATFORM_RETRIES and will call a refresh_token on auth failures. you can adjust the use case and result to whatever.
You can then use this decorator around methods:
#retry_on_read_error
def some_func():
do_something()
def retry_on_read_error(fn):
"""
Retry Feed reads on failures
If a token refresh is required it is performed before retry.
This decorator relies on the model to have a refresh_token method defined, othewise it will fail
"""
#wraps(fn)
def _wrapper(self, *args, **kwargs):
for i in range(settings.NUM_PLATFORM_RETRIES):
try:
res = fn(self, *args, **kwargs)
try:
_res = json.loads(res)
except ValueError:
# not a json response (could be local file read or non json data)
return res
if 'error' in _res and _res['error']['status'] in (401, 400):
raise AccessRefusedException(_res['error']['message'])
return res
except (urllib2.URLError, IOError, AccessRefusedException) as e:
if isinstance(e, AccessRefusedException):
self.refresh_token()
continue
raise ApiRequestFailed(
"Api failing, after %s retries: %s" % (settings.NUM_PLATFORM_RETRIES, e), args, kwargs
)
return _wrapper

You can use something like this
# 401 retry strategy
import requests
from requests import Request, Session, RequestException
class PreparedRequest:
"""
Class to make Http request with 401 retry
"""
failedRequests = []
defaultBaseUrl = "https://jsonplaceholder.typicode.com"
MAX_RETRY_COUNT = 0
def __init__(self, method, endpoint,
baseurl=defaultBaseUrl, headers=None, data=None, params=None):
"""
Constructor for PreparedRequest class
#param method: Http Request Method
#param endpoint: endpoint of the request
#param headers: headers of the request
#param data: data of request
#param params: params of the request
"""
self.method = method
self.url = baseurl + endpoint
self.headers = headers
self.data = data
self.params = params
self.response = None
def send(self):
"""
To send http request to the server
#return: response of the request
"""
req = Request(method=self.method, url=self.url, data=self.data,
headers=self.headers,params=self.params)
session = Session()
prepared = session.prepare_request(req)
response = session.send(prepared)
if response.status_code == 200:
PreparedRequest.failedRequests.append(self)
PreparedRequest.refresh_token()
elif response.status_code == 502:
raise Exception(response.raise_for_status())
else:
self.response = session.send(prepared)
#staticmethod
def refresh_token():
if PreparedRequest.MAX_RETRY_COUNT > 3:
return
print("Refreshing the token")
# Write your refresh token strategy here
PreparedRequest.MAX_RETRY_COUNT += 1
total_failed = len(PreparedRequest.failedRequests)
for i in range(total_failed):
item = PreparedRequest.failedRequests.pop()
item.send()
r = PreparedRequest(method="GET", endpoint="/todos/")
r.send()
print(r.response.json())

You need to send in the header of the request the authentication param
import requests
from requests.auth import HTTPBasicAuth
auth = HTTPBasicAuth("username", "password")
response = requests.get("http://serverIpOrName/html", auth=auth)
if response.status_code == 401 :
print("Authentication required")
if response.status_code == 200:
print(response.content)

Set useragent on Client __init__ (Python suds)

I would like to know how to set the useragent in all SOAP request with suds in Python, including WSDL get.
Indeed, on the following code :
Client('http://...')
The WSDL is get with the default Python useragent.
The WSDL is available on the server only for specific useragent.
Thank you

I don't know whether that's the easiest way to do it, but it is certainly possible to do using httplib2 (this trick also gives you keep-alive connections) :
from suds.transport import Transport
import httplib2, StringIO
class Httplib2Response:
pass
class Httplib2Transport(Transport):
def __init__(self, **kwargs):
Transport.__init__(self)
self.http = httplib2.Http()
def send(self, request):
url = request.url
message = request.message
headers = request.headers
headers['User-Agent']='XYZ'
response = Httplib2Response()
response.headers, response.message = self.http.request(url,
"PUT", body=message, headers=headers)
return response
def open(self, request):
response = Httplib2Response()
request.headers['User-Agent']='XYZ'
response.headers, response.message = self.http.request(request.url, "GET",
body=request.message, headers=request.headers)
return StringIO.StringIO(response.message)
And then you need to pass the transport class to the suds.client:
http = Httplib2Transport()
client = Client(url,transport=http)

You can override the u2opener method of Transport class to set your own addheaders attribute:
class HttpTransportCustomUserAgent(HttpTransport):
def __init__(self, **kwargs):
self.user_agent = kwargs.get('user_agent', 'Python-urllib/%s' % urllib2.__version__)
if 'user_agent' in kwargs:
del(kwargs['user_agent'])
HttpTransport.__init__(self, **kwargs)
def u2opener(self):
"""
Create a urllib opener.
#return: An opener.
#rtype: I{OpenerDirector}
"""
if self.urlopener is None:
result = urllib2.build_opener(*self.u2handlers())
result.addheaders = [('User-agent', self.user_agent)]
return result
else:
return self.urlopener
Now you can use this new transporter class for suds.client:
http = HttpTransportCustomUserAgent(user_agent='My custom User Agent')
client = Client(url, transport=http)

does urllib2 support preemptive authentication authentication?

I am trying access a REST API.
I can get it working in Curl/REST Client (the UI tool), with preemptive authentication enabled.
But, using urllib2, it doesn't seem to support this by default and I can't find a way to turn it on.
Thanks :)

Here's a simple Preemptive HTTP basic auth handler, based on the code from urllib2.HTTPBasicAuthHandler. It can be used in the exact same manner, except an Authorization header will be added to every request with a matching URL. Note that this handler should be used with a HTTPPasswordMgrWithDefaultRealm. That's because there is no realm coming back in a WWW-Authenticate challenge since you're being preemptive.
class PreemptiveBasicAuthHandler(urllib2.BaseHandler):
def __init__(self, password_mgr=None):
if password_mgr is None:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
self.passwd = password_mgr
self.add_password = self.passwd.add_password
def http_request(self,req):
uri = req.get_full_url()
user, pw = self.passwd.find_user_password(None,uri)
#logging.debug('ADDING REQUEST HEADER for uri (%s): %s:%s',uri,user,pw)
if pw is None: return req
raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip()
req.add_unredirected_header('Authorization', auth)
return req

similar to #thom-nichols's answer; but subclassing HTTPBasicAuthHandler also handling HTTPS requests.
import urllib2
import base64
class PreemptiveBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
'''Preemptive basic auth.
Instead of waiting for a 403 to then retry with the credentials,
send the credentials if the url is handled by the password manager.
Note: please use realm=None when calling add_password.'''
def http_request(self, req):
url = req.get_full_url()
realm = None
# this is very similar to the code from retry_http_basic_auth()
# but returns a request object.
user, pw = self.passwd.find_user_password(realm, url)
if pw:
raw = "%s:%s" % (user, pw)
auth = 'Basic %s' % base64.b64encode(raw).strip()
req.add_unredirected_header(self.auth_header, auth)
return req
https_request = http_request
here is an example for dealing with a jenkins server which does not send you 401 http errors (retry with auth). I'm using urllib2.install_opener to make things easy.
jenkins_url = "https://jenkins.example.com"
username = "johndoe"
api_token = "some-cryptic-value"
auth_handler = PreemptiveBasicAuthHandler()
auth_handler.add_password(
realm=None, # default realm.
uri=jenkins_url,
user=username,
passwd=api_token)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)

Depending on what kind of authentication is required, you can send the Authorization headers manually by adding them to your request before you send out a body.