Develop Reference

InvalidSchema: No connection adapters were found for "{'url':

I am trying to do pagination using pyspark and getting below error. My pagination link is in the header as Key [Link] and value [rel="next"]. The error is displayed at this line r1 = requests.get(response.links['next']).The issue is baseURL missing from the "next" URL being passed.
getURL = 'https://api.xxx.com/v3/direct-access/abc'
baseURL = 'https://api.xxx.com/v3/direct-access'
headers = {
"accept" : "application/json",
"Content-Type": "application/json",
"Authorization": "Bearer " + str(token)
}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links['next']: ## != response.links['last']:
r1 = requests.get(response.links['next'])
r = r1.json()
for i in r:
results.append(i)
Error: InvalidSchema: No connection adapters were found for "{'url': '/abc? action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000', 'rel': 'next'}"
InvalidSchema Traceback (most recent call last) <ipython-input-45-
f27cc7bf373e> in <module> 17 18
while response.links['next']: ## != response.links['last']: ---> 19
r1 = requests.get(response.links['next'])
20 r = r1.json()
21 for i in r:
InvalidSchema: No connection adapters were found for "{'url':
'/liners?
action=next&next_page=linerid+%3C+1010031264&pagesize=10000', 'rel':
'next'}"
How can i merge both baseURL and url into one link and pass it in while loop? Something like below
https://api.xxx.com/v3/direct-access/abc?action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
response = requests.get(baseURL + response.links['next']
['url'],headers=headers)
r1 = response.json()
for i in response:
results.append(i)
#######below not returning results, running for ever ######
return results
rdd = spark.sparkContext.parallelize((results))
print(rdd)
df = spark.read.option('multiline','true').json(rdd)
df.repartition(1).write.json(stagingpath,mode="overwrite")

There are several problems with your code.
response.links['next'] is a dict {'url': ...}. requests.get(...) expects a URL.
# requests.get(response.links['next'])
requests.get(response.links['next']['url'])
# requests.get(baseURL + response.links['next']['url']) # With baseURL
headers are not passed in the subsequent calls.
# requests.get(response.links['next']['url'])
requests.get(response.links['next']['url'], headers=headers)
response is not modified, resulting in infinite loop.
while response.links['next']:
# r1 = requests.get(response.links['next']['url'], headers=headers)
response = requests.get(response.links['next']['url'], headers=headers)
For the last link, 'next' will not exist.
# while response.links['next']:
while response.links.get('next'):
Minimal, reproducible example:
import requests
getURL = 'https://api.github.com/users/acjh/repos'
baseURL = ''
headers = {}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
nextURL = baseURL + response.links['next']['url']
response = requests.get(nextURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
assert len(results) == requests.get(getURL[:-6]).json()['public_repos']

like this:
while response.links['next']:
next_link = response.links['next']['url']
r1 = requests.get(baseURL + next_link, headers=headers)
r = r1.json()
for i in r:
results.append(i)

API Shows image only if ModHeader is on

I am requesting a user image from an api but the image only shows up if I'm using the chrome extension ModHeader which takes the authorization header and url pattern. In my code I am passing the header info so I'm not sure why it doesn't display.
#app.route('/cardholder/<name>', methods=['GET', 'POST'])
def cardholder(name):
response = requests.get("https://commandcentre-api-us.security.gallagher.cloud/api/cardholders", verify=False, headers=Headers)
r = json.loads(response.text)
data = {}
for x in r['results']:
data[x["firstName"]] = x["id"]
chid = data[''+name+'']
url = ("https://commandcentre-api-us.security.gallagher.cloud/api/cardholders/" + chid)
ch = requests.get(url, verify=False, headers=Headers)
chl = json.loads(ch.text)
try:
chimage = chl["#DL"]["href"]
except:
chimage = "../static/img/noimg.png"
return render_template('cardholder.html', name=name, chid=chid, chl=chl, chimage=chimage)

Replacing requests with urllib

I'm currently using the following function to create a bearer token for further API Calls:
import ujson
import requests
def getToken():
#create token for Authorization'
url = 'https://api.XXX.com/login/admin'
payload = "{\n\t\"email\":\"test#user.com\",\n\t\"password\":\"password\"\n}"
headers1 = {
'Content-Type': 'application/json'
}
response = requests.request('POST', url, headers = headers1, data = payload)
#create string to pass on to api request
jsonToken = ujson.loads(response.text)
token = jsonToken['token']
return token
How can I do the same by using urllib.request?

Is this what you're looking for?
from urllib.request import Request, urlopen
import ujson
def getToken():
url = 'https://api.xxx.com/login/admin'
payload = """{"email":"test#user.com","password":"password"}"""
headers = {
'Content-Type': 'application/json'
}
request = Request(method='POST',
data=payload.encode('utf-8'),
headers=headers,
url=url)
with urlopen(request) as req:
response = req.read().decode('utf-8')
jsonToken = ujson.loads(response)
token = jsonToken['token']
return token

Python Webhook: Passing through a URL + payload

I'm a beginner with Python and trying to build a service that takes information from api.ai, passes it to an API, then returns a confirmation message from the JSON it returns.
app.py:
#!/usr/bin/env python
from __future__ import print_function
from future.standard_library import install_aliases
install_aliases()
from urllib.parse import urlparse, urlencode
from urllib.request import urlopen, Request
from urllib.error import HTTPError
import json
import os
import sys
import logging
from flask import Flask, render_template
from flask import request
from flask import make_response
# Flask app should start in global layout
app = Flask(__name__)
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
#app.route('/webhook', methods=['POST'])
def webhook():
req = request.get_json(silent=True, force=True)
print("Request:")
print(json.dumps(req, indent=4))
res = processRequest(req)
res = json.dumps(res, indent=4)
# print(res)
r = make_response(res)
r.headers['Content-Type'] = 'application/json'
return r
def processRequest(req):
if req.get("result").get("action") != "bookMyConference":
return {}
#oauth
orequest = req.get("originalRequest") # work down the tree
odata = orequest.get("data") # work down the tree
user = odata.get("user") # work down the tree
access_token = user.get("access_token")
#data
result = req.get("result") # work down the tree
parameters = result.get("parameters") # work down the tree
startdate = parameters.get("start-date")
meetingname = parameters.get("meeting-name")
payload = {
"start-date": startdate,
"end-date": startdate,
"meeting-name": meetingname
}
# POST info to join.me
baseurl = "https://api.join.me/v1/meetings"
p = Request(baseurl)
p.add_header('Content-Type', 'application/json; charset=utf-8')
p.add_header('Authorization', 'Bearer ' + access_token) #from oauth
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
jresult = urlopen(p, jsondataasbytes).read()
data = json.loads(jresult)
res = makeWebhookResult(data)
return res
def makeWebhookResult(data):
speech = "Appointment scheduled!"
print("Response:")
print(speech)
return {
"speech": speech,
"displayText": speech,
# "data": data,
"source": "heroku-bookmyconference"
}
if __name__ == '__main__':
port = int(os.getenv('PORT', 5000))
print("Starting app on port %d" % port)
app.run(debug=False, port=port, host='0.0.0.0')
Edit 4: Here's the error I'm getting in my Heroku logs:
2017-03-21T19:06:09.383612+00:00 app[web.1]: HTTPError: HTTP Error
400: Bad Request

Borrowing from here, using urlib modules inside processRequest() you could add your payload to urlopen like this:
req = Request(yql_url)
req.add_header('Content-Type', 'application/json; charset=utf-8')
jsondata = json.dumps(payload)
jsondataasbytes = jsondata.encode('utf-8') # needs to be bytes
result = urlopen(req, jsondataasbytes).read()
data = json.loads(result)
Things get more succinct if using the requests module:
headers = {'content-type': 'application/json'}
result = requests.post(yql_url, data=json.dumps(payload), headers=headers)
data = result.json()
EDIT: Adding some details specific to the join.me api
Looking at the join.me docs you'll need to obtain an access token to add to your header. But you also need an app auth code before you can get an access token. You can get the app auth code manually, or by chaining some redirects.
To get started, try this url in your browser and get the code from the callback params. Using your join.me creds:
auth_url = 'https://secure.join.me/api/public/v1/auth/oauth2' \
+ '?client_id=' + client_id \
+ '&scope=scheduler%20start_meeting' \
+ '&redirect_uri=' + callback_url \
+ '&state=ABCD' \
+ '&response_type=code'
print(auth_url) # try in browser
To get an access token:
token_url = 'https://secure.join.me/api/public/v1/auth/token'
headers = {'content-type': 'application/json'}
token_params = {
'client_id': client_id,
'client_secret': client_secret,
'code': auth_code,
'redirect_uri': callback_url,
'grant_type': 'authorization_code'
}
result = requests.post(token_url, data=json.dumps(token_params), headers=headers)
access_token = result.json().get('access_token')
Then your header for the post to /meetings would need to look like:
headers = {
'content-type': 'application/json',
'Authorization': 'Bearer ' + access_token
}

Problems with authorization

I have problems with authorization in python. I want automatic enter to website, but i can't. I used many libraries: Grab, urlib2, request, but i never entered(
For check myself i enter pege with account data
It's real site, login and password
URL="http://pin-im.com/accounts/login/"
LOGIN="testuser"
PASSWORD="test12345user"
urlib2:
def authorization():
import urllib2
gh_url = 'http://pin-im.com/accounts/login/'
gh_user= 'testuser'
gh_pass = 'test12345user'
req = urllib2.Request(gh_url)
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(None, gh_url, gh_user, gh_pass)
auth_manager = urllib2.HTTPBasicAuthHandler(password_manager)
opener = urllib2.build_opener(auth_manager)
urllib2.install_opener(opener)
handler = urllib2.urlopen(req)
Grab:
def autorization():
g = Grab()
g.setup(post={'username':'testuser', 'Password':'test12345user', 'act': 'submit'})
g.go("http://pin-im.com/accounts/login/")
g.go("http://pin-im.com/user/my-profile/")
print g.response.code
Request(i used all methods in Request Lib for authorization, one of them):
from requests.auth import HTTPBasicAuth
requests.get('http://pin-im.com/accounts/login/', auth=HTTPBasicAuth('testuser', 'test12345user'))
r. get("http://pin-im.com/user/my-profile/")
r.status_code
I'm despair, can you help me login to this site? and what i did wrong?

userData = "Basic " + ("testuser:test12345user").encode("base64").rstrip()
req = urllib2.Request('http://pin-im.com/accounts/login')
req.add_header('Accept', 'application/json')
req.add_header("Content-type", "application/x-www-form-urlencoded")
req.add_header('Authorization', userData)
res = urllib2.urlopen(req)

This site uses CSRF protection, so you should get csrftoken cookie and send it back to server with your request:
import Cookie
from urllib import urlencode
import httplib2
URL="http://pin-im.com/accounts/login/"
LOGIN="testuser"
PASSWORD="test12345user"
http = httplib2.Http()
response, _ = http.request(URL)
cookies = Cookie.BaseCookie()
cookies.load(response["set-cookie"])
csrftoken = cookies["csrftoken"].value
headers = {'Content-type': 'application/x-www-form-urlencoded'}
headers['Cookie'] = response['set-cookie']
data = {
"csrfmiddlewaretoken": csrftoken,
"username":LOGIN,
"password": PASSWORD
}
response, _ = http.request(URL, "POST", headers=headers, body=urlencode(data))
response, content = http.request(
"http://pin-im.com/user/my-profile/",
"GET",
headers={'Cookie': response['set-cookie']}
)
print response, content