I want to get all the values for the key 'id' in data (without all the other extra information, simply just all of the id values (e.g 60220611, 76744679)
import requests
import json
import urllib, json
import time
proxies = {"http": "http://176.9.75.42:3128",
"http": "http://88.198.50.103:8080"}
user_id = 4913866
def jprint(obj):
text = json.dumps(obj, sort_keys=True, indent=4)
print(text)
user_id = 367
URL = f"https://badges.roblox.com/v1/users/{user_id}/badges"
data = []
payload = {"limit": 100, "sortOrder": "Asc"}
resp = requests.get(URL, params=payload)
resp.json()
blob = resp.json()
data.extend(blob["data"])
cursor = blob["nextPageCursor"]
while cursor is not None:
payload.update({"cursor": cursor})
resp = requests.get(URL, params=payload, proxies=proxies)
blob = resp.json()
data.extend(blob["data"])
cursor = blob["nextPageCursor"]
data.append([cursor])
print(data)
import requests
import json
import urllib, json
import time
proxies = {"http": "http://176.9.75.42:3128",
"http": "http://88.198.50.103:8080"}
user_id = 4913866
def jprint(obj):
text = json.dumps(obj, sort_keys=True, indent=4)
print(text)
user_id = 367
URL = f"https://badges.roblox.com/v1/users/{user_id}/badges"
data = []
payload = {"limit": 5, "sortOrder": "Asc"}
resp = requests.get(URL, params=payload)
resp.json()
blob = resp.json()
data.extend(blob["data"])
cursor = blob["nextPageCursor"]
while cursor is not None:
payload.update({"cursor": cursor})
resp = requests.get(URL, params=payload, proxies=proxies)
blob = resp.json()
data.extend(blob["data"])
cursor = blob["nextPageCursor"]
data.append([cursor])
print(data)
# iterate over data list and get the key.
id_list = [each['id'] for each in data]
print(id_list)
Related
I am trying to do pagination using pyspark and getting below error. My pagination link is in the header as Key [Link] and value [rel="next"]. The error is displayed at this line r1 = requests.get(response.links['next']).The issue is baseURL missing from the "next" URL being passed.
getURL = 'https://api.xxx.com/v3/direct-access/abc'
baseURL = 'https://api.xxx.com/v3/direct-access'
headers = {
"accept" : "application/json",
"Content-Type": "application/json",
"Authorization": "Bearer " + str(token)
}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links['next']: ## != response.links['last']:
r1 = requests.get(response.links['next'])
r = r1.json()
for i in r:
results.append(i)
Error: InvalidSchema: No connection adapters were found for "{'url': '/abc? action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000', 'rel': 'next'}"
InvalidSchema Traceback (most recent call last) <ipython-input-45-
f27cc7bf373e> in <module> 17 18
while response.links['next']: ## != response.links['last']: ---> 19
r1 = requests.get(response.links['next'])
20 r = r1.json()
21 for i in r:
InvalidSchema: No connection adapters were found for "{'url':
'/liners?
action=next&next_page=linerid+%3C+1010031264&pagesize=10000', 'rel':
'next'}"
How can i merge both baseURL and url into one link and pass it in while loop? Something like below
https://api.xxx.com/v3/direct-access/abc?action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
response = requests.get(baseURL + response.links['next']
['url'],headers=headers)
r1 = response.json()
for i in response:
results.append(i)
#######below not returning results, running for ever ######
return results
rdd = spark.sparkContext.parallelize((results))
print(rdd)
df = spark.read.option('multiline','true').json(rdd)
df.repartition(1).write.json(stagingpath,mode="overwrite")
There are several problems with your code.
response.links['next'] is a dict {'url': ...}. requests.get(...) expects a URL.
# requests.get(response.links['next'])
requests.get(response.links['next']['url'])
# requests.get(baseURL + response.links['next']['url']) # With baseURL
headers are not passed in the subsequent calls.
# requests.get(response.links['next']['url'])
requests.get(response.links['next']['url'], headers=headers)
response is not modified, resulting in infinite loop.
while response.links['next']:
# r1 = requests.get(response.links['next']['url'], headers=headers)
response = requests.get(response.links['next']['url'], headers=headers)
For the last link, 'next' will not exist.
# while response.links['next']:
while response.links.get('next'):
Minimal, reproducible example:
import requests
getURL = 'https://api.github.com/users/acjh/repos'
baseURL = ''
headers = {}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
nextURL = baseURL + response.links['next']['url']
response = requests.get(nextURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
assert len(results) == requests.get(getURL[:-6]).json()['public_repos']
like this:
while response.links['next']:
next_link = response.links['next']['url']
r1 = requests.get(baseURL + next_link, headers=headers)
r = r1.json()
for i in r:
results.append(i)
I have this python function and i wish to execute this using lambda handler function hence I have written this code. When I execute in Pycharm I don't see any output in console. Can someone guide what is the problem with below code?
import json
from json import loads
import requests
from requests import exceptions
from requests.auth import HTTPBasicAuth
def lambda_handler(event, context):
test_post_headers_body_json()
return {"statusCode": 200, "body": json.dumps("Hello from Lambda!")}
def test_post_headers_body_json():
client_id = "WJRYDHNGROIZHL8B"
client_secret = "V5VXK6FLG1YI0GD2XY3H"
user = "automation-store-admin1#abc.com"
password = "c0Ba5PBdvVl2"
access_point = "https://api.platform.abc.com/auth/oauth/token"
grant_type = "password"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
# auth = auth.HTTPBasicAuth(client_id, client_secret)
data = {"grant_type": grant_type, "username": user, "password": password}
resp = None
try:
resp = requests.post(
access_point,
auth=HTTPBasicAuth(client_id, client_secret),
data=data,
headers=headers,
)
except exceptions.ConnectionError:
exit(1)
if resp.status_code == 200:
resp = loads(resp.text)
if "access_token" in resp:
print(resp["access_token"])
exit(0)
exit(1)
It is normal because when running you code, Python only declare the function not using it. You should add a __main__ entry point at the end of your file:
import json
from json import loads
import requests
from requests import exceptions
from requests.auth import HTTPBasicAuth
def lambda_handler(event, context):
test_post_headers_body_json()
return {"statusCode": 200, "body": json.dumps("Hello from Lambda!")}
def test_post_headers_body_json():
client_id = "WJRYDHNGROIZHL8B"
client_secret = "V5VXK6FLG1YI0GD2XY3H"
user = "automation-store-admin1#abc.com"
password = "c0Ba5PBdvVl2"
access_point = "https://api.platform.abc.com/auth/oauth/token"
grant_type = "password"
headers = {"Content-Type": "application/x-www-form-urlencoded"}
# auth = auth.HTTPBasicAuth(client_id, client_secret)
data = {"grant_type": grant_type, "username": user, "password": password}
resp = None
try:
resp = requests.post(
access_point,
auth=HTTPBasicAuth(client_id, client_secret),
data=data,
headers=headers,
)
except exceptions.ConnectionError:
exit(1)
if resp.status_code == 200:
resp = loads(resp.text)
if "access_token" in resp:
print(resp["access_token"])
exit(0)
exit(1)
# added part
if __name__ == '__main__':
test_post_headers_body_json()
I have written code for calling the AlchemyLanguage API of Bluemix in Python. I need the keywords and entities, but it is only showing the first keyword and first entity for the text file. Where am I going wrong?
import requests
import urllib
import urllib2
def call_alchemy_api(text, API_KEY):
payload = {'outputMode':'json','extract':'entities,keywords','sentiment':'1','maxRetrieve':'1', 'url':'https://www.ibm.com/us-en/'}
payload['apikey'] = API_KEY
encoded_text = urllib.quote_plus(text)
payload['text'] = text
data = urllib.urlencode(payload)
url = 'https://gateway-a.watsonplatform.net/calls/text/TextGetCombinedData'
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
return response
if __name__ == "__main__":
api_key = 'xxxxxxxxxxxxxxxxxxxxxmyapi'
f = open('in0.txt','r')
text = f.read()
print text
response = call_alchemy_api(text, api_key)
print response.read()
Change the maxRetrieve keyword's value.
Example:
payload = {'outputMode':'json','extract':'entities,keywords','sentiment':'1','maxRetrieve':'3', 'url':'https://www.ibm.com/us-en/'}
API Link:
http://www.ibm.com/watson/developercloud/alchemy-language/api/v1/
I'm trying to add a torrent with uTorrent web api (http://www.utorrent.com/community/developers/webapi), in Python using Requests library.
import requests
import re
UTORRENT_URL = 'http://%s:%s/gui/' % ('localhost', '55655')
UTORRENT_URL_TOKEN = '%stoken.html' % UTORRENT_URL
REGEX_UTORRENT_TOKEN = r'<div[^>]*id=[\"\']token[\"\'][^>]*>([^<]*)</div>'
auth = requests.auth.HTTPBasicAuth('x', 'x')
headers = {'content-type': 'application/json'}
r = requests.get(UTORRENT_URL_TOKEN, auth=auth, headers=headers)
token = re.search(REGEX_UTORRENT_TOKEN, r.text).group(1)
guid = r.cookies['GUID']
cookies = dict(GUID = guid)
headers = {'content-type': 'multipart/form-data'}
params = {'action':'add-file','token': token}
files = {'torrent_file':'C:\\x.torrent'}
r = requests.post(UTORRENT_URL, auth=auth, cookies=cookies, headers=headers, params=params, files=files)
print r.json()
Error - torrent file content not supplied in form parameter
Any help appreciated
Ok, the problem was setting the headers. Removing them and it finally works!
import requests
import re
UTORRENT_URL = 'http://%s:%s/gui/' % ('192.168.1.80', '55655')
UTORRENT_URL_TOKEN = '%stoken.html' % UTORRENT_URL
REGEX_UTORRENT_TOKEN = r'<div[^>]*id=[\"\']token[\"\'][^>]*>([^<]*)</div>'
auth = requests.auth.HTTPBasicAuth('x', 'x')
r = requests.get(UTORRENT_URL_TOKEN, auth=auth)
token = re.search(REGEX_UTORRENT_TOKEN, r.text).group(1)
guid = r.cookies['GUID']
cookies = dict(GUID = guid)
params = {'action':'add-file','token': token}
files = {'torrent_file': open('C:\\x.torrent', 'rb')}
r = requests.post(url=UTORRENT_URL, auth=auth, cookies=cookies, params=params, files=files)
You aren't sending the file data. You should read the relevant section of the documentation. The correct code would be this (I marked the line I changed):
import requests
import re
UTORRENT_URL = 'http://%s:%s/gui/' % ('localhost', '55655')
UTORRENT_URL_TOKEN = '%stoken.html' % UTORRENT_URL
REGEX_UTORRENT_TOKEN = r'<div[^>]*id=[\"\']token[\"\'][^>]*>([^<]*)</div>'
auth = requests.auth.HTTPBasicAuth('x', 'x')
headers = {'content-type': 'application/json'}
r = requests.get(UTORRENT_URL_TOKEN, auth=auth, headers=headers)
token = re.search(REGEX_UTORRENT_TOKEN, r.text).group(1)
guid = r.cookies['GUID']
cookies = dict(GUID = guid)
headers = {'content-type': 'multipart/form-data'}
params = {'action':'add-file','token': token}
files = {'torrent_file': open('C:\\x.torrent', 'rb')} # Changed this line
r = requests.post(UTORRENT_URL, auth=auth, cookies=cookies, headers=headers, params=params, files=files)
print r.json()
I am trying to loop through a CSV and run a web request. I can't get it past the first row in the CSV.
The CSV is being exported from Mac Excel as a list of 10 items in 10 rows / 1 column.
def AddTokens(request):
import csv
tokenList = []
output = 0
apikey = "12345"
restkey = "12345"
URL = "https://api.web.com/1/install/"
headers = {'content-type': 'application/json', 'X-web-Application-Id': apikey, 'X-web-REST-API-Key': restkey}
with open('/Users/name/Desktop/test.csv', 'rU') as csvfile:
deviceTokens = csv.reader(csvfile, delimiter=',')
for token in deviceTokens:
deviceToken = token[0].replace("/", "")
deviceType = "ios"
pushToken = "pushtoken_" + deviceToken
payload = {"deviceType": deviceType, "deviceToken": deviceToken, "channels": ["", pushToken]}
r = requests.post(URL, data=json.dumps(payload), headers=headers)
t = get_template('addpush.html')
html = t.render(Context({'output': output, 'tokenList': tokenList, 'deviceTokens': deviceTokens, 'token': token}))
return HttpResponse(html)