Develop Reference

InvalidSchema: No connection adapters were found for "{'url':

I am trying to do pagination using pyspark and getting below error. My pagination link is in the header as Key [Link] and value [rel="next"]. The error is displayed at this line r1 = requests.get(response.links['next']).The issue is baseURL missing from the "next" URL being passed.
getURL = 'https://api.xxx.com/v3/direct-access/abc'
baseURL = 'https://api.xxx.com/v3/direct-access'
headers = {
"accept" : "application/json",
"Content-Type": "application/json",
"Authorization": "Bearer " + str(token)
}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links['next']: ## != response.links['last']:
r1 = requests.get(response.links['next'])
r = r1.json()
for i in r:
results.append(i)
Error: InvalidSchema: No connection adapters were found for "{'url': '/abc? action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000', 'rel': 'next'}"
InvalidSchema Traceback (most recent call last) <ipython-input-45-
f27cc7bf373e> in <module> 17 18
while response.links['next']: ## != response.links['last']: ---> 19
r1 = requests.get(response.links['next'])
20 r = r1.json()
21 for i in r:
InvalidSchema: No connection adapters were found for "{'url':
'/liners?
action=next&next_page=linerid+%3C+1010031264&pagesize=10000', 'rel':
'next'}"
How can i merge both baseURL and url into one link and pass it in while loop? Something like below
https://api.xxx.com/v3/direct-access/abc?action=next&next_page=%28id%2Ccompletionid%29+%3C+%28840430000754002%2C840430413029241%29&pagesize=10000
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
response = requests.get(baseURL + response.links['next']
['url'],headers=headers)
r1 = response.json()
for i in response:
results.append(i)
#######below not returning results, running for ever ######
return results
rdd = spark.sparkContext.parallelize((results))
print(rdd)
df = spark.read.option('multiline','true').json(rdd)
df.repartition(1).write.json(stagingpath,mode="overwrite")

There are several problems with your code.
response.links['next'] is a dict {'url': ...}. requests.get(...) expects a URL.
# requests.get(response.links['next'])
requests.get(response.links['next']['url'])
# requests.get(baseURL + response.links['next']['url']) # With baseURL
headers are not passed in the subsequent calls.
# requests.get(response.links['next']['url'])
requests.get(response.links['next']['url'], headers=headers)
response is not modified, resulting in infinite loop.
while response.links['next']:
# r1 = requests.get(response.links['next']['url'], headers=headers)
response = requests.get(response.links['next']['url'], headers=headers)
For the last link, 'next' will not exist.
# while response.links['next']:
while response.links.get('next'):
Minimal, reproducible example:
import requests
getURL = 'https://api.github.com/users/acjh/repos'
baseURL = ''
headers = {}
results = []
response = requests.get(getURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
while response.links.get('next'):
nextURL = baseURL + response.links['next']['url']
response = requests.get(nextURL, headers=headers)
r = response.json()
for i in r:
results.append(i)
assert len(results) == requests.get(getURL[:-6]).json()['public_repos']

like this:
while response.links['next']:
next_link = response.links['next']['url']
r1 = requests.get(baseURL + next_link, headers=headers)
r = r1.json()
for i in r:
results.append(i)

Requests: Error sending payload in python post request

The code behaving strangely that I am unable to understand what's going on..
The code that works fine:
link = "https://api.luminati.io/dca/trigger_immediate?collector=XXXxxxXXXxxxXXXX"
head = {"Authorization": "Bearer xxXXxxXXxx" ,"Content-Type": "application/json"}
data = '{"url":"https://www.practo.com/pune/doctor/XXXXXXxXXXXX"}'
res = requests.post(link, headers = head, data = data)
print("Status: "+str(res.status_code), "Message: "+res.text)
Output:
Status: 202 Message: {"response_id":"z7627t1617552745375r14623bt37oo"}
But I want to load "url":"https://www.practo.com/pune/doctor/XXXXXXxXXXXX" this thing dynamically.
url = "https://www.practo.com/pune/doctor/XXXXXXxXXXXX"
link = "https://api.luminati.io/dca/trigger_immediate?collector=XXXxxxXXXxxxXXXX"
head = {"Authorization": "Bearer xxXXxxXXxx" ,"Content-Type": "application/json"}
data = {"url":url}
res = requests.post(link, headers = head, data = data)
print("Status: "+str(res.status_code), "Message: "+res.text)
Output:
Status: 500 Message: 'Unexpected token u in JSON at position 7'

To load data dynamically try using %s string feature, like that:
url = "https://www.practo.com/pune/doctor/XXXXXXxXXXXX"
data = '{"url":"%s"}' % url
or you can convert dictionary entirely to str, like:
import json
data = {"url":link}
res = requests.post(link, headers=head, data=json.dumps(data))
by the way, you can pass body not like data, but like json, here's documents:
:param json: (optional) json data to send in the body of the :class:Request. So your request will look like:
data = {"url":link}
res = requests.post(link, headers=head, json=data)

any_dynamic_variable='XXXXXXxXXXXX'#You can make your websit or part of the url to be dynamic
url = f'https://www.practo.com/pune/doctor/{any_dynamic}'
headers = {"Authorization": "Bearer xxXXxxXXxx" ,"Content-Type": "application/json"}
payload={
'your_key':'your_value',
'currancy':'CA',#<==== This is an example
}
r = requests.post(url,headers=headres,data=payload}
print(r.status_code)#check your status to be able to find the error if you have any
print(r.text)

I want to send email to user from API response in Django?

[data image][1]I want to send email to user after this function calling so in first function i m creating customer on razor pay and after fetching customer id i m passing to other function and on.
so in this function get_virtual_account() i am getting all the response from API provided by razor-pay and i need to send that response to user who created the account so how can i do that i am not able to send this response in email.
def create_razor_customer(data):
logger.info("Inside create_razor_customer")
headers = {'Content-Type': 'application/json',}
data=json.dumps(data)
response = requests.post(settings.API_RAZORPAY+'/customers', headers=headers, data=data, auth=(settings.API_RAZORPAY_KEY, settings.API_RAZORPAY_SECRET))
logger.info(json.loads(response.content))
json_response = response.json()
customer_id = json_response['id']
logger.info(customer_id)
get_razor_customer(customer_id)
return response
def get_razor_customer(customer_id):
logger.info("Inside get_razor_customer")
headers = {'Content-Type': 'application/json',}
response = requests.get(settings.API_RAZORPAY+'/customers/'+customer_id, headers=headers, auth=(settings.API_RAZORPAY_KEY, settings.API_RAZORPAY_SECRET))
logger.info(json.loads(response.content))
create_razor_virtual_account(customer_id)
return response
def create_razor_virtual_account(customer_id):
logger.info("Inside create_razor_virtual_account")
headers = {'Content-Type': 'application/json',}
data = {"receivers": {"types": ["bank_account"]},"description": "razorpay","customer_id": customer_id,"close_by": 1761615838,"notes": {"reference_key": "reference_value"}}
data=json.dumps(data)
response = requests.post(settings.API_RAZORPAY+'/virtual_accounts', headers=headers, data=data, auth=(settings.API_RAZORPAY_KEY, settings.API_RAZORPAY_SECRET))
json_response = response.json()
virtual_id = json_response['id']
logger.info(virtual_id)
logger.info(json_response)
return response
def get_virtual_account(virtual_id):
logger.info("Inside get_virtual_account")
logger.info(virtual_id)
headers = {'Content-Type': 'application/json',}
response = requests.get(settings.API_RAZORPAY+'/virtual_accounts/'+virtual_id,headers=headers, auth=(settings.API_RAZORPAY_KEY, settings.API_RAZORPAY_SECRET))
json_response = response.json()
logger.info(json_response)
send_account_details()
return response
def send_account_details():
logger.info('Inside send_account_details')
send_mail('Account Details', 'Details for Razorpay account', settings.EMAIL_HOST_USER, ['abhishek#byond.travel',])
logger.info("sent")
return "sent"
[1]: https://i.stack.imgur.com/proIH.png

Suppose your response JSON is
res_data={"duration":1201,"number":6,"result":"FAILURE","url":"http://localhost:8080/job/git_checkout/6/"}
Then you need to pass this into send_account_details function.
def send_account_details(res_data):
import json
logger.info('Inside send_account_details')
body = "JSON Response is : " + json.dumps(res_data) + "\n\n" + "Details for Razorpay account"
send_mail('Account Details', body, settings.EMAIL_HOST_USER, ['abhishek#byond.travel',])
logger.info("sent")
return "sent"
Using json.dumps(res_data) you can add your JSON to body of email.

my python requests return 401 while postman returns 200

i make my requests class, the code is as below:
class RunMethod:
def post_main(self, url, data, header=None):
res = None
if header != None:
res = requests.post(url=url, data=data, headers=header)
else:
res = requests.post(url=url, data=data)
return res.json()
def get_main(self, url, data=None,header=None):
res = None
if header != None:
res = requests.get(url=url,data=data,headers=header,verify=False)
else:
res = requests.get(url=url,data=data,verify=False)
return res.json()
def run_main(self, method,url,data=None,headers=None):
res = None
if method == 'get':
res = self.get_main(url, data, headers)
else:
res = self.post_main(url, data, headers)
and i capture an api from charles and test it in postman, it returns 200. i export python code from postman and it is like this:
import requests
url = "https://stargate.ar.elenet.me/minimart.service/intelligent/invoke"
querystring = {"traceId": "1000000294010",
"shelfCode": "lu8ssMgCpgq00FDYdpX76Q..", "tracedAt": "1545641563164"}
payload = ""
headers = {
'X-STARGATE-ACCESS-TOKEN': "d7594351-0663-43a8-ad55-180c8b29db82",
'Cookie': "SID=NTVMAu8FKskyj06ln8J9uhS45fgcRNk1V3jQ; USERID=2228440841",
'Authorization': "ElemeAPI token",
'cache-control': "no-cache"
}
response = requests.request(
"GET", url, data=payload, headers=headers, params=querystring)
print(response.text)
it works, and i changed my class, put the data like this:
url = "https://stargate.ar.elenet.me/minimart.service/intelligent/invoke?traceId=1000000294010&shelfCode=lu8ssMgCpgq00FDYdpX76Q..&tracedAt=1545641563164"
headers = {
'X-STARGATE-ACCESS-TOKEN': "d7594351-0663-43a8-ad55-180c8b29db82",
'Cookie': "SID=NTVMAu8FKskyj06ln8J9uhS45fgcRNk1V3jQ; USERID=2228440841",
'Authorization': "ElemeAPI token",
'cache-control': "no-cache",
'Content-Type':'application/json'
}
exam = RunMethod()
res = exam.run_main('get', url, headers)
i just put querystring into url, but it returns 401.
i don't know where it's wrong. could anybody help me? thanks a lot!

Update your code in get_main method as per below code.
res = requests.get(url=url,params=data,headers=header,verify=False)
Here we are calling instantiated RunMethod class and calling run_main method.
exam = RunMethod()
res = exam.run_main('get', url, querystring, headers)

The run_main accepts 4 arguments, exam.run_main('get', url, headers) provides only 3 of them (method='get', url=url, data=headers, headers=None).
I would recommend to use named arguments when skipping some of the optional ones:
exam.run_main('get', url, headers=headers)

Why there is different response in python and ruby mechanize for the same HTTP GET request?

Why there is different response in python and ruby mechanize for the same HTTP GET request?
Python:
def send(cls, url, values = None, method = 'GET', header={}):
print(url)
c = cls()
data = None
context = ssl._create_unverified_context()
if values is not None:
# data = urllib.parse.urlencode(values)
# data = data.encode('utf-8') # data should be bytes
data = json.dumps(values)
data = data.encode('utf-8')
if data is None:
c.request = urllib.request.Request(url)
else:
c.request = urllib.request.Request(url, data)
c.request.add_header('Content-Type', 'application/json')
for key, value in header.items():
c.request.add_header(key,value)
c.request.method = method
# print(c.request.get_header('Content-Type'))
try:
c.response = urllib.request.urlopen(c.request, context=context)
except urllib.error.HTTPError as e:
c.response = e
# print(e.hdrs)
return c
resp = HTTPSender.send("https://10.5.110.143/__api__/logon/704383e332d46036369a510e2673fe94?method=144", method="GET")
print(resp.body)
Output :
{
"message": "The method is not allowed for the requested URL."
}
Ruby mechanize:
#agent=Mechanize.new
$resp = .get("https://10.5.110.143/__api__/logon/704383e332d46036369a510e2673fe94?method=144")
puts $resp.content
Output:
405 => Net::HTTPMethodNotAllowed for https://10.5.110.143/api/logon/9c7ca3f5
34ea1c8a5bfe64d7cf08797a?method=144 -- unhandled response
Is it something expected?