I'm using requests lib to fetch data from remote server and I'm saving the data in the model, but I need to handle pagination, currently I'm only loading one page from server.
I have a url for pagination like so
{
"status": "success",
"count": 32,
"total": 32,
"next": "https://pimber.ly/api/v2/products?sinceId=5c3ca8470985af0016229b5b",
"previous": "https://pimber.ly/api/v2/products?maxId=5c3ca8470985af0016229b04",
"sinceId": "5c3ca8470985af0016229b04",
"maxId": "5c3ca8470985af0016229b5b",
"data": [
{
"Primary ID": "API_DOCS_PROD1",
"Product Name": "Example Product 1",
"Product Reference": "Example Reference 1",
"Buyer": "Example Buyer 1",
"_id": "5c3ca8470985af0016229b04",
"primaryId": "API_DOCS_PROD1"
},
I've tried to use python generator to handle current situation but, that does not do anything
_plimber_data = response.json()
yield _plimber_data
_next = _plimber_data['next']
print(_next)
for page in _next:
_next_page = session.get(_plimber_data, params={'next': page}).json()
yield _next_page['next']
for _data in page:
Product.objects.create(
qr_id=_data['primaryId'],
ean_code=_data['EAN'],
description=_data['Description105'],
category=_data['Category'],
marketing_text=_data['Marketing Text'],
bullet=_data['Bullet 1'],
brand_image=_data['Brand Image'],
image=_data['Images']
)
logger.debug(f'Something went wrong {_data}')
print(f'This is the Data:{_data}')
Can someone please explain me how to handle this so I can load all the data into the database, thanks.
Ok so I've solved it, two thinks first generator function
def _get_product():
"""
TODO: Fetch data from server
"""
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json',
'Authorization': settings.TOKEN
}
try:
response = requests.get(
url=f'{settings.API_DOMAIN}',
headers=headers
)
response.raise_for_status()
except HTTPError as http_err:
print(f'HTTP error occurred: {http_err}')
else:
_plimber_data = response.json()
while _plimber_data['next'] is not None:
response = requests.get(
_plimber_data['next'],
headers=headers
)
_plimber_data = response.json()
for _data in _plimber_data['data']:
yield _data
Then I iterate over generator function, and save the data:
def run(self):
_page_data = _get_product()
for _product in _page_data:
Product.objects.create(
qr_id=_product['primaryId'],
ean_code=_product['EAN'],
description=_product['Description105'],
category=_product['Category'],
marketing_text=_product['Marketing Text'],
bullet=_product['Bullet 1'],
brand_image='\n'.join(_product['Brand Image']),
image='\n'.join(_product['Images'])
)
logger.debug(f'Something went wrong {_product}')
print(f'This is the Data:{_product}')
Related
So how does python call the "next" page in a post request?
I know what i neeed to do, but not sure how to implement, all the examples in youtube use a # and not a cursor, being a semi beginner i am a bit confused
This is my code so far:
def main_request(headers, url1, params):
response = requests.post(url1, headers=headers, json=params, verify=False)
jsonData = response.json()
has_next_key = False
nextKey = ""
if "next_key" in jsonData:
next = True
nextKey = jsonData["next"]
while has_next_key:
data = {"limit_count":500, "limit_size":10000,"curr_key":nextKey}
params = {"data":json.dumps(data, separators=(",", ":"))}
req = requests.post(url1, headers=headers, json=params, verify=False) ## this should do GET request for the third page and so on...
if "next_key" in req:
nextKey = req["next_key"]
print(nextKey) # this returns "3321" which is the value for "next_key" in second page
else:
has_next_key = False
# no next_key, stop the loop
This is the value is brings back at the end of each request
{
"data": [],
"metadata": {},
"links": [
{
"href": "https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2",
"rel": "first"
},
{
"href": "https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2",
"rel": "self"
},
{
"href": "https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=1&size=2&cursor=1542252837:::_S:::12474375-34a7-40a3-9821-28db0b5cc90e-default-asset-10",
"rel": "next"
},
{
"href": "https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=1097&size=2",
"rel": "last"
}
]
}
according to rapid7 support, i need to use the cursor value
Given your jsonData as input, you can use this code to get the next url and assign it to url1:
for item in jsonData.get("links", []):
if item["rel"] == "next":
url1 = item["href"]
break
This only finds the first url. If you need all urls, I'd recommend adding all urls to a list.
For examples:
links = [item["href"] for item in jsonData.get("links", []) if item["rel"] == "next"]
I am trying to use the POST method of Amadeus flight availabilities API on Python, but is is still giving me error 400. Any suggestions? Many thanks.
from amadeus import Client, ResponseError, Location
import requests
amadeus = Client(
client_id='My ID',
client_secret='My Secret')
try:
flights = amadeus.get('/v2/shopping/flight-offers',originLocationCode = "GIG",destinationLocationCode = "ATL",departureDate = "2023-01-31",nonStop = "true",travelClass = "BUSINESS",adults = 1)
body = flights.data[0]
print(body)
except ResponseError as error:
print(error)
try:
availability = amadeus.post('/v1/shopping/availability/flight-availabilities', body)
print(availability.result)
except ResponseError as error:
print('headers: ', error.response.request.headers)
print('body: ', error.response.request.params)
As per their API documentation, the body of the second API call needs to be in the following format:
{
"originDestinations": [
{
"id": "1",
"originLocationCode": "BOS",
"destinationLocationCode": "MAD",
"departureDateTime": {
"date": "2021-11-14",
"time": "21:15:00"
}
}
],
"travelers": [
{
"id": "1",
"travelerType": "ADULT"
}
],
"sources": [
"GDS"
]
}
Right now you are just feeding the second call the return from the first, which doesn't seem to be what they're looking for.
You might also need to feed the proper headers, something like:
headers = {"Content-Type": "application/json; charset=utf-8"}
then you would do
response = requests.post(url, headers=headers, json=body)
I had created App to exporting excel file from Zendisk to help my to Analysis data
But yesterday stop working with this error
line 20, in <module>
tickets = page_data['tickets'] # extract the "tickets" list from the page
KeyError: 'tickets'
I know this error mean key tickets not found in JSON file but actually there are tickets in file
Code:
print(f'Getting tickets from view ID {view_id}')
url = f'https://****.zendesk.com/api/v2/views/{view_id}/tickets.json'
while url:
response = requests.get(url, auth=auth)
page_data = response.json()
tickets = page_data['tickets'] # ERROR here
view_tickets.extend(tickets)
url = page_data['next_page']
JSON Page
{
"tickets": [
{
"url": "https://****.zendesk.com/api/v2/tickets/55251281.json",
"id": *******,
"external_id": null,
"via": {
"channel": "api",
"source": {
"from": {},
"to": {},
"rel": null
}
},
Current code:
...
response = requests.get(url, auth=auth)
page_data = response.json()
tickets = page_data['tickets'] # ERROR here
...
Required code in order to find the problem:
(The code below will catch 2 situations. 1) HTTP GET failed 2) No 'tickets' in the response )
...
response = requests.get(url, auth=auth)
if response.status_code == 200:
page_data = response.json()
print(page_data)
if 'tickets' in page_data:
tickets = page_data['tickets']
else:
print(f'no tickets in the response (url: {url})')
else:
print(f'HTTP GET (url: {url}) failed with status code {response.status_code}')
...
I am facing this error while making request to fetch json from api.
I can get json data using the "/v1/articles' path.
conn = http.client.HTTPSConnection("api.xxxx.com.tr")
headers = {
'accept': "application/json",
'apikey': "cd6b6c96799847698d87dec9f9a731d6"
}
filter = "daily"
conn.request("GET", "/v1/articles", headers=headers)
reader = codecs.getreader("utf-8")
res = conn.getresponse()
data = json.load(reader(res))
json.dumps(data)
return data
But i am having JSONDecodeError if i set filter. Code:
conn = http.client.HTTPSConnection("api.xxxx.com.tr")
headers = {
'accept': "application/json",
'apikey': "cd6b6c96799847698d87dec9f9a731d6"
}
conn.request("GET", "/v1/articles?$filter=Path eq '/daily/'", headers=headers)
reader = codecs.getreader("utf-8")
res = conn.getresponse()
data = json.load(reader(res))
json.dumps(data)
return data
I tried same filter using Postman with no error and i can get Json data.
Returned Json data from Postman:
[
{
"Id": "40778196",
"ContentType": "Article",
"CreatedDate": "2018-03-20T08:28:05.385Z",
"Description": "İspanya'da 2016 yılında çalınan lüks otomobil, şasi numarası değiştirilerek Bulgaristan üzerinden getirildiği Türkiye'de bulundu.",
"Files": [
{
"FileUrl": "http://i.xxxx.com/i/xxxx/98/620x0/5ab0c6a9c9de3d18a866eb54.jpg",
"Metadata": {
"Title": "",
"Description": ""
}
}
],
"ModifiedDate": "2018-03-20T08:32:12.001Z",
"Path": "/gundem/",
"StartDate": "2018-03-20T08:32:12.001Z",
"Tags": [
"ispanya",
"Araç",
"Hırsız",
"Dolandırıcı"
],
"Title": "İspanya'da çalınan lüks araç Türkiye'de bulundu!",
"Url": "http://www.xxxx.com.tr/gundem/ispanyada-calinan-luks-arac-turkiyede-bulundu-40778196"
}
]
I can not figure out the problem. It would be great if anyone help me about this issue. Thank you.
I finally figured out the problem! Using the requests library have solved my problem now I can filter the api request.
data = requests.get('https://api.xxxxx.com.tr/v1/articles', headers =
headers, params={"$filter":"Path eq '/xxxxxx/'"}).json()
I am leaving this answer here for anyone else who can need this solution in the future.
Thanks for all your suggestions.
The problem is in the following line
data = json.load(reader(res))
when your response is not a json string, JSONDecodeError occurs. so, add an additional logic to see if the response is None or a json string. First thing, print the reader(res) and see what the return is
Hi I am using urrlib2 to send some data and custom headers to a link. Am getting 500 internal server error. I have contacted the service, and they are saying the data(json data) is correct but there is some error in my python code. What am I doing wrong ?
Following is the code.
import urllib2,urllib
import json
PREPAYMENT_URL = "https://services.gharpay.in/rest/GharpayService/"
PREPAYMENT_USERNAME = "somename"
PREPAYMENT_PASSWORD = "somepass"
data = {
"customerDetails":{
"address": "ads",
"contactNo": "9663924147",
"email": "a#c.com",
"firstName": "akash",
"lastName": "deshpande",
"prefix": "Mr."
},
"orderDetails" : {
"pincode": "411036",
"clientOrderID": "21234",
"deliveryDate": "13-10-2013",
"orderAmount": "123",
"clientComments": "please be carefull",
"paymentMode": "Cash",
"productDetails": {
"productID": "21334",
"productQuantity": "1",
"unitCost": "123",
"productDescription": "tshirt"
},
"templateID": ""
},
}
def create(request):
function = 'createOrder'
url = PREPAYMENT_URL
url = url+ function
headers= {'username':PREPAYMENT_USERNAME,'password':PREPAYMENT_PASSWORD,'Content-type':'application/json'}
data1 = urllib.urlencode(data)
req = urllib2.Request(url,data1,headers)
try:
contents = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
error_message = e.read()
print error_message # this error message is being printed. It is showing 500 error.
Your code is perfect except for one teensy-weensy detail:
The header should be Content-Type, not Content-type.
Maybe try changing this header and let me know if it works!