Develop Reference

How to write a python loop to change a value for a dictionary key in API request?

I am writing an API request that gives paginated results.
To get results from the next page I need to take a value of 'next_page_cursor' and put it in the parameters of my request that is a dictionary.
This is what I have tried so far. Need to keep changing cursor value in params until there are no more pages.
params = {'title': 'Cybertruck',
'per_page':100,
'cursor': '*'
}
response = requests.get("https://api.aylien.com/news/stories",
headers = headers, params=params).json()
if "next_page_cursor" in response:
cursor = response["next_page_cursor"]

You can use a while loop:
params = {
"title": "Cybertruck",
"per_page": 100,
"cursor": "initial_cursor"
}
def make_request(params)
return requests.get("https://api.aylien.com/news/stories",
headers=headers, params=params).json()
result = []
response = make_request(params)
while "next_page_cursor" in response:
params["cursor"] = response["next_page_cursor"]
response = make_request(params)
result.append(response["information_your_are_interested_in"])

python web scraping post form data using requests not working

I am trying to post input data into a form using a requests.session and it's returning a 500 status.
I am expecting to see the search results retrieved.
I was able to get around a previous login issue with __RequestVerificationToken and cookies - thanks to the help of Bertrand Martel. The next step in my process is to get the Search page, which I was able to get successfully. Now failing when I try to post data into the date fields on the form, which make up the search criteria. Works when I manually complete the form and press submit. All seems very straightforward to me, but not sure why it won't work. Is it still a cookies issue? Any help would be appreciated.
Here is my code:
import requests
from bs4 import BeautifulSoup
EMAIL = 'myemail#gmail.com'
PASSWORD = 'somepwd'
LOGIN_URL = 'https://www.idocmarket.com/Security/LogOn'
SEARCH_URL = 'https://www.idocmarket.com/RIOCO/Document/Search'
s = requests.Session()
s.get(LOGIN_URL)
result = s.post(LOGIN_URL, data = {
"Login.Username": EMAIL,
"Login.Password": PASSWORD
})
soup = BeautifulSoup(result.text, "html.parser")
# Report successful login
print("Login succeeded: ", result.ok)
print("Status code:", result.status_code)
result = s.get(SEARCH_URL)
auth_token = soup.find("input", {'name': '__RequestVerificationToken'}).get('value')
print('auth token:', auth_token )
print("Get Search succeaeded: ", result.ok)
print("get Search Statusa code:", result.status_code)
result = s.post(SEARCH_URL, data = {
"__RequestVerificationToken": auth_token,
"StartRecordDate": "03/01/2019",
"EndRecordDate": "03/31/2019",
"StartDocNumber": "",
"EndDocNumber": "",
"Book": "",
"Page": "",
"Instrument": "",
"InstrumentGroup": "",
"PartyType": "Either",
"PartyMatchType": "Contains",
"PartyName": "",
"Subdivision": "",
"StartLot": "",
"EndLot": "",
"Block": "",
"Section":"",
"Township": "",
"Range": "",
"Legal": "",
"CountyKey": "RIOCO"
})
print("post Dates succeeded: ", result.ok)
print("post Dates Status code:", result.status_code)
print(result.text)

It seems that this time, the xsrf token is needed in the post along with all the existing parameters. A simple solution is to get all the input value & pass it to the request :
import requests
from bs4 import BeautifulSoup
LOGIN_URL = 'https://www.idocmarket.com/Security/LogOn'
SEARCH_URL = 'https://www.idocmarket.com/RIOCO/Document/Search'
EMAIL = 'myemail#gmail.com'
PASSWORD = 'somepwd'
s = requests.Session()
s.get(LOGIN_URL)
r = s.post(LOGIN_URL, data = {
"Login.Username": EMAIL,
"Login.Password": PASSWORD
})
if (r.status_code == 200):
r = s.get(SEARCH_URL)
soup = BeautifulSoup(r.text, "html.parser")
payload = {}
for input_item in soup.select("input"):
if input_item.has_attr('name'):
payload[input_item["name"]] = input_item["value"]
payload["StartRecordDate"] = '09/01/2019'
payload["EndRecordDate"] = '09/30/2019'
r = s.post(SEARCH_URL, data = payload)
soup = BeautifulSoup(r.text, "html.parser")
print(soup)
else:
print("authentication failure")
Also using comprehension list for the payload you can write :
temp_pl = [
(t['name'], t['value'])
for t in soup.select("input")
if t.has_attr('name')
]
payload = dict(temp_pl)
payload["StartRecordDate"] = '09/01/2019'
payload["EndRecordDate"] = '09/30/2019'

Python & API : Why do I get KeyErrors when run by python but the value is there in postman?

I am using the below code to return the raw json data from Harvest, by changing the URL called I have successfully created running scripts for 6 files, however I am having trouble with one and I can't work out why ...
import requests, json
AUTH = "Bearer REDACTED"
ACCOUNT = "REDACTED"
URL = "https://api.harvestapp.com/v2/clients/?"
HEADERS = { "Authorization": AUTH,
"Harvest-Account-ID": ACCOUNT,
"Accept":"application/json"}
r = requests.get(url=URL, headers=HEADERS).json()
total_pages = int(r['total_pages'])
total_entries = int(r['total_entries'])
results = []
for x in range(1, total_pages):
response = requests.get(URL+"page="+str(x), headers=HEADERS)
data = response.json()
next_page = data["next_page"]
results.extend(data["time_entries"])
filepath = "Z:/System Administrator/System Backups/08. Harvest/HARVEST_Clients.json"
with open(filepath, 'w') as outfile:
json.dump(results, outfile)
print('Done!')
print('Total Pages : '+str(total_pages))
print('Total Entries : '+str(total_entries))
When I run the above it gives me the desired result of
Done!
Total Pages : 3
Total Entries : 237
However if I try to use a URL with date variables I get KeyErrors. All I do is change the code from :
URL = "https://api.harvestapp.com/v2/clients/?"
to
URL = "https://api.harvestapp.com/v2/time_entries?from=2017-04-01&to=2018-03-31/?"
And the results.extend variable from
results.extend(data["clients"])
to
results.extend(data["time_entries"])
I get the error
Traceback (most recent call last):
File "Z:\System Administrator\System Backups\08. Harvest\Scripts\API_Harvest_Timesheets 2017-18.py", line 19, in
total_pages = int(r['total_pages'])
KeyError: 'total_pages'
When I run the URL and authorisation through postman I get the following result
{
"time_entries": [FULL DATA RESULT HERE]
"per_page": 100,
"total_pages": 138,
"total_entries": 13711,
"next_page": 2,
"previous_page": null,
"page": 1,
"links": {
"first": "https://api.harvestapp.com/v2/time_entries?from=2017-04-01&page=1&per_page=100&to=2018-03-31",
"next": "https://api.harvestapp.com/v2/time_entries?from=2017-04-01&page=2&per_page=100&to=2018-03-31",
"previous": null,
"last": "https://api.harvestapp.com/v2/time_entries?from=2017-04-01&page=138&per_page=100&to=2018-03-31"
}
}
So I can see that the 'total_pages' value is returned from that url and the value is 138 - so why doesn't this code run for this specific URL but runs fine for others ?

Get data by pages and merge it into one using Python (pagination)

I'm connecting to API which has 500 rows limit per call.
This is my code for a single API call (Works great):
def getdata(data):
auth_token = access_token
hed = {'Authorization': 'Bearer ' + auth_token, 'Accept': 'application/json'}
urlApi = 'https://..../orders?Offset=0&Limit=499'
datar = requests.get(urlApi, data=data, headers=hed, verify=True)
return datar
Now I want to scale it up so it will get me all the records.
This is what I tried to do:
In order to make sure that I have all the rows, I must iterate until there is no more data:
get 1st page
get 2nd page
merge
get 3rd page
merge
etc...
each page is an API call.
This is what I'm trying to do:
def getData(data):
auth_token = access_token
value_offset = 0
hed = {'Authorization': 'Bearer ' + auth_token, 'Accept': 'application/json'}
datarALL = None
while True:
urlApi = 'https://..../orders?Offset=' + value_offset + '&Limit=499'
responsedata = requests.get(urlApi, data=data, headers=hed, verify=True)
if responsedata.ok:
value_offset = value_offset + 499
#to do: merge the result of the get request
datarALL= datarALL+ responsedata (?)
# to do: check if response is empty then break out.
return datarALL
I couldn't find information about how I merge the results of the API calls nor how do I check if I can break the loop.
Edit:
To clear what I'm after.
I can see the results of the API call using:
logger.debug('response is : {0}'.format(datar.json()))
What I want to be able to do:
logger.debug('response is : {0}'.format(datarALL.json()))
and it will show all results from all calls. This requires generate API calls until there is no more data to get.
This is the return sample of API call:
"offset": 0,
"limit": 0,
"total": 0,
"results": [
{
"field1": 0,
"field2": "string",
"field3": "string",
"field4": "string"
}
]
}

In this case, you are almost correct with the idea.
is_valid = True
while is_valid:
is_valid = False
...
...
responsedata = requests.get(urlApi, data=data, headers=hed, verify=True)
if responsedata.status_code == 200: #Use status code to check request status, 200 for successful call
responsedata = responsedata.text
value_offset = value_offset + 499
#to do: merge the result of the get request
jsondata = json.loads(responsedata)
if "results" in jsondata:
if jsondata["results"]:
is_valid = True
if is_valid:
#concat array by + operand
datarALL = datarALL + jsondata["results"]
As I don't know if "results" still exists when the data ran out, so I checked both level.

Add entries into JSON

I am working with an API that doesn't have all the information I need in a single call, and I need to the project code it came from into the call that I am making. Right now it appends the project data to the list, but I really need it to be part of the original call. Here is my output now:
[{"committer_email": "justin.m.boucher#example.com", "short_id": "981147b9", "title": "Added .gitignore", "author_email": "justin.m.boucher#example.com", "authored_date": "2017-08-29T08:31:11.000-07:00", "created_at": "2017-08-29T08:31:11.000-07:00", "author_name": "Justin Boucher", "parent_ids": [], "committed_date": "2017-08-29T08:31:11.000-07:00", "message": "Added .gitignore\n", "committer_name": "Justin Boucher", "id": "981147b905913a60796283ce10f915c53679df49"}, {"project_id": "2"}]
Here is the output I want to achieve:
[{"project_id": "2", "committer_email": "justin.m.boucher#example.com", "short_id": "981147b9", "title": "Added .gitignore", "author_email": "justin.m.boucher#example.com", "authored_date": "2017-08-29T08:31:11.000-07:00", "created_at": "2017-08-29T08:31:11.000-07:00", "author_name": "Justin Boucher", "parent_ids": [], "committed_date": "2017-08-29T08:31:11.000-07:00", "message": "Added .gitignore\n", "committer_name": "Justin Boucher", "id": "981147b905913a60796283ce10f915c53679df49"}]
Here is my code so far:
get_commits.py:
import gitlab
import json
gitlab = gitlab.Gitlab()
projects = gitlab.getProjectID()
for i in projects:
api_str = '/projects/' + str(i) + '/repository/commits'
connect = gitlab.connectAPI(apiCall=api_str)
data = json.dumps(connect)
# Append project id to json, since it isn't created
# in the commits from Gitlab
commit = json.loads(data)
commit.append({'project_id': str(i)})
# make it pretty again for Splunk to read
commit = json.dumps(commit)
print commit
gitlab.py
import os
import ConfigParser
import requests
import json
# Setup Splunk Environment
APPNAME = 'splunk_gitlab'
CONFIG = 'appconfig.conf'
SPLUNK_HOME = os.environ['SPLUNK_HOME']
parser = ConfigParser.SafeConfigParser()
class Gitlab():
# # Load Settings
# parser.read(SPLUNK_HOME + '/etc/apps/' + APPNAME + '/local/' + CONFIG)
# if parser.has_section('Authentication'):
# pass
# else:
# parser.read(SPLUNK_HOME + '/etc/apps/' + APPNAME + '/default/' + CONFIG)
#
# GITLAB_URL = parser.get('Authentication', 'GITLAB_URL')
# API_KEY = parser.get('Authentication', 'API_KEY')
# Used for testing only
GITLAB_URL = 'http://<my_address>'
API_KEY = '<my_key>'
API_SERVER = GITLAB_URL + '/api/v4'
# Place api call to retrieve data
def connectAPI(self, apiCall='/projects'):
headers = {
'PRIVATE-TOKEN': self.API_KEY
}
final_url = self.API_SERVER + apiCall
resp = requests.get(final_url, headers=headers)
status_code = resp.status_code
resp = resp.json()
if status_code == 200:
return resp
else:
raise Exception("Something went wrong requesting (%s): %s" % (
resp['errors'][0]['errorType'], resp['errors'][0]['message']))
def getProjectID(self):
connect = self.connectAPI(apiCall='/projects')
data = json.dumps(connect)
projects = json.loads(data)
project_list = []
for i in projects:
project_list.append(i['id'])
return project_list

If you want to add a new element to the first dictionary in the list instead of appending a new dictionary to the list, try using assignment instead of append.
commit[0]['project_id'] = str(i)