I used requests.get to get json from my webpage but I have recived some error when I run my code.
This is my code:
def get(t):
while 1:
r = requests.get('http://raspberry.site11.com/controller.php')
while r.status_code != 200:
r = requests.get('http://raspberry.site11.com/controller.php')
d=r.json()
dv1=d["Dv1"]
dv2=d["Dv2"]
dv3=d["Dv3"]
dv4=d["Dv4"]
if dv1=="1":
GPIO.output(6,0)
else:
GPIO.output(6,1)
if dv2=="1":
GPIO.output(13,0)
else:
GPIO.output(13,1)
if dv3=="1":
GPIO.output(19,0)
else:
GPIO.output(19,1)
if dv4=="1":
GPIO.output(26,0)
else:
GPIO.output(26,1)
time.sleep(t)
And this is error I was recived:
Unhandled exception in thread started by <function get at 0xb6506930>
Traceback (most recent call last):
File "thread.py", line 33, in get
d=r.json()
File "/usr/lib/python2.7/dist-packages/requests/models.py", line 793, in json
return json.loads(self.text, **kwargs)
File "/usr/lib/python2.7/json/__init__.py", line 338, in loads
return _default_decoder.decode(s)
File "/usr/lib/python2.7/json/decoder.py", line 369, in decode
raise ValueError(errmsg("Extra data", s, end, len(s)))
ValueError: Extra data: line 1 column 4 - line 1 column 10 (char 3 - 9)
May everyone help me, thanks all so much.
I think you can do
def get_json(data):
try:
return data.json()
except ValueError:
return False
Then
def get(t):
while 1:
r = requests.get('http://raspberry.site11.com/controller.php')
while not get_json(r):
r = requests.get('http://raspberry.site11.com/controller.php')
d = get_json(r)
Related
I am trying to get data from the below seen URLs
import requests
import json
service_a = requests.get('https:xxxxx')
service_b = requests.get('https:xxxxx')
service_a.json()
service_b.json()
While for the case of service_a, service_a.json() gives me the expected data, for service_b, the call service_b.json() gives me the following error:
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "/usr/lib/python3/dist-packages/requests/models.py", line 892, in json
return complexjson.loads(self.text, **kwargs)
File "/usr/lib/python3/dist-packages/simplejson/__init__.py", line 518, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 370, in decode
obj, end = self.raw_decode(s)
File "/usr/lib/python3/dist-packages/simplejson/decoder.py", line 400, in raw_decode
return self.scan_once(s, idx=_w(s, idx).end())
File "/usr/lib/python3/dist-packages/simplejson/scanner.py", line 79, in scan_once
return _scan_once(string, idx)
File "/usr/lib/python3/dist-packages/simplejson/scanner.py", line 70, in _scan_once
raise JSONDecodeError(errmsg, string, idx)
simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Why is this happening?
ps. I notced that when I try to visit the https link of service_a, the data it contains are shown in the browser, while for sevice_b, no data are shown in the browser, but when I click it a file with its data is downloaded in my computer.
Try to check the Content-Type of service_b response.
import csv
response = requests.get(API_URL)
decoded_data = response.content.decode("utf-8")
csv_data = csv.reader(decoded_data.splitlines(), delimiters=",")
csv_list = list(csv_data)
in most cases your
JSONDecodeError: Expecting value: line 1 column 1 (char 0)
error is due to:
non-JSON conforming quoting
XML/HTML output (that is, a string starting with <), or
incompatible character encoding
I've looked for similar questions, but I couldn't find anything what can help me. When i execute this code (which should work) I get "json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)"
TOKEN = '_EXAMPLE_TOKEN_'
def getPagedData(start,size):
url = 'http://EXAMPLE_URL'.format(start, size)
response = requests.get(url, headers={'token': TOKEN}, verify=False)
return response
def getDataInBatches():
start = 0
size = 1000
allData = []
data = getPagedData(start, size)
allData.extend(data.json()['data'])
total = data.json()['count']
for i in range(1, round(total/size) + 1):
print(allData[len(allData) - 1])
allData.extend(getPagedData(i * size, size).json()['data'])
return allData
print(getDataInBatches())
Here full traceback:
Traceback (most recent call last):
File "C:/Users/userxy/Documents/workProject/API/Get_Information.py", line 30, in <module>
print(getDataInBatches())
File "C:/Users/userxy/Documents/workProject/API/Get_Information.py", line 23, in getDataInBatches
allData.extend(data.json()['data'])
File "C:\Users\userxy\Anaconda3\lib\site-packages\requests\models.py", line 898, in json
return complexjson.loads(self.text, **kwargs)
File "C:\Users\userxy\Anaconda3\lib\json\__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "C:\Users\userxy\Anaconda3\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\userxy\Anaconda3\lib\json\decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
What can it be?
Make sure the response is in json format. The standard way to ensure the response is in json format is to add a headers={'Content-Type': 'application/json'}
But to be sure, check the documentation of an API you're accessing.
I tried many different ways to fix this error, but it failed.
Could you fix this error?
p.s.
formatted_url --> There's a problem that comma turns %2c when it calls request, and this way works.
https://opendart.fss.or.kr/api/fnlttMultiAcnt.json?crtfc_key=cd155118bef391723b04f8879878921476a97b9f&corp_code=01015160,00288495,00365590,01035678,00625942,00287812,00220622,00173351,00186559,00476036,00899556&bsns_year=2020&reprt_code=11013
request post --> params text is really long, and request get method occurs 414 error, so I changed it to post.
print(resp) :
<urllib3.response.HTTPResponse object at 0x7fb822c79ee0>
params.items() :
dict_items([('crtfc_key', 'cd155118bef391723b04f8879878921476a97b9f'), ('corp_code', '00260985,...........00173351,00186559,00476036,00899556'), ('bsns_year', '2020'), ('reprt_code', '11013')])
error message
enter image description here
def convertFnltt(url, items, item_names, params):
formatted_url = url + '?' + '&'.join(["{}={}".format(k,v) for k,v in params.items()])
print(formatted_url)
http = urllib3.PoolManager(ca_certs=certifi.where())
resp = http.request('POST',formatted_url)
print(resp)
print('1')
json_dict = json.load(resp) <--here!!
print('2')
data = []
#if self.get_status(json_dict['status']):
print(json_dict['status'])
if json_dict['status']=='000':
for line in json_dict['list']:
data.append([])
for itm in items:
if itm in line.keys():
data[-1].append(line[itm])
else:
data[-1].append('')
else:
return json_dict['status']
df = pd.DataFrame(data)
df.columns = item_names
return df #df.to_excel('data.xlsx')
1
Traceback (most recent call last):
File "main.py", line 106, in <module>
print(get_fnlttMultiAcnt(crtfc_key, get_corpcode(crtfc_key), bsns_year, reprt_code))
File "main.py", line 103, in get_fnlttMultiAcnt
return convertFnltt(url,items,item_names,params)
File "main.py", line 61, in convertFnltt
json_dict = json.load(resp)
File "/usr/lib/python3.8/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
It was the issue of the format of 'params'.
I used 'append' to make 'params', which caused a series of problems.
I changed 'append' to 'extend' , and it worked.
res = requests.get(url, params) <---'params format issue'
json_dict = json.loads(res.text)
data = []
for line in json_dict["list"]:
data.append([])
for itm in items:
if itm in line.keys():
data[-1].append(line[itm])
else:
data[-1].append('')
dfp = pd.DataFrame(data, columns = item_names)
Version: Python 2.7.10.
I have the following:
r = requests.post(url = API_ENDPOINT, headers = headers, data = data)
print(r.text)
print(type(r.text))
print(r.text[2])
The output
{"type":"quote","symbol":"SPY","bid":266.52,"bidsz":5,"bidexch":"P","biddate":"1513040398000","ask":266.55,"asksz":100,"askexch":"P","askdate":"1513040399000"}
{"type":"trade","symbol":"SPY","exch":"P","price":"266.31","size":"0","cvol":"83077533","date":"1513040400000","last":"266.31"}
{"type":"summary","symbol":"SPY","open":"265.58","high":"266.38","low":"265.4793","prevClose":"265.51","close":"266.31"}
<type 'unicode'>
b
I would like to get the output "SPY".
I added the following:
new = simplejson.loads(r.text)
print(new)
Now, I get the following:
Traceback (most recent call last):
File "example.py", line 63, in <module>
new = simplejson.loads(r.text)
File "/Library/Python/2.7/site-packages/simplejson/__init__.py",
line 518, in loads
return _default_decoder.decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py",
line 373, in decode
raise JSONDecodeError("Extra data", s, end, len(s))
simplejson.errors.JSONDecodeError: Extra data: line 1 column 160 -
line 1 column 407 (char 159 - 406)
I changed:
r = requests.post(url = API_ENDPOINT, headers = headers, data =
data).json()
Now, I get:
Traceback (most recent call last):
File "example.py", line 51, in <module>
r = requests.post(url = API_ENDPOINT, headers = headers, data =
data).json()
File "/Library/Python/2.7/site-packages/requests/models.py", line
884, in json
self.content.decode(encoding), **kwargs
File "/Library/Python/2.7/site-packages/simplejson/__init__.py",
line 518, in loads
return _default_decoder.decode(s)
File "/Library/Python/2.7/site-packages/simplejson/decoder.py",
line 373, in decode
raise JSONDecodeError("Extra data", s, end, len(s))
simplejson.errors.JSONDecodeError: Extra data: line 1 column 160 -
line 1 column 407 (char 159 - 406)
I added:
new = json.dumps(r.text)
print(new)
print(type(new))
Now, output is:
"{\"type\":\"quote\",\"symbol\":\"SPY\",\"bid\":266.52,\"bidsz\":5,\"bidexch\":\"P\",\"biddate\":\"1513040398000\",\"ask\":266.55,\"asksz\":100,\"askexch\":\"P\",\"askdate\":\"1513040399000\"}
{\"type\":\"trade\",\"symbol\":\"SPY\",\"exch\":\"P\",\"price\":\"266.31\",\"size\":\"0\",\"cvol\":\"83077533\",\"date\":\"1513040400000\",\"last\":\"266.31\"}
{\"type\":\"summary\",\"symbol\":\"SPY\",\"open\":\"265.58\",\"high\":\"266.38\",\"low\":\"265.4793\",\"prevClose\":\"265.51\",\"close\":\"266.31\"}"
<type 'str'>
If I do:
for line in r.text.splitlines():
d = json.loads(line)
I get:
Traceback (most recent call last):
File "example.py", line 54, in <module>
d = json.loads(line)
File "/System/Library/Frameworks/Python.
framework/Versions/2.7/lib/python2.7/json/__init__.py", line 338, in
loads
return _default_decoder.decode(s)
File "/System/Library/Frameworks/Python.framework/
Versions/2.7/lib/python2.7/json/decoder.py", line 369, in decode
raise ValueError(errmsg("Extra data", s, end, len(s)))
ValueError: Extra data: line 1 column 158 - line 1 column 404
(char 157 - 403)
So I tried:
for line in r.text.splitlines():
print(line)
print("\n\n")
And I got:
{"type":"quote","symbol":"SPY","bid":267.18,"bidsz":1,"bidexch":"P","biddate":"1513213200000","ask":267.22,"asksz":3,"askexch":"P","askdate":"1513213200000"}
{"type":"quote","symbol":"SPY","bid":267.18,"bidsz":1,"bidexch":"P","biddate":"1513213200000","ask":267.22,"asksz":3,"askexch":"P","askdate":"1513213200000"}
So even though there should be two lines, it interprets everything as a single line.
How can I convert r.text to a dictionary?
You should be using this .json():
r.json()
I believe what you are receiving is JSON Lines format. Each line of the response is a single json string. I simulated the response:
import json
class r: pass
r.text = u'''\
{"type":"quote","symbol":"SPY","bid":266.52,"bidsz":5,"bidexch":"P","biddate":"1513040398000","ask":266.55,"asksz":100,"askexch":"P","askdate":"1513040399000"}
{"type":"trade","symbol":"SPY","exch":"P","price":"266.31","size":"0","cvol":"83077533","date":"1513040400000","last":"266.31"}
{"type":"summary","symbol":"SPY","open":"265.58","high":"266.38","low":"265.4793","prevClose":"265.51","close":"266.31"}
'''
print(r.text)
print(type(r.text))
print(r.text[2])
# Parse JSON a line at a time:
for line in r.text.splitlines():
d = json.loads(line)
print d['symbol']
Output:
{"type":"quote","symbol":"SPY","bid":266.52,"bidsz":5,"bidexch":"P","biddate":"1513040398000","ask":266.55,"asksz":100,"askexch":"P","askdate":"1513040399000"}
{"type":"trade","symbol":"SPY","exch":"P","price":"266.31","size":"0","cvol":"83077533","date":"1513040400000","last":"266.31"}
{"type":"summary","symbol":"SPY","open":"265.58","high":"266.38","low":"265.4793","prevClose":"265.51","close":"266.31"}
<type 'unicode'>
t
SPY
SPY
SPY
I'm trying to run this code to create a data frame from a JSON link. Sometimes, the code will run. Other times, I will get an error message (below). I'm not sure why this occurs, even though the code is the same.
import requests
import json
url = "http://stats.nba.com/stats/leaguedashplayerstats?College=&Conference=&Country=&DateFrom=&DateTo=&Division=&DraftPick=&DraftYear=&GameScope=&GameSegment=&Height=&LastNGames=0&LeagueID=00&Location=&MeasureType=Advanced&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=Totals&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2016-17&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&VsConference=&VsDivision=&Weight="
jd = requests.get(url).json()
df = []
for item in requests.get(url).json()['resultSets']:
print("got here")
row_df = []
for row in item['rowSet']:
row_df.append(str(row).strip('[]'))
df.append("\n")
df.append(row_df)
print(df)
Error Message:
Traceback (most recent call last):
File "/Users/K/PycharmProjects/mousefun/fun", line 8, in <module>
jd = requests.get(url).json()
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/models.py", line 812, in json return complexjson.loads(self.text, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/json/__init__.py", line 318, in loads return _default_decoder.decode(s)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/json/decoder.py", line 343, in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/json/decoder.py", line 361, in raw_decode raise ValueError(errmsg("Expecting value", s, err.value)) from None
ValueError: Expecting value: line 1 column 1 (char 0)
Change your request logic to this and try again:
r = requests.get(url)
r.raise_for_status()
df = []
for item in r.json()["resultSets"]:
# ...
r.raise_for_status() will raise if the status is not OK .
Also, this does not do the request two times like your code does.