I want to convert HTTP GET response (I am using requests library) to python object. Here's my code:
# Full, pure, response
response = requests.get(url)
# Getting request data/content represented in byte array
content = response.content
# Byte array to string
data = content.decode('utf8')
# This line causes "ValueError: malformed node or string: <_ast.Name object at 0x7f35068be128>"
#data = ast.literal_eval(data)
# I tried this also but data is still string after those 2 lines
data = json.dumps(data)
data = json.loads(data, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
You can get the response as a dictionary using content = response.json(), and then pass that content to json.loads directly (This is assuming your response comes as a json)
# Full, pure, response
response = requests.get(url)
# Getting response as dictionary
content = response.json()
#Loading dictionary as json
data = json.loads(content, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
Related
I am currently trying to read out the locations of a company. The information about the locations is inside a script tag (json). So I read out the contet inside the corresponding script tag.
This is my code:
sauce = requests.get('https://www.ep.de/store-finder', verify=False, headers = {'User-Agent':'Mozilla/5.0'})
soup1 = BeautifulSoup(sauce.text, features="html.parser")
all_scripts = soup1.find_all('script')[6]
all_scripts.contents
The output is:
['\n\t\twindow.storeFinderComponent = {"center":{"lat":51.165691,"long":10.451526},"bounds":[[55.655085,5.160441],[46.439648,15.666775]],"stores":[{"code":"1238240","lat":51.411572,"long":10.425264,"name":"EP:Schulze","url":"/schulze-breitenworbis","showAsClosed":false,"isBusinessCard":false,"logoUrl":"https://cdn.prod.team-ec.com/logo/retailer/retailerlogo_epde_1238240.png","address":{"street":"Weststraße 6","zip":"37339","town":"Breitenworbis","phone":"+49 (36074) 31193"},"email":"info#ep-schulze-breitenworbis.de","openingHours":[{"day":"Mo.","openingTime":"09:00","closingTime":"18:00","startPauseTime":"13:00","endPauseTime":"14:30"},{"day":"Di.","openingTime":"09:00","closingTime":"18:00","startPauseTime":"13:00","endPauseTime":"14:30"},{"day":"Mi.","openingTime":"09:00","closingTime":"18:00","startPauseTime":"13:00","endPauseTime":"14:30"},...]
I have problems converting the content to a dictionary and reading all lat and long data.
When I try:
data = json.loads(all_scripts.get_text())
all_scripts.get_text() returns an empty list
So i tryed:
data = json.loads(all_scripts.contents)
But then i get an TypeError: the JSON object must be str, bytes or bytearray, not list
I dont know ho to convert the .content method to json:
data = json.loads(str(all_scripts.contents))
JSONDecodeError: Expecting value: line 1 column 2 (char 1)
Can anyone help me?
You could use regex to pull out the json and read that in.
import requests
import re
import json
html = requests.get('https://www.ep.de/store-finder', verify=False, headers = {'User-Agent':'Mozilla/5.0'}).text
pattern = re.compile('window\.storeFinderComponent = ({.*})')
result = pattern.search(html).groups(1)[0]
jsonData = json.loads(result)
You can removed first part of data and then last character of data and then load data to json
import json
data=all_scripts.contents[0]
removed_data=data.replace("\n\t\twindow.storeFinderComponent = ","")
clean_data=data[:-3]
json_data=json.loads(clean_data)
Output:
{'center': {'lat': 51.165691, 'long': 10.451526},
'bounds': [[55.655085, 5.160441], [46.439648, 15.666775]],
'stores': [{'code': '1238240',
'lat': 51.411572,
....
I'm making some data visualization from movies database api and I already access the data in the normal way but when i load the json data and for loop to print it, the data that out is just the column but I need to access the object inside.
url = "https://api.themoviedb.org/3/discover/movie?api_key="+ api_key
+"&language=en- US&sort_by=popularity.desc&include_adult=
false&include_video=false&page=1" # api url
response = urllib.request.urlopen(url)
raw_json = response.read().decode("utf-8")
data = json.loads(raw_json)
for j in data:
print(j)
i expect the output would be
[{'popularity': 15,
'id': 611,
'video': False,
'vote_count': 1403,
'vote_average': 8.9,
'title': 'lalalalo'},{....}]
but the actual output is
page
total_results
total_pages
results
The results are one level down. You are looping through the metadata.
Try changing your code to
import json
import urllib.request
api_key = "your api code"
url = "https://api.themoviedb.org/3/discover/movie?api_key=" + api_key +"&language=en- US&sort_by=popularity.desc&include_adult=false&include_video=false&page=1" # api url
response = urllib.request.urlopen(url)
raw_json = response.read().decode("utf-8")
data = json.loads(raw_json)
for j in data['results']:
print(j)
You need to change
data
to
data['results']
you can simply use requests module...
import requests
import json
your_link = " "
r = requests.get(your_link)
data = json.loads(r.content)
You shall have the json loaded up, then use your key "results" ["results"] and loop through the data you got.
I got an error : JSONDecodeError: Expecting value: line 1 column 1 (char 0). But don't understand why.
Here is my code :
import json
import urllib.request
url = "apiurl"
data = json.loads(url)
# Open the URL as Browser, not as python urllib
page = urllib.request.Request(url,headers={'User-Agent': 'Mozilla/5.0'})
infile = urllib.request.urlopen(page).read()
data = infile.decode('ISO-8859-1') # Read the content as string decoded with ISO-8859-1
command_obj = {x['command']: x for x in data}
with open('new_command.json', 'w') as f:
json.dump(command_obj, f, indent=2)
With this fonction, i'm just trying to fetch data from an api and modify its format. Thanks for your help
You're trying to read the URL itself (and not its content) as JSON:
data = json.loads(url)
... instead you want to read the content returned from the API as JSON:
# Open the URL as Browser, not as python urllib
page = urllib.request.Request(url,headers={'User-Agent': 'Mozilla/5.0'})
infile = urllib.request.urlopen(page).read()
data = infile.decode('ISO-8859-1')
# avoid re-using `data` variable name
json_data = json.loads(data)
However, be aware that JSON should always be returned as UTF-8, never as ISO-8859-1 / latin-1.
I am trying to get a certain value in a string of json but I can't figure out how exactly to do it. I don't want to convert it into a string and strip / replace the unwanted pieces because then I won't be able to get the other values. My current code is:
username = "Dextication"
url = f"https://minecraft-statistic.net/api/player/info/{username}/"
response = requests.get(url)
json_data = json.loads(response.text)
print(json_data)
Edit:
when I run this, json.data = "{"status":"ok","data":{"online":0,"total_time_play":46990,"last_play":1513960562,"license":1,"name":"Dextication","uuid":"74d57a754855410c90b3d51bc99b8beb"}}"
I would like to only print the value: 46990
Try below code
import json, requests
username = "Dextication"
url = f"https://minecraft-statistic.net/api/player/info/{username}/"
response = requests.get(url)
json_data = json.loads(response.text)
result = json_data['data']['total_time_play']
print (result)
I am trying to send and receive a png image via Post.
I am trying to send like this:
with bytes as <class 'bytes'> datatype
def call(videoid, count, bytes):
print(videoid)
opener = request.build_opener()
param = {'videoid' : '{0}'.format(videoid), 'count':'{0}'.format(count), 'bytes':'{0}'.format(bytes)}
data=json.dumps(param).encode('utf8')
req = request.Request(os.environ['ImageProcessURL'], data=data, headers={'Content-Type': 'application/json'})
response = opener.open(req)
And the receiving part looks like this:
postreqdata = json.loads(open(os.environ['req']).read())
videoid = postreqdata['videoid']
count = postreqdata['count']
bytesstr = postreqdata['bytes']
But here bytes is now a string.
How can I convert it back to the original bytes type?