ok so im using python 3
i was able to get the data of the api using print(endpoint.json())
but i want to make it readable with pandas, so i can iterate through it easier.
this is the code (keep in mind i discarded my own api key and im using rapid api as a resource (specificly the movie database)
import requests
import json
import pandas
url = "https://movies-tvshows-data-imdb.p.rapidapi.com/"
querystring = {"type":"get-popular-movies","page":"1","year":"2020"}
headers = {
'x-rapidapi-host': "movies-tvshows-data-imdb.p.rapidapi.com",
'x-rapidapi-key': my key
}
response = requests.request("GET", url, headers=headers, params=querystring)
data=response.json()
df=pandas.read_json(data)
print(df)
i get this error
Traceback (most recent call last):
File "c:\Users\Home\Documents\studying\newproject\newproject.py", line 15, in <module>
df=pandas.read_json(data)
File "C:\Users\Home\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\util\_decorators.py", line 199, in wrapper
return func(*args, **kwargs)
File "C:\Users\Home\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\util\_decorators.py", line 296, in wrapper
return func(*args, **kwargs)
File "C:\Users\Home\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\json\_json.py", line 593, in read_json
filepath_or_buffer, _, compression, should_close = get_filepath_or_buffer(
File "C:\Users\Home\AppData\Local\Programs\Python\Python38\lib\site-packages\pandas\io\common.py", line 243, in get_filepath_or_buffer
raise ValueError(msg)
ValueError: Invalid file path or buffer object type: <class 'dict'>
In your case data is a dict.
So, try with:
pandas.DataFrame.from_dict(data)
Related
Hi I am trying to a hit an API using requests module of python. The Api has to be hit 20000 times as the number of pages are around 20000. In every hit the data comes around 10 mb. By the end of the process it creates a json file of around 100gb. Here is the code I have written
with open('file.json','wb',buffering=100*1048567) as f:
while(next_page_cursor != ""):
with request.get(url,headers=headers) as response:
json_response = json.loads(response.content.decode('utf-8'))
"""
json response looks something like this
{
content:[{},{},{}........50 dictionaries]
next_page_cursor : "abcd"
}
"""
next_page_cursor = json_response['next_page_cursor']
for data in json_response['content']:
f.write((json.dumps(data) + "\n").encode())
But after running successfully for few pages the code fails giving the below error:
Traceback (most recent call last):
File "<command-1206920060120926>", line 65, in <module>
with requests.get(data_url, headers = headers) as response:
File "/databricks/python/lib/python3.7/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/databricks/python/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/databricks/python/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/databricks/python/lib/python3.7/site-packages/requests/sessions.py", line 686, in send
r.content
File "/databricks/python/lib/python3.7/site-packages/requests/models.py", line 828, in content
self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b''
File "/databricks/python/lib/python3.7/site-packages/requests/models.py", line 753, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: OSError("(104, \'ECONNRESET\')")', OSError("(104, 'ECONNRESET')"))
you need to use response.iter_content
https://2.python-requests.org/en/master/api/#requests.Response.iter_content
I'm pulling data from my database and trying to pass it my web app via the API using python requests. I've done this several times, but I'm having a hard time on this script.
It gives me "ValueError: not enough values to unpack (expected 2, got 1)"
rpt_cursor = rpt_conn.cursor()
sql = """SELECT `field` FROM `db`.`table`;"""
rpt_cursor.execute(sql)
#Creating list of field names.
num_fields = len(rpt_cursor.description)
field_names = [i[0] for i in rpt_cursor.description]
# getting results and defining dict to load them into.
results = rpt_cursor.fetchall()
dictofemails = []
print('These are the SQL results....')
print(results) # These look fine.
# Appending data to the dict.
for row in results:
dictofdata.append(dict(zip(field_names,row)))
print('These are the dict results...')
print(dictofdata) # Once again this looks like a fine dict with a format of [{'field_name' : 'xyz'}, {'field_name' : 'abc'}].
api_request_url = 'https://api.domainname.com/api/list/' + str(target) +'/Api_Key/' + my_apikey
print('api_request_url') # This looks fine.
response = requests.put(api_request_url, headers=headers, data=dictofdata)
print(response)
print(response.content)
Any clues you guys could give me would be appreciated.
Edit:
Traceback as requested....
Traceback (most recent call last):
File "NGM_ListMaker_WeeklyBounceLoad.py", line 306, in <module>
response = requests.put(api_request_url, headers=headers, data=dictofemails)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\api.py", line 126, in put
return request('put', url, data=data, **kwargs)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\sessions.py", line 494, in request
prep = self.prepare_request(req)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\sessions.py", line 437, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\models.py", line 308, in prepare
self.prepare_body(data, files, json)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\models.py", line 499, in prepare_body
body = self._encode_params(data)
File "C:\Program Files (x86)\Python36-32\lib\site-packages\requests\models.py", line 97, in _encode_params
for k, vs in to_key_val_list(data):
ValueError: not enough values to unpack (expected 2, got 1)
Edit with the Answer:
The API was expecting a string instead of the list that I was passing it. Changing the API call to data=str(dictofdata) was all that was needed. Thanks for the help.
The API was expecting a string instead of the list I was trying to pass.
Change this...
response = requests.put(api_request_url, headers=headers, data=dictofdata)
to this...
response = requests.put(api_request_url, headers=headers, data=str(dictofdata))
Works fine now. Thanks everyone.
I'm trying to create a Python Wrapper for an upcoming API, so far I've been getting along well, but I keep bumping my head into the same problem. When making a POST request to add a mod to a game through the API, I feed the following dict item into the POST request
{'name': "Necro's Test Mod", 'name_id': None, 'summary': "This is a test mod submitted through the API using python's request library.", 'description': "I'm only about 43% sure this will actually work, whats more likely is that it will fail and return some messed up error with some even weirder error message before i am contacted by the mod.io developper saying they have personally revoked my api key (This isn't actually true i just need words", 'homepage': 'www.edain.wikia.com', 'metadata_blob': 'None', 'stock': 1, 'tags': ['cool', 'python', 'api'], 'logo': <_io.BufferedReader name='C:\\Users\\Clement\\Pictures\\Background\\7z6cSaI-lord-of-the-rings-wallpaper-hd.jpg'>}
And that dictionnary is added as follows
BASE_PATH = "https://api.test.mod.io/v1"
headers = {
'Authorization': 'Bearer ' + client.access_token,
'Content-Type': 'multipart/form-data',
'Accept': 'application/json'
}
r = requests.post(BASE_PATH + '/games/181/mods'.format(self.id), files = dict, headers = headers)
However it returns the following traceback, leading me to believe that the API returns a blank response
Traceback (most recent call last):
File "C:\Users\Clement\Desktop\mod.io\test.py", line 40, in <module>
new_mod = game.add_mod(newmod)
File "C:\Users\Clement\Desktop\mod.io\modio\game.py", line 83, in add_mod
r = requests.post(BASE_PATH + '/games/{}/mods'.format(self.id), files = mod.__dict__, headers = headers)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 494, in request
prep = self.prepare_request(req)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\sessions.py", line 437, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 308, in prepare
self.prepare_body(data, files, json)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 496, in prepare_body
(body, content_type) = self._encode_files(files, data)
File "C:\Users\Clement\AppData\Local\Programs\Python\Python36\lib\site-packages\requests\models.py", line 159, in _encode_files
fdata = fp.read()
AttributeError: 'NoneType' object has no attribute 'read'
Here's the documentation for the particular POST request I am attempting.
The files argument is meant for file uploads. You probably want to use data instead here.
r = requests.post(
BASE_PATH + '/games/181/mods'.format(self.id),
data = dict,
headers = headers
)
You get this error because requests expects the value of name_id to be a file pointer with a read() method. But in your payload it's None.
And don't use dict as a variable name.
macOS 10.12.3 python 2.7.13 requests 2.13.0
I use requests package to send post request.This request need to login before post data.So I use request.Session() and load a logined cookie.
Then I use this session to send post data in cycle mode.
It is no error that I used to run this code in Windows and Linux.
Simple Code:
s = request.Session()
s.cookies = cookieslib.LWPCookieJar('cookise')
s.cookies.load(ignore_discard=True)
for user_id in range(100,200):
url = 'http://xxxx'
data = { 'user': user_id, 'content': '123'}
r = s.post(url, data)
...
But the program frequently (about every interval) crash, the error isAttributeError: 'module' object has no attribute 'kqueue'
Traceback (most recent call last):
File "/Users/gasxia/Dev/Projects/TgbookSpider/kfz_send_msg.py", line 90, in send_msg
r = requests.post(url, data) # catch error if user isn't exist
File "/usr/local/lib/python2.7/site-packages/requests/sessions.py", line 535, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python2.7/site-packages/requests/sessions.py", line 488, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python2.7/site-packages/requests/sessions.py", line 609, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python2.7/site-packages/requests/adapters.py", line 423, in send
timeout=timeout
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 588, in urlopen
conn = self._get_conn(timeout=pool_timeout)
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 241, in _get_conn
if conn and is_connection_dropped(conn):
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/util/connection.py", line 27, in is_connection_dropped
return bool(wait_for_read(sock, timeout=0.0))
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 33, in wait_for_read
return _wait_for_io_events(socks, EVENT_READ, timeout)
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/util/wait.py", line 22, in _wait_for_io_events
with DefaultSelector() as selector:
File "/usr/local/lib/python2.7/site-packages/requests/packages/urllib3/util/selectors.py", line 431, in __init__
self._kqueue = select.kqueue()
AttributeError: 'module' object has no attribute 'kqueue'
This looks like a problem that commonly arises if you're using something like eventlet or gevent, both of which monkeypatch the select module. If you're using those to achieve asynchrony, you will need to ensure that those monkeypatches are applied before importing requests. This is a known bug, being tracked in this issue.
I'm trying to retrieve the active and completed items under a list of specified sellers. I've read a few things in a few different places, but haven't really been able to find something that I understand or anything I could work with. I think it might have something to do with my headers or maybe authentication or something, but I'm also uncertain if its just the way that I'm setting up my request. I have only ever used the Finding API, and it looks like I need to use Trading instead, but I'm not sure what little details I might be missing when I tried to switch API's
from ebaysdk.trading import Connection as trading
api = trading(appid='API_KEY_HERE', config_file=None)
api_request = {
'keywords': 'new'}
'outputSelector': 'SellerInfo',
'itemFilter': [
{'name': 'UserId',
'value': 'SELLER_NAME_HERE'},
{'name': 'StartTimeFrom',
'value': QueryTime_Start},
{'name': 'StartTimeTo',
'value': QueryTime_Complete}]}
response = api.execute('GetSellerList', api_request).
and my complete error message:
Traceback (most recent call last): File "C:\Users\dc500\Downloads\All
3.0\All 3\Python\3Pies\Project_Ordered Tools\practice_getsellerlist.py", line 24, in response =
api.execute('GetSellerList', api_request) File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\ebaysdk\connection.py",
line 120, in execute self.execute_request() File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\ebaysdk\connection.py",
line 184, in execute_request allow_redirects=True File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\requests\sessions.py",
line 576, in send r = adapter.send(request, **kwargs) File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\requests\adapters.py",
line 376, in send timeout=timeout File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\requests\packages\urllib3\connectionpool.py",
line 559, in urlopen body=body, headers=headers) File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\site-packages\requests\packages\urllib3\connectionpool.py",
line 353, in _make_request conn.request(method, url,
**httplib_request_kw) File "C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\http\client.py",
line 1084, in request self._send_request(method, url, body, headers)
File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\http\client.py",
line 1124, in _send_request self.putheader(hdr, value) File
"C:\Users\dc500\AppData\Local\Programs\Python\Python35\lib\http\client.py",
line 1060, in putheader if _is_illegal_header_value(values[i]):
TypeError: expected string or bytes-like object
from ebaysdk.trading import Connection as Trading
try:
api = Trading(config_file='ebay.yaml')
except ConnectionError as e:
print(e)
print(e.response.dict())
log.error('Attempting to get an API object failed with %s', e)
def get_active_listings(page):
acitvelist = api.execute('GetMyeBaySelling', {'ActiveList': True,
'DetailLevel': 'ReturnAll',
'PageNumber': page})
return acitvelist.dict()
This works for me and returns all active listings - maybe GetMyeBaySellingis a better call to use ?
Other than that
https://developer.ebay.com/devzone/XML/docs/Reference/eBay/GetSellerList.html#Input
does not seem to give the inputFilter as an option for parameters? there is outputSelector
??