I am extracting data through POST method and let's say we have different post data in a list and for each request there will be a different post data like this:
cookies = {
"_sp_id.cf1a": "205e16a5-8970-4c92-97b8-969eebfcbb63.1647289721.7.1648545896.1648465133.73852927-e047-4c36-bae7-90c001509900",
"_sp_ses.cf1a": "*",
}
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
"Accept": "text/plain, */*; q=0.01",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"Referer": "https://www.tradingview.com/",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Origin": "https://www.tradingview.com",
"Connection": "keep-alive",
# Requests sorts cookies= alphabetically
# 'Cookie': '_sp_id.cf1a=205e16a5-8970-4c92-97b8-969eebfcbb63.1647289721.7.1648545896.1648465133.73852927-e047-4c36-bae7-90c001509900; _sp_ses.cf1a=*',
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-site",
"Sec-GPC": "1",
# Requests doesn't support trailers
# 'TE': 'trailers',
}
data = [
'{"filter":[{"left":"name","operation":"nempty"},{"left":"exchange","operation":"in_range","right":["AMEX","NASDAQ","NYSE"]},{"left":"High.All","operation":"eless","right":"high"},{"left":"is_primary","operation":"equal","right":true},{"left":"subtype","operation":"nequal","right":"preferred"}],"options":{"lang":"en","active_symbols_only":true},"markets":["america"],"symbols":{"query":{"types":[]},"tickers":[]},"columns":["logoid","name","close","change","change_abs","Recommend.All","volume","Value.Traded","market_cap_basic","price_earnings_ttm","earnings_per_share_basic_ttm","number_of_employees","sector","description","type","subtype","update_mode","pricescale","minmov","fractional","minmove2","currency","fundamental_currency_code"],"sort":{"sortBy":"name","sortOrder":"asc"},"range":[0,150]}',
'{"filter":[{"left":"name","operation":"nempty"},{"left":"exchange","operation":"in_range","right":["AMEX","NASDAQ","NYSE"]},{"left":"High.All","operation":"eless","right":"high"},{"left":"is_primary","operation":"equal","right":true},{"left":"subtype","operation":"nequal","right":"preferred"}],"options":{"lang":"en","active_symbols_only":true},"markets":["america"],"symbols":{"query":{"types":[]},"tickers":[]},"columns":["logoid","name","change|1","change|5","change|15","change|60","change|240","change","change|1W","change|1M","Perf.3M","Perf.6M","Perf.YTD","Perf.Y","beta_1_year","Volatility.D","description","type","subtype","update_mode","currency","fundamental_currency_code"],"sort":{"sortBy":"name","sortOrder":"asc"},"range":[0,150]}',
]
with httpx.Client() as client:
for d in data:
r = client.post(
"https://scanner.tradingview.com/america/scan",
headers=headers,
cookies=cookies,
data=d,
)
I store post data in a list and iterating over it and passing it in request object, so far so good. Post requests returns json object and problem begins here, every request returns different json object except keys, they are the same for e.g
'{"filter":[{"left":"name","operation":"nempty"},{"left":"exchange","operation":"in_range","right":["AMEX","NASDAQ","NYSE"]},{"left":"High.All","operation":"eless","right":"high"},{"left":"is_primary","operation":"equal","right":true},{"left":"subtype","operation":"nequal","right":"preferred"}],"options":{"lang":"en","active_symbols_only":true},"markets":["america"],"symbols":{"query":{"types":[]},"tickers":[]},"columns":["logoid","name","change|1","change|5","change|15","change|60","change|240","change","change|1W","change|1M","Perf.3M","Perf.6M","Perf.YTD","Perf.Y","beta_1_year","Volatility.D","description","type","subtype","update_mode","currency","fundamental_currency_code"],"sort":{"sortBy":"name","sortOrder":"asc"},"range":[0,150]}'
this post data returns this json:
{
"totalCount": 141,
"data": [
{
"s": "NYSE:ABBV",
"d": [
"abbvie",
"ABBV",
0,
0.13602918,
0.2662209,
0.37497288,
0.6400696,
0.39670241,
0.39670241,
9.60952832,
20.52236029,
48.81477398,
19.62333826,
51.75676942,
0.5128781,
1.56710337,
"AbbVie Inc.",
"stock",
"common",
"delayed_streaming_900",
"USD",
"USD"
]
}
]
}
and post data
{"filter":[{"left":"name","operation":"nempty"},{"left":"exchange","operation":"in_range","right":["AMEX","NASDAQ","NYSE"]},{"left":"High.All","operation":"eless","right":"high"},{"left":"is_primary","operation":"equal","right":true},{"left":"subtype","operation":"nequal","right":"preferred"}],"options":{"lang":"en","active_symbols_only":true},"markets":["america"],"symbols":{"query":{"types":[]},"tickers":[]},"columns":["logoid","name","close","change","change_abs","Recommend.All","volume","Value.Traded","market_cap_basic","price_earnings_ttm","earnings_per_share_basic_ttm","number_of_employees","sector","description","type","subtype","update_mode","pricescale","minmov","fractional","minmove2","currency","fundamental_currency_code"],"sort":{"sortBy":"name","sortOrder":"asc"},"range":[0,150]}'
returns this which is similar to the last one i.e. keys but values are different
{
"totalCount": 141,
"data": [
{
"s": "NYSE:ABBV",
"d": [
"abbvie",
"ABBV",
161.97,
0.39670241,
0.64,
0.42121212,
4516453,
731529892.41,
286085169370,
25.01356652,
6.4775,
50000,
"Health Technology",
"AbbVie Inc.",
"stock",
"common",
"delayed_streaming_900",
100,
1,
"false",
0,
"USD",
"USD"
]
}
]
}
Hence I am getting just one json object which is the last one in the loop instead of both.
Desired Output:
{
"overview": {
"totalCount": 141,
"data": [
{
"s": "NYSE:ABBV",
"d": [
"abbvie",
"ABBV",
161.97,
0.39670241,
0.64,
0.42121212,
4516453,
731529892.41,
286085169370,
25.01356652,
6.4775,
50000,
"Health Technology",
"AbbVie Inc.",
"stock",
"common",
"delayed_streaming_900",
100,
1,
"false",
0,
"USD",
"USD"
]
}
]
},
"performance": {
"totalCount": 141,
"data": [
{
"s": "NYSE:ABBV",
"d": [
"abbvie",
"ABBV",
0,
0.13602918,
0.2662209,
0.37497288,
0.6400696,
0.39670241,
0.39670241,
9.60952832,
20.52236029,
48.81477398,
19.62333826,
51.75676942,
0.5128781,
1.56710337,
"AbbVie Inc.",
"stock",
"common",
"delayed_streaming_900",
"USD",
"USD"
]
}
]
}
}
I am looking for some solution to handle this kind of duplication in json response and get data from different post data on each request by making the duplicate keys to unique.
headers = {
"User-Agent": "Mozilla/5.0",
'accept': 'application/json',
'Content-Type': 'application/json',
}
with open("jsonattempt.txt","r") as f:
data = f.read()
json_data = "'" + data + "'"
response = requests.post('https://www.pathofexile.com/api/trade/search/Standard', headers=headers, data=json_data)
print(response)
Generally, there is a curl request like this:
curl -X 'POST' \
'https://www.pathofexile.com/api/trade/search/Standard' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"query": {
"status": {
"option": "online"
},
"type": "Turquoise Amulet",
"stats": [
{
"type": "and",
"filters": [
{
"id": "pseudo.pseudo_total_mana",
"value": {
"min": 47,
"max": 49
},
"disabled": false
}
]
}
]
},
"sort": {
"price": "asc"
}
}'
Which returns a bunch of unnecessary things.
My json_data variable and jsonattempt.txt is the same as -d parameter, I add ' ' to start and to end:
{
"query": {
"status": {
"option": "online"
},
"type": "Turquoise Amulet",
"stats": [
{
"type": "and",
"filters": [
{
"id": "pseudo.pseudo_total_mana",
"value": {
"min": 47,
"max": 49
},
"disabled": false
}
]
}
]
},
"sort": {
"price": "asc"
}
}
I convert curl request to python which is the code on the top, I add the data as json_data and yeet the post request but keep getting 400 Bad Request.
Request 401 is Unauthorized AFAIK, so I dont need an OAuth2 key for this. How can I insert my json file appropiately to the request?
(Same exact json works on https://app.swaggerhub.com/apis-docs/Chuanhsing/poe/1.0.0#/Trade/get_api_trade_fetch__items_ I am just asking how to insert my json file to requests.post correctly.)
Why are you adding quotes around the JSON content? That doesn't make any sense. Those quotes aren't part of your curl request. If you just write...
import requests
headers = {
"User-Agent": "Mozilla/5.0",
"accept": "application/json",
"Content-Type": "application/json",
}
with open("jsonattempt.txt", "r") as f:
data = f.read()
response = requests.post(
"https://www.pathofexile.com/api/trade/search/Standard",
headers=headers,
data=data,
)
print(response)
...it works as expected.
And in fact you can further simplify that; you don't need to read in the file data yourself, you can let requests do that:
import requests
headers = {
"User-Agent": "Mozilla/5.0",
"accept": "application/json",
"Content-Type": "application/json",
}
with open("jsonattempt.txt", "r") as f:
response = requests.post(
"https://www.pathofexile.com/api/trade/search/Standard",
headers=headers,
data=f,
)
print(response)
import json
# some codes here
with open("jsonattempt.txt","r") as f:
data = f.read()
json_data = json.loads(data)
# rest of codes here
Your request requires json type data, while you are passing a string. The json.loads method converts string to json. Try it out.
I'm trying to translate Curl to Python, but I'm getting something wrong. Please help.
CURL:
body=$(cat << EOF
{
"order": {
"units": "-100",
"instrument": "EUR_USD",
"timeInForce": "FOK",
"type": "MARKET",
"positionFill": "DEFAULT"
}
}
EOF
)
curl \
-X POST \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <AUTHENTICATION TOKEN>" \
-d "$body" \
"https://api-fxtrade.oanda.com/v3/accounts/<ACCOUNT>/orders"
PYTHON:
import requests
import json
def market_buy():
header = {"Accept": "application/json",
"Authorization": "Bearer <my auth code>"
}
data = {
"order": {
"units": "100",
"instrument": "EUR_USD",
"timeInForce": "FOK",
"type": "MARKET",
"positionFill": "DEFAULT"
}
}
url = "https://api-fxtrade.oanda.com/v3/accounts/<myaccount>/orders"
r = requests.post(url, data=data, headers=header)
print(r.text)
market_buy()
Error Message:
{"errorMessage":"Insufficient authorization to perform request."}
I've double-checked my credentials. I'm thinking something is wrong with the code.
A direct translation from cURL to Python requests would be this:
from requests import post
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer <AUTHENTICATION TOKEN>",
}
data = {
"order": {
"units": "100",
"instrument": "EUR_USD",
"timeInForce": "FOK",
"type": "MARKET",
"positionFill": "DEFAULT",
}
}
post(
"https://api-fxtrade.oanda.com/v3/accounts/<myaccount>/orders",
headers=headers,
data=data,
)
I guess you are missing the SSL cert verification and basic authentication.
You can turn off SSL cert verification with the verify flag, and use basic authentication by specifying auth.
from requests.auth import HTTPBasicAuth
import requests
import json
def market_buy():
header = {"Accept": "application/json",
"Authorization": "Bearer <my auth code>"
}
data = {
"order": {
"units": "100",
"instrument": "EUR_USD",
"timeInForce": "FOK",
"type": "MARKET",
"positionFill": "DEFAULT"
}
}
url = "https://api-fxtrade.oanda.com/v3/accounts/<myaccount>/orders"
r = requests.post(url, data=data, headers=header, auth=HTTPBasicAuth('admin', 'admin'), verify=False)
print(r.text)
market_buy()
I guess the above should work.
I am working on REST API of a site that requires this request type when I want to upload a file:
'Authorization' and multi-part content type in header
File as binary string in form (body)
File Type in request URL
So I did this code:
import requests
url = 'http://httpbin.org/post'
parameters = {
'format': 'pdf',
}
headers = {
'Content-Type': 'multipart/form-data',
'Accept': 'application/json',
'Authorization' : 'Some authorization code'
}
data = {'file': open('1.pdf', 'rb')}
r = requests.post(url, params=parameters, headers=headers, data=data)
print(r.text)
BUT it seems to requests is dropping data :
{
"args": {
"format": "pdf"
},
"data": "",
"files": {},
"form": {},
"headers": {
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate",
"Authorization": "Some authorization code",
"Connection": "close",
"Content-Length": "30",
"Content-Type": "multipart/form-data",
"Host": "httpbin.org",
"User-Agent": "python-requests/2.18.1"
},
"json": null,
"origin": "x.x.x.x",
"url": "http://httpbin.org/post?format=pdf"
}
it works when I remove 'headers' part in request:
r = requests.post(url, params=parameters, data=data)
Because response is :
{
"args": {
"format": "pdf"
},
"data": "",
"files": {},
"form": {
"fax_file": "some samplae texts\n"
},
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate",
"Connection": "close",
"Content-Length": "30",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "httpbin.org",
"User-Agent": "python-requests/2.18.1"
},
"json": null,
"origin": "x.x.x.x",
"url": "http://httpbin.org/post?format=pdf"
}
I have tried prepared request too and result is same.
You are trying to post file data, so use the files option:
r = requests.post(url, params=parameters, files=data, headers=headers)
You should really not set the Content-Type header, however; it is set for you when you use the files option. The header in this case includes the field boundary, to really want the library to take care of this for you:
headers = {
'Accept': 'application/json',
'Authorization' : 'Some authorization code'
}
If you leave the Content-Type header in, you’d have to generate the content body up front to be able to supply the required boundary info to the receiving server.
You could experiment with dropping the Accept header too; by default requests will add Accept: */* if you don't specify that header, signalling that anything is acceptable.
When using only the data argument, the parameters are encoding to application/x-www-form-urlencoded form, which doesn't support large file data, and your Content-Type header doesn't match the actual POST body content.
See Post a Multipart-Encoded File in the requests documentation and application/x-www-form-urlencoded or multipart/form-data? here on Stack Overflow.
Demo:
>>> import requests
>>> url = 'http://httpbin.org/post'
>>> parameters = {'format': 'pdf'}
>>> headers = {
... 'Accept': 'application/json',
... 'Authorization' : 'Some authorization code',
... }
>>> data = {'file': open('1.pdf', 'rb')}
>>> r = requests.post(url, params=parameters, files=data, headers=headers)
>>> print(r.text)
{
"args": {
"format": "pdf"
},
"data": "",
"files": {
"file": "<file data as base64>"
},
"form": {},
"headers": {
"Accept": "application/json",
"Accept-Encoding": "gzip, deflate",
"Authorization": "Some authorization code",
"Cache-Control": "max-age=0",
"Connection": "close",
"Content-Length": "374751",
"Content-Type": "multipart/form-data; boundary=d4b84f8bfd464e3f97e3de584d7315fc",
"Host": "httpbin.org",
"O2Gw-Id": "03",
"User-Agent": "python-requests/2.18.4",
"X-Gateway": "wap.london.02.net"
},
"json": null,
"origin": "10.120.6.78, 82.132.221.209",
"url": "http://httpbin.org/post?format=pdf"
}
Note the multipart/form-data; boundary=d4b84f8bfd464e3f97e3de584d7315fc value for the Content-Type header!