Develop Reference

cURL request into Swift vs Python

I am trying to implement a cURL request into Swift App, however, the server is not accepting my request for some reason.
cURL Request:
curl 'https://test.com' \
-H 'Connection: keep-alive' \
-H 'Accept: application/json, text/javascript, */*; q=0.01' \
-H 'cache-control: no-cache' \
-H 'X-Requested-With: XMLHttpRequest' \
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36' \
-H 'Content-Type: application/json' \
-H 'Origin: https://yyy.com' \
-H 'Sec-Fetch-Site: same-origin' \
-H 'Sec-Fetch-Mode: cors' \
-H 'Sec-Fetch-Dest: empty' \
-H 'Referer: https://zzz.com' \
-H 'Accept-Language: en-GB,en-US;q=0.9,en;q=0.8' \
--data-binary '{"method":"NewUser","params":{"Username":"James","Password":"secret"}}' \
--compressed
I have managed to do this in Python and it worked:
import requests
url = "https://test.com"
headers = CaseInsensitiveDict
headers["Connection"] = "keep-alive"
headers["Accept"] = "application/json, text/javascript, */*; q=0.01"
headers["cache-control"] = "no-cache"
headers["X-Requested-With"] = "XMLHttpRequest"
headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
headers["Content-Type"] = "application/json"
headers["Origin"] = "https://yyy.com"
headers["Sec-Fetch-Site"] = "same-origin"
headers["Sec-Fetch-Mode"] = "cors"
headers["Sec-Fetch-Dest"] = "empty"
headers["Referer"] = "https://zzz"
headers["Accept-Language"] = "en-GB,en-US;q=0.9,en;q=0.8"
data = '{"method":"NewUser","params":{"Username":"James","Password":"Secret"}}'
raw_response = requests.post(url, headers=headers, data=data)
What I am trying to do in Swift is:
let url = URL(string: "https://test.com")!
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.addValue("application/json", forHTTPHeaderField: "Content-Type")
request.addValue("keep-alive", forHTTPHeaderField: "Connection")
request.addValue("application/json, text/javascript, */*; q=0.01", forHTTPHeaderField: "Accept")
request.addValue("no-cache", forHTTPHeaderField: "cache-control")
request.addValue("XMLHttpRequest", forHTTPHeaderField: "X-Requested-With")
request.addValue("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", forHTTPHeaderField: "User-Agent")
request.addValue("https://yyy.com", forHTTPHeaderField: "Origin")
request.addValue("same-origin", forHTTPHeaderField: "Sec-Fetch-Site")
request.addValue("cors", forHTTPHeaderField: "Sec-Fetch-Mode")
request.addValue("empty", forHTTPHeaderField: "Sec-Fetch-Dest")
request.addValue("https://zzz.com", forHTTPHeaderField: "Referer")
request.addValue("en-GB,en-US;q=0.9,en;q=0.8", forHTTPHeaderField: "Accept-Language")
let json: [String: Any] = ["method": "NewUser","params": ["Username":"James", "Password":"Secret"]]
let jsonData = try! JSONSerialization.data(withJSONObject: json, options: [])
request.httpBody = jsonData
let task = URLSession.shared.dataTask(with: url) {(data, response, error) in
guard let data = data else { return }
print(String(data: data, encoding: .utf8)!)
}
task.resume()
Any idea what is the difference between Python/Swift code that it's not working?
Thanks.

Get response from Curl but not Python Requests

I tried to run the Curl command in the terminal. I can get the response in JSON. But when I convert the Curl to Python Code and run the python script, the script returned an html page. It looks like it is a captcha page.
Why they can detect the difference? Any way I can bypass that?
curl --location --request GET 'https://www.99.co/api/v2/web/search/listings?query_type=city&property_segments=residential&listing_type=sale&rental_type=unit&page_size=1&page_num=100&zoom=11&show_future_mrts=true&show_cluster_preview=true&show_internal_linking=true' \
--header 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0' \
--header 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \
--header 'accept-language: en-US,en;q=0.5' \
--header 'if-none-match: W/"5cb28158470b6eb63b0db1f5dd2adda7d10c328c"' \
--header 'cookie: _xsrf=2|6be6c7ee|de091e966dba87e0b2c8637a63127715|1616479899' \
--header 'cookie: lotameid=undefined' \
--header 'cookie: started_since=1616480838' \
--header 'cookie: country=SG' \
--header 'cookie: nid=2de067d4-fed8-4c35-b7db-14f479997f8a' \
--header 'cookie: g_state={"i_p":1627551635034,"i_l":1}' \
--header 'upgrade-insecure-requests: 1' \
--header 'te: trailers' \
--header 'sec-gpc: 1'
import requests
url = "https://www.99.co/api/v2/web/search/listings?query_type=city&property_segments=residential&listing_type=sale&rental_type=unit&page_size=1&page_num=100&zoom=11&show_future_mrts=true&show_cluster_preview=true&show_internal_linking=true"
payload={}
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'accept-language': 'en-US,en;q=0.5',
'if-none-match': 'W/"5cb28158470b6eb63b0db1f5dd2adda7d10c328c"',
'cookie': '_xsrf=2|6be6c7ee|de091e966dba87e0b2c8637a63127715|1616479899',
'cookie': 'lotameid=undefined',
'cookie': 'started_since=1616480838',
'cookie': 'country=SG',
'cookie': 'nid=2de067d4-fed8-4c35-b7db-14f479997f8a',
'cookie': 'g_state={"i_p":1627551635034,"i_l":1}',
'upgrade-insecure-requests': '1',
'te': 'trailers',
'sec-gpc': '1'
}
response = requests.request("GET", url, headers=headers, data=payload)
print(response.text)

Python requests: POST form data, some fields updating but others not

I'm trying to post form data to a printers Web interface, i can post fields successfully but there's a certain type of form data which doesn't seem to work/update.
My Code:
import sys
import requests
IP='xxx.xxx.xxx.xxx'
PrinterSession = requests.session()
csrfdata = PrinterSession .get('http://xxx.xxx.xxx.xxx/properties/authentication/login.php')
csrftoken = str(csrfdata.content)[str(csrfdata.content).index('CSRFToken')+18:str(csrfdata.content).index('CSRFToken')+146]
login_data ={
'webUsername' : 'admin',
'webPassword' : 'xxxx',
'CSRFToken' : csrftoken,
'NextPage' : '/properties/authentication/luidLogin.php',
'_fun_function' : 'HTTP_Authenticate_fn',
'frmaltDomain' : 'default',
}
LoginPrinter = PrinterSession.post("http://xxx.xxx.xxx.xxx/userpost/printer.set", data=login_data )
headersSMTP = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,* /*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Content-Length': '328',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': 'PageToShow=; statusNumNodes=8; statusSelected=n1; frmCompany=; frmIFax=; frmFaxNumber=; frmProtocol=SMB; frmDocumentPath=; frmLoginName=Xerox; frmServerName=; frmNdsContext=; frmSmbShare=Scans; frmNdsTree=; frmIpv6_Host_1=%3A%3A; frmFirstName=David; frmLastName=Fester; frmFriendlyName=SMB%20Scan; frmEmail=dawie180#gmail.com; frmDisplayName=Fester%2C%20David; frmServerVolume=Scans; frmIpv4_1_1=12; frmIpv4_1_2=12; frmIpv4_1_3=12; frmIpv4_1_4=12; frmXrxAdd_1=Hn; frmHnAdd_1=asdasdsdasda; propNumNodes=117; PHPSESSID=457428a534bf077cc6bb0fff7ee80f7f; propSelected=n49; propHierarchy=001010000010000000000000000; WebTimerPopupID=15',
'Host': '10.241.24.28',
'Origin': 'https://10.241.24.28',
'Referer': 'https://10.241.24.28/protocols/smtp/required.php?from=email_req_smtp',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"',
'sec-ch-ua-mobile': '?0',
'Sec-Fetch-Dest': 'frame',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36',
}
cookietolist=headersSMTP['Cookie'].split(";")
cookietolist[27]=' PHPSESSID='+PrinterSession.cookies.get_dict()['PHPSESSID']
StringCookie = ";".join(cookietolist)
headersSMTP['Cookie']=StringCookie
postSMTPform = {
'_fun_function': 'HTTP_Set_Config_Attrib_fn',
'_fun_function': 'HTTP_CN_Set_fn',
'_fun_function': 'HTTP_SNMP_Set_SvcMon_fn',
'NextPage': '/properties/email/required.php',
'CSRFToken': csrftoken,
'POP3_MAILBOX_ADDRESS': 'smtp.test.com',
'connectivity.smtp.server': 'xxx.xxx.xxx.xxx:25',
}
PostSMTP = PrinterSession.post('http://'+IP+'/dummypost/printer.set', data=postSMTPform, headers=headersSMTP)
Post Request inspected via Chrome:
Request Headers:
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate, br
Accept-Language: en-US,en;q=0.9
Cache-Control: max-age=0
Connection: keep-alive
Content-Length: 364
Content-Type: application/x-www-form-urlencoded
Cookie: PHPSESSID=c960f2c4fb2f99c44333e601d3cc6a79; propSelected=n1; propNumNodes=117; WebTimerPopupID=2
Host: 10.241.24.28
Origin: https://10.241.24.28
Referer: https://10.241.24.28/protocols/smtp/required.php?from=email_cfg_over
sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"
sec-ch-ua-mobile: ?0
Sec-Fetch-Dest: frame
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: same-origin
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36
Form Data:
_fun_function: HTTP_Set_Config_Attrib_fn
_fun_function: HTTP_CN_Set_fn
_fun_function: HTTP_SNMP_Set_SvcMon_fn
NextPage: /config_overview/email.php
connectivity.smtp.server: 10.10.10.10:25
POP3_MAILBOX_ADDRESS: test#home.com
CSRFToken: eb590d22d4b0c75038816bd82a26e188f9a49941029c42765dd790cb8acd07b4966f5a910f493f2d839cb36d5d0a77a7632f614e06fcd67019c0f95f508d7da1
So "POP3_MAILBOX_ADDRESS: test#home.com" works fine, updates the SMTP From Address in the printer, all other Fields using the same Uppercase and underscore style key works fine, but the other field "connectivity.smtp.server: 10.10.10.10:25" does not work, it does not update the setting in the printer and it seems every field which uses this lower case seperated by periods style does not work.
The "POP3_MAILBOX_ADDRESS: test#home.com" works fine even without parsing the headers with the post request, i added the headers after and it still doesn't work. Don't know what im doing wrong.

how to make request in python instead of curl

Here is the curl command:
curl 'http://ecard.bupt.edu.cn/User/ConsumeInfo.aspx'
-H 'Cookie: ASP.NET_SessionId=4mzmi0whscqg4humcs5fx0cf; .NECEID=1; .NEWCAPEC1=$newcapec$:zh-CN_CAMPUS; .ASPXAUTSSM=A91571BB81F620690376AF422A09EEF8A7C4F6C09978F36851B8DEDFA56C19C96A67EBD8BBD29C385C410CBC63D38135EFAE61BF49916E0A6B9AB582C9CB2EEB72BD9DE20D64A51C09474E676B188D937B84E601A981C02F51AA9C85A08EABCC1D30D7B9299E02A45D427B08A44AEBB0'
-H 'Origin: http://ecard.bupt.edu.cn'
-H 'Accept-Encoding: gzip, deflate'
-H 'Accept-Language: zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4'
-H 'Upgrade-Insecure-Requests: 1'
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'
-H 'Content-Type: application/x-www-form-urlencoded'
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
-H 'Cache-Control: max-age=0' -H 'Referer: http://ecard.bupt.edu.cn/User/ConsumeInfo.aspx'
-H 'Connection: keep-alive'
--data '__EVENTTARGET=&__EVENTARGUMENT=&__VIEWSTATE=%2FwEPDwUINDEyMzA5NDkPFgIeCFNvcnRUeXBlBQNBU0MWAmYPZBYCAgMPZBYCAgMPZBYCAgQPPCsAEQMADxYEHgtfIURhdGFCb3VuZGceC18hSXRlbUNvdW50ZmQBEBYAFgAWAAwUKwAAFgJmD2QWAmYPZBYCAgEPZBYCAgEPDxYEHghDc3NDbGFzcwUKU29ydEJ0X0FzYx4EXyFTQgICZGQYAQUiY3RsMDAkQ29udGVudFBsYWNlSG9sZGVyMSRncmlkVmlldw88KwAMAQhmZBb11RlvPBlQn5fbuqe6uh8BRZJ2jUZ5U17xEnRM%2BnbF&__VIEWSTATEGENERATOR=D99777FC&__EVENTVALIDATION=%2FwEdAAgtbl4l0SdCQqYpyy3Ex39BEoHXgPeD%2Fa3eEKPr2bG0rY8WyuyQajjUbeopYM%2Fne68pyn2l0BPEWPPnyd6MfoDZVQGbFRIKjb%2FOTPkkCDIIaJ3X6W3VKO%2FV036Pdf6jf06OTFulzGhY80%2FZ1HbCJJ6LR5Lg6mLp72ibUCB3VipRJuK11qmW%2BSDe3IOvbK4oNUdV5%2FxmkXw25tTwfJ8P8OS2&ctl00%24ContentPlaceHolder1%24rbtnType=0&ctl00%24ContentPlaceHolder1%24txtStartDate=2016-09-17&ctl00%24ContentPlaceHolder1%24txtEndDate=2016-09-24&ctl00%24ContentPlaceHolder1%24btnSearch=%E6%9F%A5++%E8%AF%A2' --compressed
I want to use request to get the return message to analyse, but i'm confused about this.
Thanks!

You can use python requests module. The answer would be something like:
response = requests.get('http://ecard.bupt.edu.cn/User/ConsumeInfo.aspx',
headers={'Origin': 'http://ecard.bupt.edu.cn'},
data={'__EVENTTARGET': '', 'rbtnType':0},
cookies={'.NECEID': 1}
)
I didn't put all parameters there but I think this example is enough to figure out how to send exact the same request you want. The only thing I am not sure is --compressed flag - as I understand from docs it is something more specific to curl(correct me if I'm wrong)

Here is a useful website to convert the curl to requests:
curl to request
Curl Command:
curl 'http://js.t.sinajs.cn/open/api/js/api/bundle.js?version=20150130.02' -H 'Pragma: no-cache' -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36' -H 'Accept: */*' -H 'Referer: http://www.liaoxuefeng.com/wiki/001374738125095c955c1e6d8bb493182103fac9270762a000/001375502200090e998439175cc4268b0ea703b3b4ed55e000' -H 'Connection: keep-alive' -H 'Cache-Control: no-cache' --compressed
Python requests:
import requests
headers = {
'Pragma': 'no-cache',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36',
'Accept': '*/*',
'Referer': 'http://www.liaoxuefeng.com/wiki/001374738125095c955c1e6d8bb493182103fac9270762a000/001375502200090e998439175cc4268b0ea703b3b4ed55e000',
'Connection': 'keep-alive',
'Cache-Control': 'no-cache',
}
requests.get('http://js.t.sinajs.cn/open/api/js/api/bundle.js?version=20150130.02', headers=headers)

How could I query the result without selenium on Python or Ruby

I'm trying to track the tickets trend
currently, I'm using selenium to simulate submitting forms.
as you know, the selenium is slow and consume much more memory.
However, when you submit the form, it will redirect you to a new url http://makeabooking.flyscoot.com/Flight/Select
Therefore, I don't have the idea how could I do this without the selenium.
Because I couldn't change the form of query like this http://makeabooking.flyscoot.com/Flight/from={TPE}&to={NYK}&date={2015-10-12} to fetch the result.
Any idea to do this with Ruby or Python with SSL proxy and HTTP proxy support ?
sample website: http://www.flyscoot.com/index.php/en/

You can get the curl requests from chrome easily and use it by:
F12 > Network > request > Right Click > Copy As cURL
curl 'http://makeabooking.flyscoot.com/Flight/Select' -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8,tr;q=0.6' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,/;q=0.8' -H 'Referer: http://www.flyscoot.com/index.php/en/' -H 'Cookie: optimizelyEndUserId=oeu1444666692081r0.12463579000905156; __utmt=1; granify.lasts#1345=1444666699786; ASP.NET_SessionId=lql5yzv1l3yatkh1lcumg2e5; dotrez=1209262602.20480.0000; optimizelySegments=%7B%222335550040%22%3A%22gc%22%2C%222344180004%22%3A%22referral%22%2C%222354350067%22%3A%22false%22%2C%222355380121%22%3A%22none%22%7D; optimizelyBuckets=%7B%223025070068%22%3A%223020800213%22%7D; __utma=185425846.733949751.1444666694.1444666694.1444666694.1; __utmb=185425846.2.10.1444666694; __utmc=185425846; __utmz=185425846.1444666694.1.1.utmcsr=stackoverflow.com|utmccn=(referral)|utmcmd=referral|utmcct=/questions/33084039/how-could-i-query-the-result-without-selenium-on-python-or-ruby; granify.uuid=68b0d8e8-d068-40d8-9068-3098e870b858; granify.session#1345=1444666699786; granify.flags#1345=8; _gr_ep_sent=1; _gr_er_sent=1; granify.session_init#1345=2; optimizelyPendingLogEvents=%5B%5D' -H 'Connection: keep-alive' -H 'X-FirePHP-Version: 0.0.6' -H 'Cache-Control: max-age=0' --compressed
If you can set the headers and cookies info correctly you can use Python requests. If you want to convert it to the Python requests, you can use the this link. By this way you can simulate the browser. See the pyton requests:
cookies = {
'optimizelyEndUserId': 'oeu1444666692081r0.12463579000905156',
'__utmt': '1',
'granify.lasts#1345': '1444666699786',
'ASP.NET_SessionId': 'lql5yzv1l3yatkh1lcumg2e5',
'dotrez': '1209262602.20480.0000',
'optimizelySegments': '%7B%222335550040%22%3A%22gc%22%2C%222344180004%22%3A%22referral%22%2C%222354350067%22%3A%22false%22%2C%222355380121%22%3A%22none%22%7D',
'optimizelyBuckets': '%7B%223025070068%22%3A%223020800213%22%7D',
'__utma': '185425846.733949751.1444666694.1444666694.1444666694.1',
'__utmb': '185425846.2.10.1444666694',
'__utmc': '185425846',
'__utmz': '185425846.1444666694.1.1.utmcsr=stackoverflow.com|utmccn=(referral)|utmcmd=referral|utmcct=/questions/33084039/how-could-i-query-the-result-without-selenium-on-python-or-ruby',
'granify.uuid': '68b0d8e8-d068-40d8-9068-3098e870b858',
'granify.session#1345': '1444666699786',
'granify.flags#1345': '8',
'_gr_ep_sent': '1',
'_gr_er_sent': '1',
'granify.session_init#1345': '2',
'optimizelyPendingLogEvents': '%5B%5D',
}
headers = {
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'en-US,en;q=0.8,tr;q=0.6',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Referer': 'http://www.flyscoot.com/index.php/en/',
'Connection': 'keep-alive',
'X-FirePHP-Version': '0.0.6',
'Cache-Control': 'max-age=0',
}
requests.get('http://makeabooking.flyscoot.com/Flight/Select', headers=headers, cookies=cookies)
If you save the result, you can see that result is as done via browser (open stack.html):
r = requests.get('http://makeabooking.flyscoot.com/Flight/Select', headers=headers, cookies=cookies
f = open("stack1.html", "w")
f.write(r.content)

I think this answer https://stackoverflow.com/a/1196151/1033953 is what you're looking for.
You'll need to inspect the parameters on that form to make sure you're posting the right values, but then you just need to use the Ruby net/http to send the HTTP Post.
I'm sure Python has something similar. Or you could use curl to post as shows in this answer https://superuser.com/a/149335