When I print the response, everything seems to be correct, and the type is also correct.
Assertion: True
Response type: <class 'scrape_pb2.ScrapeResponse'>
But on postman I get "13 INTERNAL" With no additional information:
I can't figure out what the issue is, and I can't find out how to log or print the error from the server side.
Relevant proto parts:
syntax = "proto3";
service ScrapeService {
rpc ScrapeSearch(ScrapeRequest) returns (stream ScrapeResponse) {};
}
message ScrapeRequest {
string url = 1;
string keyword = 2;
}
message ScrapeResponse {
oneof result {
ScrapeSearchProgress search_progress = 1;
ScrapeProductsProgress products_progress = 2;
FoundProducts found_products = 3;
}
}
message ScrapeSearchProgress {
int32 page = 1;
int32 total_products = 2;
repeated string product_links = 3;
}
scraper.py
def get_all_search_products(search_url: str, class_keyword: str):
search_driver = webdriver.Firefox(options=options, service=service)
search_driver.maximize_window()
search_driver.get(search_url)
# scrape first page
product_links = scrape_search(driver=search_driver, class_keyword=class_keyword)
page = 1
search_progress = ScrapeSearchProgress(page=page, total_products=len(product_links), product_links=[])
search_progress.product_links[:] = product_links
# scrape next pages
while go_to_next_page(search_driver):
page += 1
print(f'Scraping page=>{page}')
product_links.extend(scrape_search(driver=search_driver, class_keyword=class_keyword))
print(f'Number of products scraped=>{len(product_links)}')
search_progress.product_links.extend(product_links)
# TODO: remove this line
if page == 6:
break
search_progress_response = ScrapeResponse(search_progress=search_progress)
yield search_progress_response
Server:
class ScrapeService(ScrapeService):
def ScrapeSearch(self, request, context):
print(f"Request received: {request}")
scrape_responses = get_all_search_products(search_url=request.url, class_keyword=request.keyword)
for response in scrape_responses:
print(f"Assertion: {response.HasField('search_progress')}")
print(f"Response type: {type(response)}")
yield response
Turns out it's just an issue with postman. I set up a python client and it worked.
Related
I need to send an API request to SMS API.
My Python code is working but my X++/C# code is not working.
I tried with Postman, it is working as well.
Here's my Python code:
import requests
headers = {"Accept":"application/json"}
data = {"AppSid":"#############YxLtXtaN###",
"SenderID":"####23423#####",
"Body":"This is a test message.",
"Recipient":"###45645######",
"responseType":"JSON",
"CorrelationID":"",
"baseEncode":"true",
"statusCallback":"sent",
"async":"false"}
r = requests.post('http://myapi/rest/SMS/messages', auth=('user#domain.com', 'password'), headers=headers,
data=data)
Here's my X++/C# code:
class Class1
{
public static void main(Args _args)
{
str destinationUrl = 'myapi', requestXml, responseXml;
System.Net.HttpWebRequest request;
System.Net.HttpWebResponse response;
CLRObject clrObj;
System.Byte[] bytes;
System.Text.Encoding utf8;
System.IO.Stream requestStream, responseStream;
System.IO.StreamReader streamReader;
System.Exception ex;
System.Net.WebHeaderCollection httpHeader;
str byteStr;
System.Byte[] byteArray;
System.IO.Stream stream;
System.IO.Stream dataStream;
byteStr = strfmt('%1:%2', "user#domain.com", "password");
requestXml = " {\"AppSid\":\"###########\", \"SenderID\":\"########-AD\", \"Body\":\"This is a test message from ## from X++ Coding Language..\", \"Recipient\":\"######\", \"responseType\":\"JSON\", \"CorrelationID\":\"\", \"baseEncode\":\"true\", \"statusCallback\":\"sent\", \"async\":\"false\"}";
try
{
new InteropPermission(InteropKind::ClrInterop).assert();
httpHeader = new System.Net.WebHeaderCollection();
clrObj = System.Net.WebRequest::Create(destinationUrl);
request = clrObj;
utf8 = System.Text.Encoding::get_UTF8();
bytes = utf8.GetBytes(requestXml);
request.set_KeepAlive(true);
request.set_ContentType("application/xml");
request.AllowAutoRedirect=true;
utf8 = System.Text.Encoding::get_UTF8();
byteArray = utf8.GetBytes(byteStr);
byteStr = System.Convert::ToBase64String(byteArray);
httpHeader.Add("Authorization", 'Basic ' + byteStr);
request.set_ContentType("text/xml; encoding='utf-8'");
request.set_ContentLength(bytes.get_Length());
request.set_Method("POST");
request.set_Headers(httpHeader);
requestStream = request.GetRequestStream();
requestStream.Write(bytes, 0, bytes.get_Length());
response = request.GetResponse();
responseStream = response.GetResponseStream();
streamReader = new System.IO.StreamReader(responseStream);
responseXml = streamReader.ReadToEnd();
info(responseXml);
}
catch (Exception::CLRError)
{
//bp deviation documented
ex = CLRInterop::getLastException().GetBaseException();
error(ex.get_Message());
}
requestStream.Close();
streamReader.Close();
responseStream.Close();
response.Close();
}
}
I'm getting this error:
error code : The remote server returned an error: (308) Permanent
Redirect.
I made two changes.
removed the Authorization by commenting it out
//httpHeader.Add("Authorization", 'Basic ' + byteStr);
Made it HTTPS instead of HTTP. I could see the Location: HTTPS in the exception Response object in Visual Studio.
These 2 things resolved the issue for me.
I don't know about x++, but you could try explicitly allowing redirect on your webRequest object.
webRequest = System.Net.WebRequest::Create('http://myapi/rest/SMS/messages') as System.Net.HttpWebRequest;
webRequest.AllowAutoRedirect = true;
webRequest.MaximumAutomaticRedirections = 1; //Set value according to your requirements
I'm invoking a python script from node js. The python script retrieves data from a REST API and stores it in a dataframe and then there's a search function based on user input. I'm confused as to what variable type does python send the data to node js in? I've tried to convert into a string but in node js it says it is an unresolved variable type. Here's the code:
r = requests.get(url)
data = r.json()
nested = json.loads(r.text)
nested_full = json_normalize(nested)
req_data= json_normalize(nested,record_path ='items')
search = req_data.get(["name", "id"," ])
#search.head(10)
filter = sys.argv[1:]
print(filter)
input = filter[0]
print(input)
result = search[search["requestor_name"].str.contains(input)]
result = result.to_String(index=false)
response = '```' + str(result) + '```'
print(response)
sys.stdout.flush()
Here's the node js program that invokes the above python script. How do i store the output in a format which i can pass to another function in node?
var input = 'robert';
var childProcess = require("child_process").spawn('python', ['./search.py', input], {stdio: 'inherit'})
const stream = require('stream');
const format = require('string-format')
childProcess.on('data', function(data){
process.stdout.write("python script output",data)
result += String(data);
console.log("Here it is", data);
});
childProcess.on('close', function(code) {
if ( code === 1 ){
process.stderr.write("error occured",code);
process.exit(1);
}
else{
process.stdout.write('done');
}
});
According to the docs:
childProcess.stdout.on('data', (data) => {
console.log(`stdout: ${data}`);
});
I have followed the usage as per link below:
https://www.npmjs.com/package/react-native-paytm
This is my checksum generation code:
import Checksum
# initialize a dictionary
paytmParams = dict()
# put checksum parameters in Dictionary
paytmParams["MID"] = "*****************"
paytmParams["ORDER_ID"] = 'ORD001'
paytmParams["CUST_ID"] = 'CUST001'
paytmParams["INDUSTRY_TYPE_ID"] = 'Retail'
paytmParams["CHANNEL_ID"] = 'WAP'
paytmParams["TXN_AMOUNT"] = '1.00'
paytmParams["WEBSITE"] = 'WEBSTAGING'
paytmParams["EMAIL"] = '**************'
paytmParams["MOBILE_NO"] = '****************'
paytmParams["CALLBACK_URL"] = 'https://securegw-
stage.paytm.in/theia/paytmCallback?ORDER_ID=ORD001'
# Find your Merchant Key in your Paytm Dashboard at
https://dashboard.paytm.com/next/apikeys
checksum = Checksum.generate_checksum(paytmParams, "*************")
Code for react native:
import paytm from 'react-native-paytm';
import { Platform, DeviceEventEmitter, NativeModules,NativeEventEmitter} from 'react-native';
const paytmConfig = {
MID: '************',
WEBSITE: 'WEBSTAGING',
CHANNEL_ID: 'WAP',
INDUSTRY_TYPE_ID: 'Retail',
CALLBACK_URL: 'https://securegw-
stage.paytm.in/theia/paytmCallback?ORDER_ID=ORD001'
}
onPayTmResponse(response) {
// Process Response
// response.response in case of iOS
// reponse in case of Android
console.log(response);
}
runTransaction() {
const callbackUrl = 'https://securegw-
stage.paytm.in/theia/paytmCallback?ORDER_ID=ORD001'
const details = {
mode: 'Staging', // 'Staging' or 'Production'
mid: paytmConfig.MID,
industryType: paytmConfig.INDUSTRY_TYPE_ID,
website: paytmConfig.WEBSITE,
channel: paytmConfig.CHANNEL_ID,
amount: '1.00', // String
orderId: 'ORD001', // String
custId: 'CUST001', // String
email: '*****************', // String
phone: '***********', // S
checksumhash: '***********************************************', //From your server using PayTM Checksum Utility
callback: callbackUrl
};
paytm.startPayment(details);
}
The issue is that i cant even spot the error point here as there is no console. I could use some direction here. Need this for staging right now
in my cash use this npm #philly25/react-native-paytm and solved this error
are you sure onPaytmResponse is a paytm listener
like this:
componentWillMount() {
Paytm.addListener(Paytm.Events.PAYTM_RESPONSE, this.onPayTmResponse)
};
componentWillUnmount() {
Paytm.removeListener(Paytm.Events.PAYTM_RESPONSE, this.onPayTmResponse);
};
I am building a file storage client/server using python for the client, go for the server, along with gRPC. I have already successfully built the client in go and it works! I am trying to do the same in python now. Today, I have been working on it all day, yet I have made 0 progress -_-. The request I am sending to my server is probably malformed in some way judging by the error message being spit out of the gRPC library:
File "/Users/xxxxx/Desktop/clients/uploadClient.py", line 60, in upload_file
for res in stream:
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 367, in __next__
return self._next()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/grpc/_channel.py", line 361, in _next
raise self
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with:
status = StatusCode.INTERNAL
details = "Exception serializing request!"
debug_error_string = "None"
>
Not too helpful. I have not found any helpful documentation and the hours of reading and searching have not payed off yet. The only thing I can rule out so far is that the problem is server related since the go client has been working great.
Here is my proto (note: I did distill this down a bit and renamed some things):
syntax = "proto3";
import "google/protobuf/timestamp.proto";
package upload;
message Chunk {
message Index {
uint64 as_uint64 = 1;
}
Index index = 1;
bytes sha512 = 2;
bytes data = 3;
}
message Descriptor {
string author = 1; // author
string label = 2; // label
Format format = 3; //format
}
enum Format {
FORMAT_UNKNOWN = 0;
FORMAT_CSV = 1;
FORMAT_XML = 2;
FORMAT_JSON = 3;
FORMAT_PDF = 4;
}
message UploadFile {
message ToClient {
oneof details {
Finished finished = 1;
}
}
message ToService {
oneof details {
Descriptor descriptor = 1;
Chunk chunk = 2;
}
}
}
.
service FileService {
rpc Upload(stream UploadFile.ToService) returns (stream UploadFile.ToClient);
}
Here is the code (note: I did distill this down a bit and renamed some things):
import s_pb2 as s
import s_pb2_grpc as s_grpc
token = 'xxxx'
url = 'xxxx:433'
requestMetadata = [('authorization', 'Bearer ' + token)]
def create_stub():
creds = grpc.ssl_channel_credentials()
channel = grpc.secure_channel(url, creds)
return s_grpc.UploadManagerStub(channel)
def upload_file(src, label, fileFormat):
stub = create_stub()
stream = stub.Upload(
request_iterator=__upload_file_iterator(src, label, fileFormat),
metadata=requestMetadata
)
for res in stream:
print(res)
return stream
def __upload_file_iterator(src, name, fileFormat, chunk_size = 1024):
def descriptor():
to_service = s.UploadFile.ToService
to_service.descriptor = s.Descriptor(
label=label,
format=fileFormat
)
return to_service
yield descriptor()
Yes, I know my iterator is only returning 1 thing, I removed some code to try to isolate the problem.
gRPC is not my strongest skills and I would like to believe that my pea brain is just missing something obvious.
All help is appreciated!
Wrapping the "oneof" details in the constructor fixed the problem:
def chunk(data, index):
return s.UploadFile.ToService(
chunk=s.Chunk(
index=s.Chunk.Index(as_uint64=index),
data=data,
sha512=hashlib.sha512(data).digest()
)
)
def descriptor(name, fileFormat):
return s.UploadFile.ToService(
descriptor=s.Descriptor(
name=name,
format=fileFormat
)
)
I'm trying to retrieve data programmatically through websockets and am failing due to my limited knowledge around this. On visiting the site at https://www.tradingview.com/chart/?symbol=ASX:RIO I notice one of the websocket messages being sent out is ~m~60~m~{"m":"quote_fast_symbols","p":["qs_p089dyse9tcu","ASX:RIO"]}
My code is as follows:
from websocket import create_connection
import json
ws = create_connection("wss://data.tradingview.com/socket.io/websocket?from=chart%2Fg0l68xay%2F&date=2019_05_27-12_19")
ws.send(json.dumps({"m":"quote_fast_symbols","p"["qs_p089dyse9tcu","ASX:RIO"]}))
result = ws.recv()
print(result)
ws.close()
Result of the print:
~m~302~m~{"session_id":"<0.25981.2547>_nyc2-charts-3-webchart-5#nyc2-compute-3_x","timestamp":1558976872,"release":"registry:5000/tvbs_release/webchart:release_201-106","studies_metadata_hash":"888cd442d24cef23a176f3b4584ebf48285fc1cd","protocol":"json","javastudies":"javastudies-3.44_955","auth_scheme_vsn":2}
I get this result no matter what message I send out, out of the almost multitude of messages that seem to be sent out. I was hoping one of the messages sent back will be the prices info for the low and highs for RIO. Is there other steps I should include to get this data? I understand there might be some form of authorisation needed but I dont know the workflow.
Yes, there is much more to setup and it needs to be done in order. The following example written in Node.js will subscribe to the BINANCE:BTCUSDT real time data and fetch historical 5000 bars on the daily chart.
Ensure you have proper value of the origin field set in header section before connecting. Otherwise your connection request will be rejected by the proxy. I most common ws there is no way to do this. Use faye-websocket instead
const WebSocket = require('faye-websocket')
const ws = new WebSocket.Client('wss://data.tradingview.com/socket.io/websocket', [], {
headers: { 'Origin': 'https://data.tradingview.com' }
});
After connecting you need to setup your data stream. I don't know if all of this commands needs to be performed. This probably can be shrink even more but it works. Basically what you need to do is to create new quote and chart sessions and within these sessions request stream of the data of the prior resolved symbol.
ws.on('open', () => {
const quote_session = 'qs_' + getRandomToken()
const chart_session = 'cs_' + getRandomToken()
const symbol = 'BINANCE:BTCUSDT'
const timeframe = '1D'
const bars = 5000
sendMsg(ws, "set_auth_token", ["unauthorized_user_token"])
sendMsg(ws, "chart_create_session", [chart_session, ""])
sendMsg(ws, "quote_create_session", [quote_session])
sendMsg(ws, "quote_set_fields", [quote_session,"ch","chp","current_session","description","local_description","language","exchange","fractional","is_tradable","lp","lp_time","minmov","minmove2","original_name","pricescale","pro_name","short_name","type","update_mode","volume","currency_code","rchp","rtc"])
sendMsg(ws, "quote_add_symbols",[quote_session, symbol, {"flags":['force_permission']}])
sendMsg(ws, "quote_fast_symbols", [quote_session, symbol])
sendMsg(ws, "resolve_symbol", [chart_session,"symbol_1","={\"symbol\":\""+symbol+"\",\"adjustment\":\"splits\",\"session\":\"extended\"}"])
sendMsg(ws, "create_series", [chart_session, "s1", "s1", "symbol_1", timeframe, bars])
});
ws.on('message', (msg) => { console.log(`RX: ${msg.data}`) })
And finally implementation of the helper methods
const getRandomToken = (stringLength=12) => {
characters = 'abcdefghijklmnopqrstuvwxyz0123456789'
const charactersLength = characters.length;
let result = ''
for ( var i = 0; i < stringLength; i++ ) {
result += characters.charAt(Math.floor(Math.random() * charactersLength))
}
return result
}
const createMsg = (msg_name, paramsList) => {
const msg_str = JSON.stringify({ m: msg_name, p: paramsList })
return `~m~${msg_str.length}~m~${msg_str}`
}
const sendMsg = (ws, msg_name, paramsList) => {
const msg = createMsg(msg_name, paramsList)
console.log(`TX: ${msg}`)
ws.send(createMsg(msg_name, paramsList))
}