I wanna convert CSV to JSON correct data type
csv file 2nd row is data type.
data has over 300 properties
example data:
Name
DMG
HP
Human
string
number
number
boolean
knight
100
500
true
archer
50
200
true
dog
-
-
-
if string empty return null
if number empty return 0
if boolean empty return false
my node.js code:
const fs = require('fs')
const papa = require("papaparse")
const results = [];
const options = { header: true, dynamicTyping: true };
fs.createReadStream("characters.csv")
.pipe(papa.parse(papa.NODE_STREAM_INPUT, options))
.on("data", (data) => {
results.push(data);
}).on("end", () => {
console.log(results)
})
output I expecting:
[
{
"Name": "knight",
"DMG": 100,
"HP": 500,
"Human": true,
},
{
"Name": "archer",
"DMG": 50,
"HP": 200,
"Human": true,
},
{
"Name": "dog",
"DMG": 0,
"HP": 0,
"Human": false,
},
]
Options options..
in this approach I cached the headerTypes and made a small helper function to return the intended type
define the vars let i = 0, headerTypes = {};
replace your on.data code with this
.on("data", (data) => {
if(i > 0){
for(let prop in data){
if(data.hasOwnProperty(prop)){
const value = data[prop];
data[prop] = typecast(headerTypes[prop], value);
}
}
}else{
//save types from row 0
headerTypes = data;
}
i++;
})
add this helper function
function typecast(type = '', value){
switch(type){
case "number":
return +value || 0; //unary for int or float
case "boolean":
return value === !0; //typecast to Boolean will let any filled string like '-' be true, do this instead.
case "string":
default:
return String(value);
}
}
Try overriding the properties in the on.data event like so
let i = 0; //define this outside, its only to skip that first row with headers.
if(i > 0){ //skipping first row
data.DMG = parseInt(data.DMG) || 0; //an int or 0
data.HP = parseInt(data.HP) || 0; //an int or 0
data.Human = data.Human === !0; //!0 = not false, will match true string or bool
results.push(data);
}
i++;//iterate
const fs = require('fs')
const papa = require("papaparse")
const results = [];
const options = { header: true, dynamicTyping: true };
let i = 0;
fs.createReadStream("characters.csv")
.pipe(papa.parse(papa.NODE_STREAM_INPUT, options))
.on("data", (data) => {
//add this here
if(i > 0){
data.DMG = parseInt(data.DMG) || 0; //an int or 0
data.HP = parseInt(data.HP) || 0; //an int or 0
data.Human = data.Human === !0; //!0 = not false, will match true string or bool
results.push(data);
}
i++;
//end add this here
}).on("end", () => {
console.log(results)
})
I want to search for keywords from the specific field and return the documents. On top of these documents, I want to iterate through each nested object and search for keywords again from the same specific field on selected documents.
If a keyword exists, then check:
if boolean isCurrent = True, set isCurrent=0 and append this value to list
else if isCurrent = False, take the difference of current datetime, end datetime and get the value in terms of months and append this value to the list.
Finally, get the minimum value from that list of each document and sort the documents based on the minimum value.
I am able to do this custom login through script_fields and sort the documents based on the minimum value. When I use this same login in script_score it does not work. When I debug I see a problem accessing the nested field using params._source.
Any help will be much appreciated.
Please find the below elastic search query using script_fields. Here I am passing the current_milliseconds value from a python script.
query = {
'query': {
"nested": {
"path": "person.experiences",
"query": {
"query_string": {
"query": keywords,
"fields": ["person.experiences.description"],
"default_operator": "AND"
}
}
}
},
"script_fields": {
"recency": {
"script": {
"lang": "painless",
"inline": """
def myString = "";
def isCurrent = 0;
def isFound = false;
def index_position = 0;
def recency_num = 0;
def result = 0;
def list = new ArrayList();
// for loop starts here
for(int i=0; i<params._source.person.experiences.size(); i++){
myString = params._source.person.experiences[i].description;
// string match starts here
if(myString != null && myString != ''){
def matcher1 = /electric/.matcher(myString.toLowerCase());
def matcher2 = /vehicle/.matcher(myString.toLowerCase());
//if(wordMatcher.find()){
if (matcher1.find() || matcher2.find()){
isFound = true;
}
if (isFound == true){
// recency check starts here
isCurrent = params._source.person.experiences[i].isCurrent;
if(isCurrent == true){
isCurrent=0;
result+=isCurrent;
list.add(isCurrent);
} else{
ZonedDateTime now = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params['current_datetime']), ZoneId.of('Z'));
ZonedDateTime end_date = ZonedDateTime.parse(params._source.person.experiences[i].end);
isCurrent = end_date.until(now, ChronoUnit.MONTHS);
list.add(isCurrent);
result+=isCurrent;
recency_num = isCurrent;
}
}
}
}
def min = list.get(0);
for (int i : list){
min = min < i ? min : i;
}
return min;
""",
"params": {
"search_keywords": "Machine Learning",
"current_datetime": current_milliseconds
}
}
}
}
}
Thanks in advance.
A valid script_score query in your case would look something like this:
GET my-index/_search
{
"query": {
"bool": {
"must": [
{
"function_score": {
"functions": [
{
"script_score": {
"script": {
"source": """
def myString = "";
def isCurrent = 0;
def isFound = false;
def index_position = 0;
def recency_num = 0;
def result = 0;
def list = new ArrayList();
def experiences = params._source.person.experiences;
// for loop starts here
for (int i=0; i<experiences.length; i++){
def experience = experiences[i];
myString = experience.description;
// string match starts here
if(myString != null && myString != '') {
def matcher1 = /electric/.matcher(myString.toLowerCase());
def matcher2 = /vehicle/.matcher(myString.toLowerCase());
if (matcher1.find() || matcher2.find()){
isFound = true;
}
if (isFound == true){
// recency check starts here
isCurrent = experience.isCurrent;
if (isCurrent == true){
isCurrent=0;
result += isCurrent;
list.add(isCurrent);
} else {
def now = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params['current_datetime']), ZoneId.of('Z'));
def end_date = ZonedDateTime.parse(experience.end);
isCurrent = end_date.until(now, ChronoUnit.MONTHS);
list.add(isCurrent);
result += isCurrent;
recency_num = isCurrent;
}
}
}
}
if (list.length === 0) {
return 0;
}
def min = list.get(0);
for (int i : list){
min = min < i ? min : i;
}
return min;
""",
"params": {
"search_keywords": "Machine Learning",
"current_datetime": 1643036066000
}
}
}
}
]
}
}
]
}
}
}
Note that loads of regexes, iteration, and (date) parsing can drastically increase the query resolution.
I am trying to convert this Python code to C# (.NET Core ideally).
Source
My goal is to convert the QR input string to another string containing the data in json. See the provided links.
#! /usr/bin/env python3
import json
import sys
import zlib
import base45
import cbor2
from cose.messages import CoseMessage
payload = sys.argv[1][4:]
print("decoding payload: "+ payload)
# decode Base45 (remove HC1: prefix)
decoded = base45.b45decode(payload)
# decompress using zlib
decompressed = zlib.decompress(decoded)
# decode COSE message (no signature verification done)
cose = CoseMessage.decode(decompressed)
# decode the CBOR encoded payload and print as json
print(json.dumps(cbor2.loads(cose.payload), indent=2))
I couldn't find any NuGet package for Zlib, that would work correctly. So I am stuck straight after base45 decoding. Thanks for any tips.
using System.Text; //Rystem.Text.Base45 NuGet
var removedHeader = testQrData.Substring(4);
var decoded = removedHeader.FromBase45();
byte[] rawBytes = Encoding.ASCII.GetBytes(decoded);
This link might be helpful for further investigation.
Decoding scheme
IBarcodeReader reader = new BarcodeReader();//using Zxing
var barcodeBitmap = (Bitmap)Bitmap.FromFile("qrcode.png");
var barcodeReader = new BarcodeReader();
var qrcontent = barcodeReader.Decode(barcodeBitmap).Text;
var qrmessage = qrcontent.Substring(4);//remove first 4 chars
byte[] decodedBase45 = Base45Encoding.Decode(qrmessage);//using base45 lib
var cose = ZlibStream.UncompressBuffer(decodedBase45);//using zlib or similar
var decrypted = Message.DecodeFromBytes(cose).GetContent(); //using COSE
CBORObject cbor = CBORObject.DecodeFromBytes(decrypted); //using Peter.O.. CBOR
var jsonDecoded = cbor.ToJSONString(); //or deserialize it to custom class
I added "Zlib.Portable" from NuGet and used "ZlibStream.UncompressString"
byte[] decoded = Base45Encoding.Decode(input);
var stringResult = ZlibStream.UncompressString(decoded);`
I'm stuck in the next step "CoseMessage.decode" :/
I have used the ZXing library in the past to decode and create QR Codes:
https://github.com/micjahn/ZXing.Net
You could also try: https://github.com/codebude/QRCoder
A quick ZXing example from their github page:
// create a barcode reader instance
IBarcodeReader reader = new BarcodeReader();
// load a bitmap
var barcodeBitmap = (Bitmap)Image.LoadFrom("C:\\sample-barcode-image.png");
// detect and decode the barcode inside the bitmap
var result = reader.Decode(barcodeBitmap);
// do something with the result
if (result != null)
{
txtDecoderType.Text = result.BarcodeFormat.ToString();
txtDecoderContent.Text = result.Text;
}
This example reads a QR Code image.
I'm not sure what your input is but i would assume it's also an image in binary format so you might have to play around to get it working.
This solution works for me.
using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
using System;
using System.IO;
using Com.AugustCellars.COSE;
using PeterO.Cbor;
using Newtonsoft.Json;
using Microsoft.Extensions.Logging;
using System.Linq;
using System.Collections.Generic;
namespace DGCVerification.Services {
public class DGCVerificationService: IDGCVerificationService {
private readonly string _certificatePrefix = "HC1:";
private readonly ILogger < DGCVerificationService > _logger;
public DGCVerificationService(ILogger < DGCVerificationService > logger) {
_logger = logger;
}
private bool TryTrimPrefix(string certificateString, out string result) {
if (!certificateString.StartsWith(_certificatePrefix)) {
result = null;
return false;
}
result = certificateString.Substring(_certificatePrefix.Length);
return true;
}
private byte[] DecompressFromZlib(byte[] input) {
using(var memoryStream = new MemoryStream(input)) {
using(var decompressionStream = new InflaterInputStream(memoryStream)) {
using(var resultStream = new MemoryStream()) {
decompressionStream.CopyTo(resultStream);
var result = resultStream.ToArray();
return result;
}
}
}
}
private bool VerifyCoseSignature(Sign1Message coseMessage, string signature) {
var signatureAsBytes = Convert.FromBase64String(signature);
var publicKey = OneKey.FromX509(signatureAsBytes);
var result = coseMessage.Validate(publicKey);
_logger.LogInformation($"cose message signature {signature} verified as {result}");
return result;
}
private DGCPayload DecodeAndDeserialize(string certificateString, out Sign1Message coseMessage) {
coseMessage = null;
if (!TryTrimPrefix(certificateString, out
var trimmedPrefixString)) {
_logger.LogInformation($"certificate {certificateString} didn't have proper prefix {_certificatePrefix}");
return null;
}
var bytesBase256 = Base45Encoding.Decode(trimmedPrefixString);
_logger.LogInformation($"certificate {certificateString} base45 decoded to {Convert.ToBase64String(bytesBase256)}");
var decompressedFromZlib = DecompressFromZlib(bytesBase256);
_logger.LogDebug($"certificate {certificateString} zlib decompressed to {Convert.ToBase64String(decompressedFromZlib)}");
coseMessage = Message.DecodeFromBytes(decompressedFromZlib) as Sign1Message;
var coseMessagePayload = coseMessage.GetContent();
var cborResult = CBORObject.DecodeFromBytes(coseMessagePayload);
var jsonResult = cborResult.ToJSONString();
var result = JsonConvert.DeserializeObject < DGCPayloadCzechVersionRoot > (jsonResult);
return result.DGCPayloadWrap.DGCPayload;
}
private bool IsNotExpiredDGC(DGCPayload dGCPayload, UzisData uzisData) {
var vaccine = dGCPayload.Vaccination?.FirstOrDefault();
if (vaccine != null) {
var vaccinationValidityRules = uzisData.DGCValidityCzechRules.Rules
.FirstOrDefault()
.PlatnostiVakcinace;
var rule = vaccinationValidityRules.FirstOrDefault(x => x.VaccineMedicinalProduct == vaccine.Mp) ??
vaccinationValidityRules.FirstOrDefault(x => x.VaccineMedicinalProduct == null);
if (!DateTime.TryParse(vaccine.Dt, out
var vaccinatedOnDate)) {
_logger.LogError($"couldn't parse date of vaccination for DGC : {JsonConvert.SerializeObject(dGCPayload)}");
return false;
}
var result = DateTime.Now.Date <= vaccinatedOnDate.AddMonths(rule.OdolnostMesicDo);
return result;
}
var test = dGCPayload.Test?.FirstOrDefault();
if (test != null) {
var testValidityRule = uzisData.DGCValidityCzechRules.Rules
.FirstOrDefault()
.PlatnostiTestu
.FirstOrDefault(x => x.TypeOfTest == test.Tt);
if (!DateTime.TryParse(test.Tr, out
var testedOnDate)) {
_logger.LogError($"couldn't parse date of test for DGC : {JsonConvert.SerializeObject(dGCPayload)}");
return false;
}
var result = DateTime.Now.Date <= testedOnDate.AddHours(testValidityRule.PlatnostHod);
return result;
}
var recovery = dGCPayload.Recovery?.FirstOrDefault();
if (recovery != null) {
if (!DateTime.TryParse(recovery.Du, out
var recoveryValidUntil)) {
_logger.LogError($"couldn't parse recovert valid until for DGC : {JsonConvert.SerializeObject(dGCPayload)}");
return false;
}
var result = DateTime.Now.Date <= recoveryValidUntil;
return result;
}
return false;
}
private string GetCountryFromDGC(DGCPayload dGCPayload) {
var result = dGCPayload.Vaccination?.FirstOrDefault()?.Co ??
dGCPayload.Test?.FirstOrDefault()?.Co ??
dGCPayload.Recovery?.FirstOrDefault()?.Co;
if (result == null) {
throw new ArgumentException($"couldn't retrieve country from DGC. dgc : {JsonConvert.SerializeObject(dGCPayload)}");
}
return result;
}
private List < SignatureCertificate > GetFittingSignatures(DGCPayload dGCPayload, UzisData uzisData) {
try {
var country = GetCountryFromDGC(dGCPayload);
var result = uzisData.NationalCertificateSignatures.SignatureCertificate
.Where(x => x.Active)
.Where(x => x.Country == country)
.Where(x => x.CertificateType == "DSC")
.ToList();
return result;
} catch (Exception e) {
_logger.LogError(e, $"Filtering signatures from UZIS failed with exception");
return null;
}
}
private bool IsProperlySignedDGC(string certificateString, DGCPayload dGCPayload, Sign1Message coseMessage, UzisData uzisData) {
var fittingSignatures = GetFittingSignatures(dGCPayload, uzisData);
var result = false;
foreach(var signature in fittingSignatures) {
try {
var signatureVerificationResult = VerifyCoseSignature(coseMessage, signature.RawData);
_logger.LogInformation($"certificate {certificateString} signature validation against signature {signature} resulted in {signatureVerificationResult}");
result |= signatureVerificationResult;
} catch (Exception e) {
_logger.LogError(e, $"certificate {certificateString} signature validation against signature {signature} failed with exception");
}
}
return result;
}
public bool IsValidDgc(string certificateString, UzisData uzisData, out DGCPayload decodedDGCPayload, out VerificationResult verificationResult) {
decodedDGCPayload = null;
Sign1Message coseMessage = null;
try {
decodedDGCPayload = DecodeAndDeserialize(certificateString, out coseMessage);
if (coseMessage == null) {
_logger.LogInformation($"certificate {certificateString} decoded to null COSE");
verificationResult = VerificationResult.UnableToDecode;
return false;
}
} catch (Exception e) {
_logger.LogError(e, $"certificate {certificateString} decoding failed with exception");
verificationResult = VerificationResult.UnableToDecode;
return false;
}
var isProperlySigned = IsProperlySignedDGC(certificateString, decodedDGCPayload, coseMessage, uzisData);
if (!isProperlySigned) {
verificationResult = VerificationResult.InvalidSignature;
return false;
}
var isNotExpired = IsNotExpiredDGC(decodedDGCPayload, uzisData);
if (!isNotExpired) {
verificationResult = VerificationResult.Expired;
return false;
}
verificationResult = VerificationResult.Valid;
return true;
}
}
}
And the tricky base45
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace DGCVerification
{
public class Base45Encoding
{
private static readonly Dictionary<byte, char> _encodingTable = new Dictionary<byte, char>
{
{0x0,'0'},
{0x1,'1'},
{0x2,'2'},
{0x3,'3'},
{0x4,'4'},
{0x5,'5'},
{0x6,'6'},
{0x7,'7'},
{0x8,'8'},
{0x9,'9'},
{0xA,'A'},
{0xB,'B'},
{0xC,'C'},
{0xD,'D'},
{0xE,'E'},
{0xF,'F'},
{0x10,'G'},
{0x11,'H'},
{0x12,'I'},
{0x13,'J'},
{0x14,'K'},
{0x15,'L'},
{0x16,'M'},
{0x17,'N'},
{0x18,'O'},
{0x19,'P'},
{0x1A,'Q'},
{0x1B,'R'},
{0x1C,'S'},
{0x1D,'T'},
{0x1E,'U'},
{0x1F,'V'},
{0x20,'W'},
{0x21,'X'},
{0x22,'Y'},
{0x23,'Z'},
{0x24,' '},
{0x25,'$'},
{0x26,'%'},
{0x27,'*'},
{0x28,'+'},
{0x29,'-'},
{0x2A,'.'},
{0x2B,'/'},
{0x2C,':'},
};
private static readonly Dictionary<char, byte> _decodingTable;
static Base45Encoding()
{
_decodingTable = _encodingTable.ToDictionary(x => x.Value, x => x.Key);
}
private static List<byte> ToBytesBase45(string charsBase45)
{
var result = new List<byte>(charsBase45.Length);
foreach (var character in charsBase45)
{
if (!_decodingTable.TryGetValue(character, out byte asByte))
{
throw new FormatException($"input string contains {character} with numeric value {char.GetNumericValue(character)} on index {charsBase45.IndexOf(character)} which is not valid base45 character");
}
result.Add(asByte);
}
return result;
}
private static List<ushort> ToShortsBase10(List<byte> bytesBase45)
{
var result = new List<ushort>(bytesBase45.Count);
ushort num = 0;
double pow = 0;
for (int i = 0; i != bytesBase45.Count; ++i, ++pow)
{
num += (ushort)(bytesBase45[i] * Math.Pow(45, pow));
if (pow == 2 || i == (bytesBase45.Count -1))
{
result.Add(num);
num = 0;
pow = -1;
}
}
return result;
}
private static List<byte> ToBytesBase256(List<ushort> shortsBase10, int charBase45Length)
{
var result = new List<byte>(shortsBase10.Count);
for (int i = 0; i != shortsBase10.Count; ++i)
{
var num = (byte)(shortsBase10[i] / 256);
if(!(i == (shortsBase10.Count - 1)
&& charBase45Length % 3 != 0
&& num == 0))
{
result.Add(num);
}
result.Add((byte)(shortsBase10[i] % 256));
}
return result;
}
public static byte[] Decode(string charsBase45)
{
if (charsBase45.Length % 3 == 1)
{
throw new FormatException("input string does not have correct length. mod 3 == 1. it isnt base45");
}
var bytesBase45 = ToBytesBase45(charsBase45);
var shortsBase10 = ToShortsBase10(bytesBase45);
var bytesBase256 = ToBytesBase256(shortsBase10, charsBase45.Length);
return bytesBase256.ToArray();
}
}
}
Decode/encode base45:
/// <summary>
/// https://tools.ietf.org/html/draft-faltstrom-baseBaseSize-01
/// TL/DR:
/// This encoding takes a byte array, splits it into 2 byte chunks and encodes each chunk as 3 characters.
/// Any remaining byte is encoded as 2 characters, padded with a '0' when the remaining byte has value < 45.
/// </summary>
public static class Base45Encoding
{
private const int BaseSize = 45;
private const int BaseSizeSquared = 2025;
private const int ChunkSize = 2;
private const int EncodedChunkSize = 3;
private const int SmallEncodedChunkSize = 2;
private const int ByteSize = 256;
private static readonly char[] _Encoding = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
'U', 'V', 'W', 'X', 'Y', 'Z', ' ', '$', '%', '*',
'+', '-', '.', '/', ':' };
private static readonly Dictionary<char, byte> _Decoding = new(BaseSize);
static Base45Encoding()
{
for(byte i = 0; i < _Encoding.Length; ++i)
_Decoding.Add(_Encoding[i], i);
}
public static string Encode(byte[] buffer)
{
if (buffer == null)
throw new ArgumentNullException(nameof(buffer));
var wholeChunkCount = buffer.Length / ChunkSize;
var result = new char[wholeChunkCount * EncodedChunkSize + (buffer.Length % ChunkSize == 1 ? SmallEncodedChunkSize : 0)];
if (result.Length == 0)
return string.Empty;
var resultIndex = 0;
var wholeChunkLength = wholeChunkCount * ChunkSize;
for (var i = 0; i < wholeChunkLength;)
{
var value = buffer[i++] * ByteSize + buffer[i++];
result[resultIndex++] = _Encoding[value % BaseSize];
result[resultIndex++] = _Encoding[value / BaseSize % BaseSize];
result[resultIndex++] = _Encoding[value / BaseSizeSquared % BaseSize];
}
if (buffer.Length % ChunkSize == 0)
return new string(result);
result[^2] = _Encoding[buffer[^1] % BaseSize];
result[^1] = buffer[^1] < BaseSize ? _Encoding[0] : _Encoding[buffer[^1] / BaseSize % BaseSize];
return new string(result);
}
public static byte[] Decode(string value)
{
if (value == null)
throw new ArgumentNullException(nameof(value));
if (value.Length == 0)
return Array.Empty<byte>();
var remainderSize = value.Length % EncodedChunkSize;
if (remainderSize == 1)
throw new FormatException("Incorrect length.");
var buffer = new byte[value.Length];
for (var i = 0; i < value.Length; ++i)
{
if (_Decoding.TryGetValue(value[i], out var decoded))
{
buffer[i] = decoded;
continue; //Earliest return on expected path.
}
throw new FormatException($"Invalid character at position {i}.");
}
var wholeChunkCount = buffer.Length / EncodedChunkSize;
var result = new byte[wholeChunkCount * ChunkSize + (remainderSize == ChunkSize ? 1 : 0)];
var resultIndex = 0;
var wholeChunkLength = wholeChunkCount * EncodedChunkSize;
for (var i = 0; i < wholeChunkLength; )
{
var val = buffer[i++] + BaseSize * buffer[i++] + BaseSizeSquared * buffer[i++];
result[resultIndex++] = (byte)(val / ByteSize); //result is always in the range 0-255 - % ByteSize omitted.
result[resultIndex++] = (byte)(val % ByteSize);
}
if (remainderSize == 0)
return result;
result[^1] = (byte)(buffer[^2] + BaseSize * buffer[^1]); //result is always in the range 0-255 - % ByteSize omitted.
return result;
}
}
Hi I have a question concerend about protocol buffers for list of lists in json:
Example .json (testjson)
"outcome": {
"col": [
"datetime",
"return"
],
"val": [[1199232000000, -0.0066], [1199318400000, -0.0033]]
}
My .proto File (cum_ret.proto)
message CumReturn {
message period_value {
oneof types{
int32 day = 1;
float return = 2;
}
}
message period_values {
repeated period_value value = 1;
}
message outcome {
repeated string col = 1;
repeated period_value val = 2;
}
outcome outcome_returns = 2;
}
I parse the json with the following code:
testjson = {
"outcome_returns": {
"col": [
"datetime",
"cum_returns"
],
"val": [[1199232000000, -0.0066705691], [1199318400000, -0.0033641154]]
}
}
import cum_ret_pb2 as CumRet
from google.protobuf.json_format import Parse
cumrets = Parse(json.dumps(test_json), CumRet.CumReturn())
But I got the error msg:
Failed to parse 1199232000000 field: expected string or bytes-like object...
Could someone help: getting a List of List of int and float into .proto schema?
One approach to implementing a list of lists is to create a new message for your list:
message ListOfInt {
repeated int32 value = 1;
}
and when you call this, use
message outcome {
repeated string col = 1;
repeated ListOfInt val = 2;
}
However, I think you have a different issue in your code. Your "period_value" message expects either an int32 or a float. The maximum int32 value is 2,147,483,647, but you are trying to fit 1,199,232,000,000 into that field. Hence, the error message that the value cannot be parsed. Try changing the int32 to an int64:
message period_value {
oneof types {
int64 day = 1;
float return = 2;
}
}
Now when I understand how the code works, I would like to translate it to C++.
The original Python code:
def recv_all_until(s, crlf):
data = ""
while data[-len(crlf):] != crlf:
data += s.recv(1)
return data
Here's what I tried:
std::string recv_all_until(int socket, std::string crlf)
{
std::string data = "";
char buffer[1];
memset(buffer, 0, 1);
while(data.substr(data.length()-2, data.length()) != crlf)
{
if ((recv(socket, buffer, 1, 0)) == 0)
{
if (errno != 0)
{
close(socket);
perror("recv");
exit(1);
}
}
data = data + std::string(buffer);
memset(buffer, 0, 1);
}
return data;
}
But it shows:
terminate called after throwing an instance of 'std::out_of_range'
what(): basic_string::substr
I understand that the problem is inside the while loop since at first the data string is empty. So how to improve this to make it work the same as it works in Python? Thank you.
You have the problem in the first iteration of your while loop:
Since the data is an empty string, data.length() is equal to 0, and thus you're calling data.substr(-2, 0).
To fix this, you need to add a check for the line length to the while statement.
Also, there's a method of finding such mistakes faster than writing a stackoverflow question about it. Consider reading this article.
If we first change your Python code a bit:
def recv_all_until(s, crlf):
data = ""
while not data.endswith(crlf):
data += s.recv(1)
return data
What we need to do in C++ becomes much clearer:
bool ends_with(const std::string& str, const std::string& suffix)
{
return str.size() >= suffix.size() &&
std::equal(suffix.rbegin(), suffix.rend(), str.rbegin());
}
std::string recv_all_until(int socket, const std::string& crlf)
{
std::string data = "";
char buffer[1];
memset(buffer, 0, 1);
while (!ends_with(data, crlf))
{
if ((recv(socket, buffer, 1, 0)) == 0)
{
if (errno != 0)
{
close(socket);
perror("recv");
exit(1);
}
}
data = data + std::string(buffer);
memset(buffer, 0, 1);
}
return data;
}