We have a Python script that creates a csv file of enterprise data. One part of the enterprise data is a list of nacecodes (can be None) looking like this once its written to the csv file ['47299', '8690901', '4729903', '86909'] (It's one cell).
In a second script, this time written in Node.js, we parse the csv file with papaparse. We want the nacecodes to be an array but it's a string looking like "['47299', '8690901', '4729903', '86909']"
How can we parse this string to an array? I had found a possible solution by using JSON.parse but its given me a Unexpected token ' in JSON at position 1
Python script
class Enterprise:
def __init__(self):
self.enterprise_number = ''
self.vat_number = ''
self.nace_codes = set()
self.tel = ''
self.mobile = ''
self.email = ''
def to_json(self):
return {
'enterprise_number': self.enterprise_number if self.enterprise_number != '' else None,
'vat_number': self.vat_number if self.vat_number != '' else None,
'nace_codes': list(self.nace_codes) if len(self.nace_codes) > 0 else None
'tel': self.tel if self.tel != '' else None,
'mobile': self.mobile if self.mobile != '' else None,
'email': self.email if self.email != '' else None,
}
def read_data():
...
with open('enterprise_data.csv', 'w',) as file:
writer = csv.writer(file, delimiter=';')
writer.writerow(['enterprise_number', 'vat_number', 'name', 'nace_codes', 'type_of_enterprise', 'juridical_form', 'start_date', 'county', 'city', 'address', 'postal_code', 'box', 'group_part', 'group_number', 'tel', 'mobile', 'email', 'is_active'])
with open('data/enterprise_insert.csv') as file:
for line in islice(file, 1, None):
enterprise = Enterprise()
line = line.rstrip()
...
formatted_data = enterprise.to_json()
writer.writerow([formatted_data['enterprise_number'], formatted_data['vat_number'], formatted_data['nace_codes'], formatted_data['tel'], formatted_data['mobile'], formatted_data['email'])
Node.js script
const csvFilePath = 'data/enterprise_data.csv'
const readCSV = async (filePath) => {
const csvFile = fs.readFileSync(filePath);
const csvData = csvFile.toString();
return new Promise(resolve => {
Papa.parse(csvData, {
header: true,
skipEmptyLines: true,
transformHeader: header => header.trim(),
complete: results => {
console.log('Read', results.data.length, 'records.');
resolve(results.data);
}
});
});
};
const start = async () => {
try {
let parsedData = await readCSV(csvFilePath);
parsedData.map((row, i) => {
console.log(`${i} | ${row.enterprise_number}`);
const nace_codes = row.nace_codes ? JSON.parse(row.nace_codes) : '';
console.log('Parsed value: ', nace_codes);
});
} catch(error) {
console.log(`Crashed | ${error} `);
}
}
start();
Assuming that csvData does look like ['47299', '8690901', '4729903', '86909'].
What’s wrong is that single quote is not accepted in JSON, so JSON.parse throws an error.
To fix this you simply need to replace all occurrences of single quotes by double quotes like so:
const csvData = csvFile.toString().replaceAll("'", '"')
Related
I am in the process of incorporating the gate,io rest api and am currently trying to convert the signature function from python to php(laravel).
Apparently there is a bug hiding in the conversion.
Can someone take a look and tell me if this is all correct or if something is missing here?
For improvement suggestions I would be grateful
Python code:
def gen_sign(method, url, query_string=None, payload_string=None):
key = '' # api_key
secret = '' # api_secret
t = time.time()
m = hashlib.sha512()
m.update((payload_string or "").encode('utf-8'))
hashed_payload = m.hexdigest()
s = '%s\n%s\n%s\n%s\n%s' % (method, url, query_string or "", hashed_payload, t)
sign = hmac.new(secret.encode('utf-8'), s.encode('utf-8'), hashlib.sha512).hexdigest()
return {'KEY': key, 'Timestamp': str(t), 'SIGN': sign}
Source: Gate.io API Signature string generation
Php Code:
public function createSignature($method, $url, $query=null, $payload=null, $algo = 'sha256'){
$key = 'xxx';
$secret= 'xxx';
$time = microtime(true);
$hashed_payload = hash_hmac($algo,$query ?? '');
$string = "{$methode}\n{$url}\n{$query ?? ''}\n{$hashed_payload}\n{$time}"
$sign = hash_hmac($algo,$string,$secret)
return ['KEY' => $key, 'Timestamp' => "{$time}", 'SIGN' => $sign]
}
i got the answer, i hope it will helps:
public function buildSignHeaders($method, $resourcePath, $queryParams = [], $payload = null)
{
$fullPath = parse_url(config('gate-io.host'), PHP_URL_PATH) . $resourcePath;
$fmt = "%s\n%s\n%s\n%s\n%s";
$timestamp = time();
$hashedPayload = hash("sha512", ($payload !== null) ? $payload : "");
$signatureString = sprintf(
$fmt,
$method,
$fullPath,
GuzzleHttp\Psr7\build_query($queryParams, false),
$hashedPayload,
$timestamp
);
$signature = hash_hmac("sha512", $signatureString, config('gate-io.apiSecretKey'));
return [
"KEY" => config('gate-io.apiKey'),
"SIGN" => $signature,
"Timestamp" => $timestamp
];
}
I'm trying to fetch all the data i have in my mongodb collection and for some reason i can't do it.
I can get more than one result but if i try to get more than x results, it stops working.
I'm using Flask, MongoDB, pymongo to work with mongodb and React.
This is my Flask function.
# app.route("/escoller-centro-proba", methods=["POST"])
# cross_origin()
def search_proba():
if request.method == "POST":
centros = []
resultadosPing = []
resultadosNmap = []
codigo = request.json['codigo'].upper()
query = {"centro": {"$regex": codigo}}
resultados = collection.find(query)
for resultado in resultados:
centroId = str(resultado["_id"])
centros.append({"_id": centroId, "sf": resultado["sf"], "centro": resultado["centro"], "concello": resultado["concello"], "lan": resultado["lan"], "dhcp": resultado["dhcp"],
"tecnoloxia": resultado["tecnoloxia"], "tecnoloxia_respaldo": resultado["tecnoloxia_respaldo"], "eva": resultado["eva"]})
if len(centros) > 1:
return jsonify({"centros": centros, "resultadosPing": resultadosPing, "resultadosNmap": resultadosNmap})
return jsonify({"centro": centros[0], "resultadosPing": resultadosPing, "resultadosNmap": resultadosNmap})
else:
return "Método non POST"
And here the JS function.
const escollerCentro = async (e) => {
e.preventDefault()
const res = await instance.post("http://127.0.0.1:5000/escoller-centro-proba", {
codigo: codigo.trim().toUpperCase()
})
console.log(res.data)
if (res.data.centro === "O centro non existe") {
setError("O centro non existe")
setIsError(true)
return;
}
if (res.data.centros) {
tabsInfoVar[value].cras = res.data.centros
tabsInfoVar[value].centro = {
centro: "",
resultadosPing: [],
resultadosNmap: []
}
}
if (res.data.centro) {
tabsInfoVar[value].centro = {
img: img,
centro: res.data.centro.centro,
index: res.data.centro._id,
concello: res.data.centro.concello,
lan: res.data.centro.lan,
dhcp: res.data.centro.dhcp ? "Si" : "Non",
sf: res.data.centro.sf,
tecnoloxia: res.data.centro.tecnoloxia,
tecnoloxia_respaldo: res.data.centro.tecnoloxia_respaldo,
eva: res.data.centro.eva,
resultadosPing: [],
resultadosNmap: []
}
tabsInfoVar[value].cras = []
}
tabsInfoVar[value].resultadosPing = res.data.resultadosPing
tabsInfoVar[value].resultadosNmap = res.data.resultadosNmap
const resultadosPing = []
for (var i = 0; i < tabsInfoVar[value].resultadosPing.length; i++) {
if(tabsInfoVar[value].resultadosPing[i] !== null) {
resultadosPing.push(tabsInfoVar[value].resultadosPing[i])
} else {
console.log("Resultado con valor nulo")
}
}
const resultadosNmap = []
for (var i = 0; i < tabsInfoVar[value]?.resultadosNmap.length; i++) {
if(tabsInfoVar[value].resultadosNmap[i] !== null) {
resultadosNmap.push(tabsInfoVar[value].resultadosNmap[i])
} else {
console.log("Resultado con valor nulo")
}
}
tabsInfoVar[value].resultadosPing = resultadosPing;
tabsInfoVar[value].resultadosNmap = resultadosNmap;
setTabsInfo([...tabsInfoVar])
}
As I said, if i fetch less than 13 results, the code works. I get an array from the database and my frontend can work with it. Here an example: I searched "RIANXO" and it shows me all the results that contains "Rianxo".
Here I'm searching "CRA", it should show an array of 168 results. Instead, i get this:
It shows all the results but not an array and as you can see, there is a label ("Show more") that I have to press if I want to see all the data.
I think it is a problem with mongodb, because i did exactly this but working with excel instead of mongodb and had no problem fetching all the data, 1275 results.
Thank you all.
I have found the problem. When i saved all the data from excel to mongodb, empty cells filled with NaN value and when the frontend was trying to get the attribute it was displaying the data in the way i have already shown.
The answer was to modify the value of those cells with df = df.fillna (" ").
Here is the code:
for index, centro in enumerate(centros):
data = df[df["Centro"].str.contains(centro)]
id = df.iloc[index]["ID"]
centro = df.iloc[index]["Centro"]
concello = df.iloc[index]["Concello"]
lan = df.iloc[index]["LAN"]
dhcp = df.iloc[index]["DHCP"]
tecnoloxia = df.iloc[index]["Tecnoloxía Acceso Principal"]
tecnoloxia_respaldo = df.iloc[index]["Tecnoloxía Acceso Respaldo"]
eva = "Non"
if dhcp == "Si":
dhcp = True
else:
dhcp = False
lan = format_lan(lan)
df = df.fillna("")
if tecnoloxia_respaldo == "":
tecnoloxia_respaldo = "Non ten liña de backup"
centroNovo = {"sf": id, "centro": centro, "concello": concello, "lan": lan, "dhcp": dhcp,
"tecnoloxia": tecnoloxia, "tecnoloxia_respaldo": tecnoloxia_respaldo, "eva": eva}
print(centroNovo)
collection.insert_one(centroNovo)
JavaScript is throwing an error 'Uncaught Syntax Error: Unexpected token '&''
when debugged in Views.py I got he data with proper Apostrophes.
def newEntry(request):
assert isinstance(request, HttpRequest)
i = 1
for x in lines:
for line in x:
cursor.execute("select distinct regionn FROM [XYZ].[dbo].[Errors] where [Linne] like '%" +line+ "%'")
region[i] = cursor.fetchall()
i = i+1
return render(
request,
'app/newEntry.html',
{
'title': 'New Entry',
'year':datetime.now().year,
'lines': lines,
'regions': region,
}
)
and here is my JS code
var Regions= {{regions}}
function changecat(value) {
if (value.length == 0) document.getElementById("category").innerHTML = "<option>default option here</option>";
else {
var catOptions = "";
for (categoryId in Regions[value]) {
catOptions += "<option>" + categoryId+ "</option>";
}
document.getElementById("category").innerHTML = catOptions;
}
}
Thanks in advance, if this is not a best practice to carry data, suggest me some best process which fills my requirement
1.
I had to send request.POST a dict of dict,
big_dict = {k:{kk:vv, kk1: vv1}, k1:{kk:vv, kk1: vv1}}
and ajax always somehow made it into one dict like
one_dict = {k[kk]:vv, k[kk1]:vv1, k1[kk]:vv, k1[kk1]:vv1}
someone told me to
JSON.stringify(big_dict)
before ajax request and then json.loads it in my python django views.
json.loads(request.POST.get('data'))
it worked all damn fine till someone uploaded a chinese character named file.
At the third step of json.loads in my view,
(wrong)
the key and value of my big_dict is still good, looking fine, displaying proper chinese character.
(edited)
the key and value of my big_dict, chinese characters in one value looks good, and not in the other.
{'0': {'designer': 'EV', 'number': '0229', 'version': '', 'bracket': '(120门幅)', 'imgUrl': 'http://127.0.0.1:8000/media/cache/temp/EV-0229(120%E9%97%A8%E5%B9%85%EF%BC%89.jpg'}}
but when I try to
for k, v in big_dict.items():
for kk, vv in v.items():
# do something with kk and vv
chinese characters within vv becomes all wrong.
and os is telling me no such file.
It looked like a simple problem, I searched a web for over an hour and tried things, nothing worked..
Thanks anyone in advance for helping me out.
EDIT
script from template
$('.designInfoSave').click(function() {
let num = $('.designBox').length;
let bcontext = {};
for (let i = 0; i<num; i++) {
let infoBox = $('#'+i+"-infoBox");
let image = $('#image'+i);
image.toArray();
let children = infoBox.children();
children.toArray();
bcontext[i] = {
'designer': children[2].value,
'number': children[4].value,
'version': children[6].value,
'bracket': children[8].value,
'imgUrl': image[0].currentSrc,
};
}
$.ajax({
type: 'POST',
url: "{% url 'update_des_info' %}",
data: {'csrfmiddlewaretoken': '{{ csrf_token }}', 'context':JSON.stringify(bcontext)},
dataType: "json",
success: function(response){
$("#album").hide();
$(".ajaxResult").show();
}
}
)
});
views.py
import json
import os.path
def update_des_info(request):
if request.method != 'POST':
return
queryset = json.loads(request.POST.get('context'))
print(queryset)
exist = 0
created = 0
for k, v in queryset.items():
designer = v['designer']
number = v['number']
version = v['version']
bracket = v['bracket']
fp = v['imgUrl'][v['imgUrl'].rfind('/media/')+6:]
n, c = Design.objects.get_or_create(designer=designer, number=number, version=version)
if c is True:
created += 1
else:
exist += 1
if n.bracket != bracket:
n.bracket = bracket
fpp = settings.MEDIA_ROOT + fp
p, fn = os.path.split(fpp)
nfp = os.path.join(settings.MEDIA_ROOT, 'designs', 'image', fn)
shutil.move(fpp, nfp)
if not n.image:
n.image = nfp
n.save()
I have a file that starts something like:
databaseCons = {
main = {
database = "readable_name",
hostname = "hostdb1.serv.com",
instances= {
slaves = {
conns = "8"
}
}
maxconns = "5",
user = "user",
pass = "pass"
}
}
So, what I'd like to do is parse this out into a dict of sub-dicts, something like:
{'main': {'database': 'readable_name', 'hostname': 'hostdb1.serv.com', 'maxconns': '5', 'instances': {'slave': {'maxCount': '8'}}, 'user': 'user', 'pass': 'pass'}}
I think the above makes sense... but please feel free to edit this if it doesn't. Basically I want the equivalent of:
conns = '8'
slave = dict()
slave['maxCount'] = conns
instances = dict()
instances['slave'] = slave
database = 'readable_name'
hostname = 'hostdb1.serv.com'
maxconns = '5'
user = 'user'
pas = 'pass'
main = dict()
main['database'] = database
main['hostname'] = hostname
main['instances'] = instances
main['maxconns'] = maxconns
main['user'] = user
main['pass'] = pas
databaseCons = dict()
databaseCons['main'] = main
Are there any modules out there that can handle this sort of parsing? Even what I've suggested above looks messy.. there's got to be a better way I'd imagine.
Here is a pyparsing parser for your config file:
from pyparsing import *
def to_dict(t):
return {k:v for k,v in t}
series = Forward()
struct = Suppress('{') + series + Suppress('}')
value = quotedString.setParseAction(removeQuotes) | struct
token = Word(alphanums)
assignment = Group(token + Suppress('=') + value + Suppress(Optional(",")))
series << ZeroOrMore(assignment).setParseAction(to_dict)
language = series + stringEnd
def config_file_to_dict(filename):
return language.parseFile(filename)[0]
if __name__=="__main__":
from pprint import pprint
pprint(config_file_to_dict('config.txt'))