Develop Reference

Parse csv containing array in one column in Node.js

We have a Python script that creates a csv file of enterprise data. One part of the enterprise data is a list of nacecodes (can be None) looking like this once its written to the csv file ['47299', '8690901', '4729903', '86909'] (It's one cell).
In a second script, this time written in Node.js, we parse the csv file with papaparse. We want the nacecodes to be an array but it's a string looking like "['47299', '8690901', '4729903', '86909']"
How can we parse this string to an array? I had found a possible solution by using JSON.parse but its given me a Unexpected token ' in JSON at position 1
Python script
class Enterprise:
def __init__(self):
self.enterprise_number = ''
self.vat_number = ''
self.nace_codes = set()
self.tel = ''
self.mobile = ''
self.email = ''
def to_json(self):
return {
'enterprise_number': self.enterprise_number if self.enterprise_number != '' else None,
'vat_number': self.vat_number if self.vat_number != '' else None,
'nace_codes': list(self.nace_codes) if len(self.nace_codes) > 0 else None
'tel': self.tel if self.tel != '' else None,
'mobile': self.mobile if self.mobile != '' else None,
'email': self.email if self.email != '' else None,
}
def read_data():
...
with open('enterprise_data.csv', 'w',) as file:
writer = csv.writer(file, delimiter=';')
writer.writerow(['enterprise_number', 'vat_number', 'name', 'nace_codes', 'type_of_enterprise', 'juridical_form', 'start_date', 'county', 'city', 'address', 'postal_code', 'box', 'group_part', 'group_number', 'tel', 'mobile', 'email', 'is_active'])
with open('data/enterprise_insert.csv') as file:
for line in islice(file, 1, None):
enterprise = Enterprise()
line = line.rstrip()
...
formatted_data = enterprise.to_json()
writer.writerow([formatted_data['enterprise_number'], formatted_data['vat_number'], formatted_data['nace_codes'], formatted_data['tel'], formatted_data['mobile'], formatted_data['email'])
Node.js script
const csvFilePath = 'data/enterprise_data.csv'
const readCSV = async (filePath) => {
const csvFile = fs.readFileSync(filePath);
const csvData = csvFile.toString();
return new Promise(resolve => {
Papa.parse(csvData, {
header: true,
skipEmptyLines: true,
transformHeader: header => header.trim(),
complete: results => {
console.log('Read', results.data.length, 'records.');
resolve(results.data);
}
});
});
};
const start = async () => {
try {
let parsedData = await readCSV(csvFilePath);
parsedData.map((row, i) => {
console.log(`${i} | ${row.enterprise_number}`);
const nace_codes = row.nace_codes ? JSON.parse(row.nace_codes) : '';
console.log('Parsed value: ', nace_codes);
});
} catch(error) {
console.log(`Crashed | ${error} `);
}
}
start();

Assuming that csvData does look like ['47299', '8690901', '4729903', '86909'].
What’s wrong is that single quote is not accepted in JSON, so JSON.parse throws an error.
To fix this you simply need to replace all occurrences of single quotes by double quotes like so:
const csvData = csvFile.toString().replaceAll("'", '"')

How to get progress of file upload using requests.post() if the file was just uploaded via form-data? (Not sure whether this is a streaming process)

Suppose I'm running some kind of web service with python & flask aiming to provide file upload. It should be noted that my service is only a kind of transfer station, which means I'll "repost" the file elsewhere, say, a massive centralized file storage, as soon as I receive a file from frontend form-data. The flask code looks like this:
#admin.route('/data', methods=['POST'])
def data_upload():
if 'file' in request.files:
f = request.files['file']
try:
r = Requests.post(DataConfig.server + '/' + '/upload.php', files={'file': (f.filename, f)})
return {'e': 0, 'msg': r.content.decode('utf-8')}
except RequestException:
return ReturnCode.ERR_NETWORK_FAILURE
return ReturnCode.ERR_BAD_DATA
It is not allowed that the frontend directly send file or data to "upload.php", since the server key should not be exposed to users.
Now I've got two problems that confuse me.
Is this process streaming or streaming-ready? I mean, whether my python server will receive the file and store it somewhere temporarily, or the file will be repost in chunks, like iter_block? If streaming is not readily enabled, how can I do so to enable large-file uploading user experience?
Is it possible to provide the user (frontend, browser) information about the progress of uploading?
For question 2, there exists some posts indicating that tqdm will help, but tqdm is a cli tool and printing anything in the python server's terminal makes no sense, and document of tqdm doesn't show me any obvious reference about their apis to get a percentage. What's more, I think that maybe sending this percentage to frontend may require some xhr techniques, which I believe my code can't deal with.

PART 1 OF MY ANSWER : THE JAVASCRIPT
async function ubox_send_chucks (divid) {
var chunstart = 0;
var chunend = 0;
var chunksize = 1024 * 1024 * 9;
var newblobchun_id = parseInt(Math.floor(Math.random() * 100000000) + 1);
fi = $("#"+divid).files;
stubuploads = fi.length;
for (var i = 0; i < fi.length; i++) {
var thismediaby_id = (newblobchun_id + i + 3);
$("#progressx").append("<div id=\'loaderz_"+thismediaby_id+"_message\' class=\'padding3 margin3\' >FILE "+i+" (PREP)</div>");
$("#phpx").append("<div id=\'phpz_"+thismediaby_id+"_message\' class=\'padding3 margin3\' >FILE "+i+" (PREP)</div>");
}
for (var i = 0; i < fi.length; i++) {
if ( fi[i].size > 0 ) {
var numberofchunks = Math.ceil( fi[i].size / chunksize );
var thismediaby_id = (newblobchun_id + i + 3);
logx("FILE "+i+" -- size: "+fi[i].size+" name: "+fi[i].name+" n of chunks to send: "+numberofchunks );
// SEND EACH CHUNKS
for (var c = 0; c <= numberofchunks; c++) {
chunstart = (c * chunksize); chunend = chunstart + chunksize + 1; if ( chunend > fi[i].size ){ chunend = fi[i].size; }
var thischunk = fi[i].slice(chunstart, chunend);
var thismediaby_name = thismediaby_id+"--chunk_"+c;
console.log("FILE "+i+" send chunk: "+c+" start: "+chunstart+" end: "+chunend);
upload_percent = ( c / numberofchunks ) * 100;
$("#loaderz_"+thismediaby_id+"_message").html("FILE "+i+" : " + Math.round(upload_percent) + " %");
var fd = new FormData();
fd.append("data", thischunk, encodeURIComponent(thismediaby_name));
fd.append("thismedia_id", encodeURIComponent(thismediaby_id));
fd.append("thismedia_name", encodeURIComponent(fi[i].name));
fd.append("numberofchunks", encodeURIComponent(numberofchunks));
fd.append("thischunk_number", encodeURIComponent(c));
fd.append("thisfilex", encodeURIComponent(i));
fd.append("thissession", encodeURIComponent(thissession));
fd.append("howmanyfiles", encodeURIComponent(fi.length));
var pcache = (Math.floor(Math.random() * 100000000) + 1);
await fetch("/templates/tests_ubox_slice/slice_receiver.php?pcache="+pcache, { method: "POST", body: fd })
.then(function (response) { return response.text(); })
.then(function (html) { $("#php_message").html(html); })
}
// WHEN ALL CHUNKS SENT TRIGGER A RECOMBINE (SAFER)
// AJAX FUNCTION HERE https://stubs.s3.filebase.com/media/stubs/202111100393/recookies.js
var combinex = [];
combinex["thismedia_id"] = encodeURIComponent(thismediaby_id);
combinex["thissession"] = encodeURIComponent(thissession);
combinex["thismedia_name"] = encodeURIComponent(fi[i].name);
stubajax("combiner","/templates/tests_ubox_slice/slice_combiner.php?pcache="+pcache,combinex);
}
}
}
PART II PHP RECEIVER
function clean () { ....stuff to make user input more secure like rawurldecode, html_entity_decode, stripslashes, etc }
$pcache = clean( $_REQUEST['pcache'] ?? '' ) ;
$thismedia_id = clean( $_REQUEST['thismedia_id'] ?? '' ) ;
$thismedia_name = clean( $_REQUEST['thismedia_name'] ?? '' );
$thismedia_ext = pathinfo($exif_file)['extension'] ?? '';
$numberofchunks = clean( $_REQUEST['numberofchunks'] ?? '' ) ;
$thischunk_number = clean( $_REQUEST['thischunk_number'] ?? '' ) ;
$thisfilex = clean( $_REQUEST['thisfilex'] ?? '' ) ;
$howmanyfiles = clean( $_REQUEST['howmanyfiles'] ?? '' ) ;
$thissession = clean( $_REQUEST['thissession'] ?? '' ) ;
if ( $pcache != '' ) {
// DEV
// var_dump(['thismedia_id'=>$thismedia_id,'thismedia_name'=>$thismedia_name,'numberofchunks'=>$numberofchunks,'thischunk_number'=>$thischunk_number,'thisfilex'=>$thisfilex]);
// WHERE TO SAVE CHUNKS
$received_chunk_dir = '/temporary_path_here_you_receiver_chunks/'.$thissession.'/'.$thismedia_id.'/';
$received_chunk_path = $received_chunk_dir.$thismedia_id.'_chunk_'.$thischunk_number;
// IF DIRECTORY NOT THERE CREATE IT
if ( !file_exists($received_chunk_dir) ) { shell_exec('mkdir -p "'.$received_chunk_dir.'"'); }
// MOVE_UPLOADED_FILE
foreach ($_FILES as $thisfilekey => $thisfilechun) {
if ( isset($thisfilechun['name']) && isset($thisfilechun['tmp_name']) ) {
if ( filesize($thisfilechun['tmp_name']) > 1 ) { move_uploaded_file($thisfilechun['tmp_name'],$received_chunk_path); }
}
}
// ECHO PERCENT PROGRESSION FOR THAT FILE
echo '<script>$("#phpz_'.$thismedia_id.'_message").html("FILE '.$thisfilex.' : received chunk number '.$thischunk_number.' of '.$numberofchunks.' chunks");</script>';
}
PART III PHP COMBINER
$pcache = clean( $_REQUEST['pcache'] ?? '' ) ;
$thismedia_id = clean( $_REQUEST['thismedia_id'] ?? '' ) ;
$thismedia_name = accentx(clean( $_REQUEST['thismedia_name'] ?? '' ),'unlock');
$thismedia_ext = exiftensionx(['exif_file'=>$thismedia_name]);
$numberofchunks = clean( $_REQUEST['numberofchunks'] ?? '' ) ;
$thischunk_number = clean( $_REQUEST['thischunk_number'] ?? '' ) ;
$thisfilex = clean( $_REQUEST['thisfilex'] ?? '' ) ;
$howmanyfiles = clean( $_REQUEST['howmanyfiles'] ?? '' ) ;
$thissession = clean( $_REQUEST['thissession'] ?? '' ) ;
if ( $thissession != '' ) {
// PATH
$received_chunk_dir = '/temporary_path_here_you_receiver_chunks/'.$thissession;
$received_final_path = $received_chunk_dir.'/'.$thismedia_id.'/'.$thismedia_id.'.'.$thismedia_ext;
// GET OF SORTED FILES -V because chunk_1, chunk_2, ...., chunk_9, chunk_10, chunk_11 ==> dont want chunk_1,chunk_10,chunk_2 but chunk_1,chunk_2,...,chunk_10
$all_chunks_raw = shell_exec('ls '.$received_chunk_dir.'/'.$thismedia_id.'/* | sort -V');
if ( $all_chunks_raw != '' ) {
// GET LS OF ALL CHUNKS
$all_chunks_explo = array_filter(explode(PHP_EOL,$all_chunks_raw));
// IF ONLY 1 CHUNK JUST RENAME
if ( count($all_chunks_explo) == 1 ) { rename($all_chunks_explo[0],$received_final_path); #unlink($all_chunks_explo[0]); }
else {
// RECOMBINE ALL CHUNKS WITH FOPEN FREAD chunksize = 1024 * 1024 * 9 = 9437184 from javascript HAS TO BE THE SAME VALUE
foreach ( $all_chunks_explo as $chunkey => $chunx ){
$file = fopen($chunx, 'rb'); $buff = fread($file, 9437184); fclose($file);
$final = fopen($received_final_path, 'ab'); $write = fwrite($final, $buff); fclose($final);
}
// DELETE CHUNKS AFTER COMBINE
shell_exec('ls '.$received_chunk_dir.'/'.$thismedia_id.' -name "*_chunk_*" -delete');
}
}
// HERE YOU CAN FFMPEG, IMAGEMAGICK, ETC TO CONVERT TO WEB COMPATIBLE FORMAT
// HERE YOU CAN SEND FILES TO S3 BUCKET (services like filebase.com)
// DELETE FILE AFTER SENDING TO S3 BUCKET IF YOU NEED TO CLEAR SPACE
echo '<script>$("#phpz_'.$thismedia_id.'_message").append(" ---COMBINATION DONE---");</script>';
}
******** NOTE : because of async: it's important to WAIT for all chuncks BEFORE combining because of networks (internet) factors, chunks don't always arrive one after an other, some times, you get chunk 1, then 3 then 6 then 2. it's why a ajax call sends a signal to combiner to "tell" ok, all chuncks are sent

$query = http_build_query($params); $auth_hash = hash_hmac('sha512', $query, $secret_key);

I have this php api request example I want to convert to python
function sendRequest($url, $params, $apiKey, $secretKey) {
$query = http_build_query($params);
$params['api_key'] = $apiKey;
$params['auth_hash'] = hash_hmac('sha512', $query, $secretKey);
$params['nonce'] = explode(' ', microtime())[1];
$ch = curl_init();
curl_setopt_array($ch, array(
CURLOPT_URL => $url,
CURLOPT_POST => true,
CURLOPT_POSTFIELDS => http_build_query($params),
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_RETURNTRANSFER => true
));
$response = curl_exec($ch);
curl_close($ch);
return $response;}
And this is my python code but it keep returning incorrect hash, please how can I make this request in python?
import hashlib
import hmac
import requests
import time
params = {
'name':'Matthew Jackson',
'state':'nj',
'zip':'034572',
'address':'14 Main Street',
'phone':'',
'dob':''}
def request_ss( url, params, api_key, secretKey):
tosign = "&".join( [i + '=' + params[i] for i in params] )
secretKey = str.encode(secretKey)
tosign = str.encode(tosign)
sign = hmac.new(secretKey, tosign , hashlib.sha512)
res_body = {
'api_key': api_key,
'auth_hash': sign.hexdigest(),
'nonce': str(time.time())}
r = requests.post(url, data=res_body)
return r.text
create_url = 'api_url'
res = request_ss(create_url,params,api_key, api_secret)
print(res)
This code keep returning wrong hash from the server, I don't know if am right or wrong on the way I coded this on python, The goal is just to make a request and get the data from the server, the API provider only give the request example in php in which I don't understand. Your support is going to be highly appriciated

Create Signature using SHA256 encrypyion with .p12 file in python

Can any one help to do the same code in Python
<?php
$payload = '{"amount":205.16,"currency":"AED","language":"en","merchant_code":"O8Ryrfvy8aUb","merchant_order":"931","redirect_urls":{"cancel":"http://127.0.0.1:8000/api/pointspay-cancel","fail":"http://127.0.0.1:8000/api/pointspay-failed","success":"http://127.0.0.1:8000/api/pointspay-success"},"timestamp":1604722764987,"type":"direct"}';
echo x509_fingerprint($payload, '/home/tmt/Documents/Protect4less/LoyLogic/pointspay_certificate.p12', 'Protect4Less#123', 'sha256') . "\n\n";
function x509_fingerprint($payload, $cert_path, $cert_pass, $algorithm = 'sha256')
{
$algorithm = in_array($algorithm, array(
'sha1',
'md5',
'sha256'
)) ? $algorithm : 'sha1';
// 1. Capture the private key from certificate
if (!$cert_store = file_get_contents($cert_path)) {
echo "Error: Unable to read the cert file\n";
exit;
}
if (openssl_pkcs12_read($cert_store, $cert_info, $cert_pass)) {
} else {
echo "Error: Unable to read the cert store.\n";
exit;
}
$privateKey = $cert_info['pkey'];
print_r($privateKey);
// 2. create message digest
$messageDigest = openssl_digest($payload, $algorithm, true);
// 3. Sign the message digest using private key
openssl_private_encrypt($messageDigest, $encryptedData, $privateKey);
return base64_encode($encryptedData);
}
?>
This is PHP code and I want to do the same steps in PYTHON, can any 1 help me to do so.
I tried many things in python but was not able to do so.
Following is the Python code I tried, but the final signature doesn't match with the PHP signature
from OpenSSL import crypto
import hashlib
from base64 import b64encode
str1 = b'{"amount":205.16,"currency":"AED","language":"en","merchant_code":"O8Ryrfvy8aUb","merchant_order":"931","redirect_urls":{"cancel":"http://127.0.0.1:8000/api/pointspay-cancel","fail":"http://127.0.0.1:8000/api/pointspay-failed","success":"http://127.0.0.1:8000/api/pointspay-success"},"timestamp":1604722764987,"type":"direct"}'
psw = 'Protect4Less#123'
p12 = crypto.load_pkcs12(open("/home/tmt/Documents/Protect4less/LoyLogic/pointspay_certificate.p12", 'rb').read(), psw)
pkey = p12.get_privatekey()
my_hash2 = hashlib.sha256() #Choose SHA256 and update with same bytes
my_hash2.update(str1)
hash_digest = my_hash2.hexdigest()
print("Result after digesting: " + str(my_hash2.hexdigest()))
# data = str1
my_signature = crypto.sign(pkey, str1, 'sha256')
signature_b64 = b64encode(my_signature)
print(signature_b64)

Interacting with suiteCRM API V7.9 with Python requests

I've been trying for a while now to connect to the suiteCRM API using Python's request module but to no avail, the official tutorial uses PHP: https://docs.suitecrm.com/developer/api/api-4_1/
I can't seem to create the Python equivalent of this request, no matter what I get the same response:
<pre>/**
* SugarWebServiceImplv4_1.php
*
* This class is an implementation class for all the web services. Version 4_1 adds limit/off support to the
* get_relationships function. We also added the sync_get_modified_relationships function call from version
* one to facilitate querying for related meetings/calls contacts/users records.
*
*/
Class [ <user> class SugarWebServiceImplv4_1 extends SugarWebServiceImplv4 ] {
- Constants [0] {
}
- Static properties [1] {
Property [ public static $helperObject ]
}
- Static methods [0] {
}
- Properties [0] {
}
- Methods [36] {
/**
* Class Constructor Object
*
*/
Method [ <user, overwrites SugarWebServiceImplv4, ctor> public method __construct ] {
}
/**
* Retrieve a collection of beans that are related to the specified bean and optionally return relationship data for those related beans.
* So in this API you can get contacts info for an account and also return all those contact's email address or an opportunity info also.
*
* #param String $session -- Session ID returned by a previous call to login.
* #param String $module_name -- The name of the module that the primary record is from. This name should be the name the module was developed under (changing a tab name is studio does not affect the name that should be passed into this method)..
* #param String $module_id -- The ID of the bean in the specified module
* #param String $link_field_name -- The name of the lnk field to return records from. This name should be the name the relationship.
* #param String $related_module_query -- A portion of the where clause of the SQL statement to find the related items. The SQL query will already be filtered to only include the beans that are related to the specified bean.
... that continues on, it looks like some documentation.
This is my code:
def get_info():
headers = {
"Content-Type": "application/json"
}
creds = OrderedDict()
creds = {"user_auth": {'user_name':"***", "password": "***"}}
creds = json.dumps(creds)
data = {
'method':'login',
'input_type': 'JSON',
'response_type':'JSON',
'rest_data': creds
}
data = json.dumps(data)
response = requests.post("http://example.com/suitecrm/service/v4_1/rest.php", headers=headers, data=data)
print(response.text)
return response.text
Does anyone have any experience doing this? Thank you
edit: this is the PHP call from their docs:
<?php
$url = "http://example.com/suitecrm/service/v4_1/rest.php";
function restRequest($method, $arguments){
global $url;
$curl = curl_init($url);
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
$post = array(
"method" => $method,
"input_type" => "JSON",
"response_type" => "JSON",
"rest_data" => json_encode($arguments),
);
curl_setopt($curl, CURLOPT_POSTFIELDS, $post);
$result = curl_exec($curl);
curl_close($curl);
return json_decode($result,1);
}
$userAuth = array(
'user_name' => 'suitecrmuser',
'password' => md5('suitecrmpassword'),
);
$appName = 'My SuiteCRM REST Client';
$nameValueList = array();
$args = array(
'user_auth' => $userAuth,
'application_name' => $appName,
'name_value_list' => $nameValueList);
$result = restRequest('login',$args);
$sessId = $result['id'];
$entryArgs = array(
//Session id - retrieved from login call
'session' => $sessId,
//Module to get_entry_list for
'module_name' => 'Accounts',
//Filter query - Added to the SQL where clause,
'query' => "accounts.billing_address_city = 'Ohio'",
//Order by - unused
'order_by' => '',
//Start with the first record
'offset' => 0,
//Return the id and name fields
'select_fields' => array('id','name',),
//Link to the "contacts" relationship and retrieve the
//First and last names.
'link_name_to_fields_array' => array(
array(
'name' => 'contacts',
'value' => array(
'first_name',
'last_name',
),
),
),
//Show 10 max results
'max_results' => 10,
//Do not show deleted
'deleted' => 0,
);
$result = restRequest('get_entry_list',$entryArgs);
print_r($result);

Please check this working example, it should help you get started. Its using python3.
import urllib.request
import json
import hashlib
encode = hashlib.md5("MasterPass".encode('utf-8'))
encodedPassword = encode.hexdigest()
args = {'user_auth': {'user_name': 'admin','password': encodedPassword}}
crmUrl="https://yourname.crm.cr/service/v4_1/rest.php"
data = json.dumps(args)
args = {'method': 'login', 'input_type': 'json',
'response_type' : 'json', 'rest_data' : data}
params = urllib.parse.urlencode(args).encode('utf-8')
response = urllib.request.urlopen(crmUrl, params)
response = response.read().strip()
if not response:
print( "error: " , response)
result = json.loads(response.decode('utf-8'))
print (result)