Develop Reference

Extracting JSON to PHP

I have this JSON data:
{
'total': 4744.134525437842,
'produksiHarian': [14.800870530853988, 15.639301040842536, 16.358413710544085, 16.952318230836113, 17.45055097248538, 17.8754763326927, 18.242760426469818, 18.563566329550866, 18.84608147199738, 19.096480365524762, 19.319535085761938, 19.519012642165947, 19.697941287301326, 19.85879401510806, 20.003618226362324, 20.134129444022808, 20.251780420564497, 20.357813020693253, 20.453297802680865, 20.539164652786695, 20.616226805275556, 20.685199899184028, 20.746717259937547, 20.801342274156227, 20.84957850042934, 20.891877998489687, 20.928648243306515, 20.960257904532423, 20.98704170951842, 21.009304559328097, 21.02732503324213, 21.04135838755367, 21.051639134485804, 21.058383270889877, 21.061790211998733, 21.06204447637534, 21.059317160271767, 21.053767231480556, 21.045542669893685, 21.034781475111366, 21.021612560179374, 21.00615654632769, 20.988526471783015, 20.968828425574475, 20.947162115751837, 20.923621380118846, 20.898294646278586, 20.871265347053306, 20.842612296170422, 20.81241002931908, 20.780729113365243, 20.747636428352394, 20.713195424315565, 20.677466356070354, 20.640506497917034, 20.602370340344187, 20.563109770866646, 20.522774239980293, 20.481410913972358, 20.43906481622342, 20.395778956975676, 20.351594453761813, 20.306550642902003, 20.260685182530857, 20.2140341483766, 20.166632122825597, 20.118512277471417, 20.069706450240275, 20.020245217044497, 19.970157959020252, 19.91947292463175, 19.868217288461143, 19.8164172056869, 19.764097863533138, 19.711283528975187, 19.657997593949958, 19.604262617587615, 19.550100366045164, 19.495531849637345, 19.44057735828815, 19.385256494483215, 19.329588204494947, 19.27359080791663, 19.217282025359857, 19.160679004832186, 19.103798346432747, 19.046656125958414, 18.989267917047492, 18.931648812181788, 18.873813442747196, 18.81577599788325, 18.757550242343445, 18.699149533671857, 18.6405868382421, 18.581874746778492, 18.523025488812625, 18.464050946690364, 18.40496266872333, 18.345771881921006, 18.286489503922823, 18.22712615435876, 18.167692166040553, 18.10819759492825, 18.048652230580856, 17.989065605305793, 17.929447003393125, 17.869805469665625, 17.810149817913633, 17.750488638676536, 17.690830306996318, 17.631182989544868, 17.571554651719957, 17.51195306422644, 17.452385809386776, 17.392860287453175, 17.333383722465378, 17.27396316777987, 17.214605511453826, 17.155317481724854, 17.096105651663066, 17.036976444166896, 16.977936136611227, 16.918990865183517, 16.860146629088725, 16.801409294861866, 16.742784600211646, 16.68427815766247, 16.625895458611, 16.567641876376022, 16.509522670079573, 16.45154298747203, 16.393707868422275, 16.336022247883598, 16.27849095868826, 16.221118734581758, 16.163910212830586, 16.106869936959832, 16.050002359157844, 15.993311842932966, 15.936802665326628, 15.880479019354743, 15.824345016126932, 15.768404686995313, 15.712661985734963, 15.657120790467255, 15.601784905648337, 15.546658063920907, 15.491743927918497, 15.437046092169679, 15.382568084608975, 15.328313368307523, 15.274285343221132, 15.220487347578999, 15.16692265923469, 15.113594497620934, 15.0605060246799, 15.007660346458055, 14.955060514464767, 14.90270952682481, 14.850610329633396, 14.798765818107297, 14.747178837940567, 14.69585218616739, 14.644788612493286, 14.593990820280863, 14.543461467685608, 14.493203168535093, 14.443218493447299, 14.39350997078121, 14.344080087478691, 14.294931290060344, 14.246065985484877, 14.197486542047413, 14.149195290142538, 14.101194523154854, 14.053486498128224, 14.00607343682552, 13.958957526073434, 13.912140918789852, 13.865625734588422, 13.819414060513688, 13.7735079516744, 13.727909431921148, 13.682620494481162, 13.637643102609184, 13.592979190079985, 13.548630662075965, 13.504599395247906, 13.460887238869981, 13.417496014964822, 13.374427519057994, 13.331683520572508, 13.289265763416816, 13.247175966417203, 13.205415823915152, 13.163987006093256, 13.122891159524887, 13.082129907627525, 13.041704851049357, 13.00161756813468, 12.961869615439428, 12.92246252783121, 12.883397819280237, 12.844676983002383, 12.806301491915125, 12.768272798969974, 12.730592337536208, 12.693261521807058, 12.656281747111938, 12.619654390164264, 12.583380809564924, 12.547462346008738, 12.511900322601504, 12.476696045229488, 12.441850802865273, 12.407365867693482, 12.373242495726032, 12.339481926742957, 12.30608538481023, 12.273054078443513, 12.240389200883609, 12.208091930367303, 12.176163430335453, 12.144604849869609, 12.113417323629534, 12.082601972368685, 12.052159902950859, 12.022092208828521, 11.992399969993468, 11.963084253360218, 11.934146112942067, 11.905586590110847, 11.87740671362035, 11.849607500150267, 11.822189954119143, 11.795155068047137, 11.768503822944655, 11.742237188086747, 11.716356121512085, 11.690861570169744, 11.665754469830294, 11.641035745721641, 11.61670631223086, 11.592767073301818, 11.56921892259734, 11.54606274368794, 11.523299410005265, 11.500929785229562, 11.478954723325614, 11.457375068721841, 11.436191656370681, 11.415405312010517, 11.395016852308974, 11.375027084858633, 11.355436808374336, 11.3362468128885, 11.31745787988622, 11.299070782263815, 11.281086284692925, 11.26350514351713, 11.246328107054104, 11.229555915546886, 11.213189301433452, 11.19722898932358, 11.181675696265113, 11.166530131659304, 11.151792997582067, 11.137464988637976, 11.12354679236573, 11.110039089001837, 11.096942551941297, 11.084257847527535, 11.071985635306518, 11.06012656807583, 11.04868129196148, 11.037650446580678, 11.027034665116119, 11.01683457427242, 11.00705079452253, 10.997683940178195, 10.988734619313746, 10.980203434096214, 10.972090980619463, 10.964397849202978, 10.957124624281988, 10.950271884558536, 10.943840203180603, 10.93783014765815, 10.932242279952963, 10.927077156683769, 10.922335329126735, 10.91801734312764, 10.914123739478882, 10.910655053684259, 10.907611816305053, 10.904994552693946, 10.90280378342666, 10.901040024063288, 10.899703785374516, 10.898795573363255, 10.898315889314617, 10.898265229903306, 10.898644087067368, 10.899452948405138, 10.900692296872883],
'r_squared': 0.9822381604039494,
'variabel': [152821.1630401214, 55.892346845823056, 23.76556145798208, 26712.22875205879, -0.6822113529338901]
}
This is my PHP code :
$result = shell_exec('C:\xampp\htdocs\prosold\python\venv\Scripts\python.exe C:/xampp/htdocs/prosold/python/main.py ' . escapeshellarg(json_encode($testday)));
//echo $output;
//$dataParse = [];
$dataParse = json_decode(json_encode($result), true);
echo $dataParse;
This is my Python code:
ParseData = {}
ParseData['total'] = totalProduksi
ParseData['produksiHarian'] = fittedData
ParseData['r_squared'] = r_square
ParseData['variabel'] = variabel
print(ParseData)
and when try to extract total, using this code:
$dataParse = json_decode(json_encode($result), true);
echo $dataParse['total'];
I get an error:
illegal string offset
But, when I try do this:
$dataParse = json_decode(json_encode($result), true);
echo $dataParse[0];
echo $dataParse[1];
echo $dataParse[2];
echo $dataParse[3];
This is the result:
{'to
What do I miss?

That data is JSON-like, but not valid JSON. The keys are wrapped in single quotes, but the JSON standard requires double quotes.
To produce valid JSON from a Python object use json.dumps()
For example:
import json
dictionary = {'a':34, 'b':61, 'c':82}
jsonString = json.dumps(dictionary, indent=4)
print(jsonString)
Which gives:
{
"a": 34,
"b": 61,
"c": 82
}
In that form PHP should be able to read your data and decode it with json_decode()

How to split a file into multiple files based on a repeated string?

i have a file and a want to split the based on the string "async" into different files. The expected output is a little messy. I try to use a word as key ("async") to divide the file but the generated files have the first line of its function with the context of the below function. For example, the file is:
'use strict';
const shim = require('fabric-shim');
const util = require('util');
let Chaincode = class {
async Init(stub) {
let ret = stub.getFunctionAndParameters();
console.info(ret);
console.info('=========== Instantiated Marbles Chaincode ===========');
return shim.success();
}
async Invoke(stub) {
console.info('Transaction ID: ' + stub.getTxID());
console.info(util.format('Args: %j', stub.getArgs()));
let ret = stub.getFunctionAndParameters();
console.info(ret);
let method = this[ret.fcn];
if (!method) {
console.log('no function of name:' + ret.fcn + ' found');
throw new Error('Received unknown function ' + ret.fcn + ' invocation');
}
try {
let payload = await method(stub, ret.params, this);
return shim.success(payload);
} catch (err) {
console.log(err);
return shim.error(err);
}
}
async initMarble(stub, args, thisClass) {
if (args.length != 4) {
throw new Error('Incorrect number of arguments. Expecting 4');
}
// ==== Input sanitation ====
console.info('--- start init marble ---')
if (args[0].lenth <= 0) {
throw new Error('1st argument must be a non-empty string');
}
if (args[1].lenth <= 0) {
throw new Error('2nd argument must be a non-empty string');
}
if (args[2].lenth <= 0) {
throw new Error('3rd argument must be a non-empty string');
}
if (args[3].lenth <= 0) {
throw new Error('4th argument must be a non-empty string');
}
let marbleName = args[0];
let color = args[1].toLowerCase();
let owner = args[3].toLowerCase();
let size = parseInt(args[2]);
if (typeof size !== 'number') {
throw new Error('3rd argument must be a numeric string');
}
let marbleState = await stub.getState(marbleName);
if (marbleState.toString()) {
throw new Error('This marble already exists: ' + marbleName);
}
// ==== Create marble object and marshal to JSON ====
let marble = {};
marble.docType = 'marble';
marble.name = marbleName;
marble.color = color;
marble.size = size;
marble.owner = owner;
await stub.putState(marbleName, Buffer.from(JSON.stringify(marble)));
let indexName = 'color~name'
let colorNameIndexKey = await stub.createCompositeKey(indexName, [marble.color, marble.name]);
console.info(colorNameIndexKey);
console.info('- end init marble');
}
i tried this:
import re
import os
filetype = '.js'
result = ''
count = 0
start = 0
name = 'functions'
matchedLine = ''
stringToMatch = 'async'
with open ('myjson.js', 'r') as f:
for x in f.read().split("\n"):
if stringToMatch in x:
if (start == 1):
with open (name + str(count) + '.js', 'w') as opf:
matchedLine = x
opf.write(matchedLine + '\n' + result)
opf.close()
result = ''
print (count)
count+= 1
matchedLine = ''
else:
start = 1
else:
if (result == ''):
result = x
else:
result = result + '\n' + x
but the output is a little bit messy
function0.js:
async Invoke(stub) {
'use strict';
const shim = require('fabric-shim');
const util = require('util');
let Chaincode = class {
let ret = stub.getFunctionAndParameters();
console.info(ret);
console.info('=========== Instantiated Marbles Chaincode ===========');
return shim.success();
}
function1.js:
async initMarble(stub, args, thisClass) {
console.info('Transaction ID: ' + stub.getTxID());
console.info(util.format('Args: %j', stub.getArgs()));
let ret = stub.getFunctionAndParameters();
console.info(ret);
let method = this[ret.fcn];
if (!method) {
console.log('no function of name:' + ret.fcn + ' found');
throw new Error('Received unknown function ' + ret.fcn + ' invocation');
}
try {
let payload = await method(stub, ret.params, this);
return shim.success(payload);
} catch (err) {
console.log(err);
return shim.error(err);
}
}

There must be many ways to do this. Here is one:
import re
class Writer:
def __init__(self):
self._num = 0
self._fh = None
def close(self):
if self._fh:
self._fh.close()
def start_file(self):
self.close()
self._fh = open("file{}.js".format(self._num), "w")
self._num += 1
def write(self, data):
if self._fh:
self._fh.write(data)
writer = Writer()
with open('myjson.js') as f:
for line in f:
if re.match(' *async ', line):
writer.start_file()
writer.write(line)
writer.close()

If your goal is to separate all the sections that have async code into individual files, one method you might try would be to count the curly brackets for open, and then closed. To do this, you would set a variable that positively increments for every { and negatively for each } e.g (not optimized/pretty, just explaining).
brackets = 0
buffer = ""
found_async = False
for line_of_code in code:
if "async" in line_of_code:
if "{" in line_of_code:
brackets += 1
if "}" in line_of_code:
brackets -= 1
buffer += line_of_code
if brackets == 0:
write_buffer_to_file_here
buffer = ""
As a concept, this will probably not work as is, but should give you an idea of what I'm trying to say.

Avoid Selenium to be detected by javascript duck typing in python?

Is there any way to change Selenium properties in order to avoid its own detection by duck typing method in Javascript?
Last time I was hardly looking to make Selenium to be fully undetected and identically to Chrome. Almost succeed but just found this new detection method which I cannot bypass: duck typing in Javascript.
Here is the Javascript detection code:
// Opera 8.0+
var isOpera = (!!window.opr && !!opr.addons) || !!window.opera || navigator.userAgent.indexOf(' OPR/') >= 0;
// Firefox 1.0+
var isFirefox = typeof InstallTrigger !== 'undefined';
// Safari 3.0+ "[object HTMLElementConstructor]"
var isSafari = /constructor/i.test(window.HTMLElement) || (function (p) { return p.toString() === "[object SafariRemoteNotification]"; })(!window['safari'] || (typeof safari !== 'undefined' && safari.pushNotification));
// Internet Explorer 6-11
var isIE = /*#cc_on!#*/false || !!document.documentMode;
// Edge 20+
var isEdge = !isIE && !!window.StyleMedia;
// Chrome 1 - 71
var isChrome = !!window.chrome && (!!window.chrome.webstore || !!window.chrome.runtime);
// Blink engine detection
var isBlink = (isChrome || isOpera) && !!window.CSS;
var output = 'Detecting browsers by ducktyping:<hr>';
output += 'isFirefox: ' + isFirefox + '<br>';
output += 'isChrome: ' + isChrome + '<br>';
output += 'isSafari: ' + isSafari + '<br>';
output += 'isOpera: ' + isOpera + '<br>';
output += 'isIE: ' + isIE + '<br>';
output += 'isEdge: ' + isEdge + '<br>';
output += 'isBlink: ' + isBlink + '<br>';
document.body.innerHTML = output;
Source: https://jsfiddle.net/6spj1059/
Look what are the results for this js code in real Chrome browser:
results for real Chrome browser
While all of them are FALSE when open the same jsfiddle link in Selenium.
Here is what I did into Python. I tried to change window title in python by asserting it as Chrome since just by setting it directly isn't possible. Also tried to first navigate to the chrome://newtab url and after that to the url I want but also fails.
browser.get("chrome://newtab")
try:
assert u'New Tab' in browser.title
print "Assertion succeed!"
print browser.title
except AssertionError:
_, _, tb = sys.exc_info()
traceback.print_tb(tb) # Fixed format
tb_info = traceback.extract_tb(tb)
filename, line, func, text = tb_info[-1]
print('An error occurred on line {} in statement {}'.format(line, text))
exit(1)
print("Page Title is :", browser.title)
clear_cache_and_cookies(browser)
browser.get(url16)
print browser.title
print type(browser.title)
print u'New Tab'
print type(u'New Tab')
Thank you for your help!

Should be:
driver.execute_script("""
window.chrome.webstore = true
""")

Convert a file to static C string declaration

I would like convert number of files to static string declarations in C. I have trying writing a quick script in Python (shown below), but it doesn't seem exactly simple and a number of issue came up trying to compile the output.
import os, sys
from glob import glob
from re import sub
test_dirs = ('basics', 'float', 'import', 'io', 'misc')
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
def cfunc_name(t):
return sub(r'/|\.|-', '_', t)
for t in tests:
print("void {}(void* data) {{".format(cfunc_name(t)))
with open(t) as f:
lines = ''.join(f.readlines())
cstr = sub('"', '\\"', lines)
cstr = sub('\n', '\"\n\"', cstr)
print(" const char * pystr = \"\"\n\"{}\";".format(cstr))
print("end:\n ;\n}")
print("struct testcase_t core_tests[] = {")
for t in tests:
print(" {{ \"{}\", test_{}_fn, TT_ENABLED_, 0, 0 }},".format(t, cfunc_name(t)))
print("END_OF_TESTCASES };")
Looking for an existing tool is not exactly obvious (may be my search keywords are not quite right)... Is there a simple UNIX tool that does this or has anyone come across something similar?

Does this work for you? https://code.google.com/p/txt2cs/
The main issue I can think of is new lines and escaping if you want to roll your own.

I have used the txt2cs implementation as a reference and ended-up with just a few lines of Python that do all the escaping. As I didn't want to add extra things to the build system, it's easier to have this done in Python. This is going to be integrated in test automation, which is already a complex beast.
The main takeaway is that RE substitutions have to be done in a certain order and aren't the ideal tool for this purpose.
import os, sys
from glob import glob
from re import sub
def escape(s):
lookup = {
'\0': '\\0',
'\t': '\\t',
'\n': '\\n\"\n\"',
'\r': '\\r',
'\\': '\\\\',
'\"': '\\\"',
}
return "\"\"\n\"{}\"".format(''.join([lookup[x] if x in lookup else x for x in s]))
def chew_filename(t):
return { 'func': "test_{}_fn".format(sub(r'/|\.|-', '_', t)), 'desc': t.split('/')[1] }
def script_to_map(t):
r = { 'name': chew_filename(t)['func'] }
with open(t) as f: r['script'] = escape(''.join(f.readlines()))
return r
test_function = (
"void {name}(void* data) {{\n"
" const char * pystr = {script};\n"
" do_str(pystr);\n"
"}}"
)
testcase_struct = (
"struct testcase_t {name}_tests[] = {{\n{body}\n END_OF_TESTCASES\n}};"
)
testcase_member = (
" {{ \"{desc}\", {func}, TT_ENABLED_, 0, 0 }},"
)
testgroup_struct = (
"struct testgroup_t groups[] = {{\n{body}\n END_OF_GROUPS\n}};"
)
testgroup_member = (
" {{ \"{name}/\", {name}_tests }},"
)
test_dirs = ('basics', 'float', 'import', 'io', 'misc')
output = []
for group in test_dirs:
tests = glob('{}/*.py'.format(group))
output.extend([test_function.format(**script_to_map(test)) for test in tests])
testcase_members = [testcase_member.format(**chew_filename(test)) for test in tests]
output.append(testcase_struct.format(name=group, body='\n'.join(testcase_members)))
testgroup_members = [testgroup_member.format(name=group) for group in test_dirs]
output.append(testgroup_struct.format(body='\n'.join(testgroup_members)))
print('\n\n'.join(output))
Below is what the output looks like, as you can see the initial ""\n and '\\n\"\n\"' make it quite a bit more readable:
void test_basics_break_py_fn(void* data) {
const char * pystr = ""
"while True:\n"
" break\n"
"\n"
"for i in range(4):\n"
" print('one', i)\n"
" if i > 2:\n"
" break\n"
" print('two', i)\n"
"\n"
"for i in [1, 2, 3, 4]:\n"
" if i == 3:\n"
" break\n"
" print(i)\n"
"";
do_str(pystr);
}

Python pexpect sendline contents being buffered?

I have the following code in Python using the pexpect module to run a perl script which asks a series of questions. I have used this for quite some time and works well until now.
pexpect python
index = child.expect(['Question 1:'])
os.system('sleep 2')
print 'Question 1:' + 'answer1'
child.sendline('answer1')
index = child.expect(['Question 2:'])
os.system('sleep 2')
print 'Question 1:' + 'answer2'
child.sendline('answer2')
index = child.expect(['Question 3:'])
os.system('sleep 2')
print 'Question 1:' + 'answer2'
child.sendline('answer2')
At this point, i have some code that checks if error will output when Question 2 & Question 3 does not match. I checked the pexpect log and the statements being sent are exactly what i want (number of bytes and string).
However when I check what gets accepted by the perl code its getting the following:
Question 1: 'answer 1' <-- CORRECT
Question 2: 'answer 1' <-- INCORRECT
Question 3: 'answer 2answer 2' <-- its being combined into one for some reason
Any ideas out there? I'm not using anything special when spawning my pexpect object: child=pexpect.spawn(cmd,timeout=140)
Edit: Added the perl code function that asks for Question 2 & 3
sub getpw
sub getpw
{ my ($name, $var, $encoding) = #_;
my $pw = $$var;
PWD: while(1) {
system("/bin/stty -echo");
getvar($name, \$pw, 1);
print "\n";
system("/bin/stty echo");
return if $pw eq $$var && length($pw) == 80;
if (length($pw) > 32) {
print STDERR "ERROR: Password cannot exceed 32 characters, please reenter.\n";
next PWD;
}
return if $pw eq $$var;
my $pw2;
system("/bin/stty -echo");
getvar("Repeat password", \$pw2, 1);
print "\n";
system("/bin/stty echo");
print "#1: ";
print $pw;
print "\n";
print "#2: ";
print $pw2;
if ($pw2 ne $pw) {
print STDERR "ERROR: Passwords do not match, please reenter.\n";
next PWD;
}
last;
}
# Escape dangerous shell characters
$pw =~ s/([ ;\*\|`&\$!#\(\)\[\]\{\}:'"])/\\$1/g;
my $correctlength=80;
my $encoded=`$AVTAR --quiet --encodepass=$pw`;
chomp($encoded);
if($? == 0 && length($encoded) == $correctlength) {
$$var = $encoded;
} else {
print "Warning: Password could not be encoded.\n";
$$var = $pw;
}
}
sub getvar
sub getvar
{ my ($name, $var, $hide) = #_;
my $default = $$var;
while(1) {
if($default) {
$default = "*****" if $hide;
print "$name [$default]: ";
} else {
print "$name: ";
}
my $val = <STDIN>;
chomp $val;
### $val =~ s/ //g; # do not mess with the password
$$var = $val if $val;
last if $$var;
print "ERROR: You must enter a value\n";
}
}

There is nothing wrong with your code on the python side. Check your perl side of the code. To demonstrate I have created below a python questions and answers scripts that call each other. When you execute answers.py it will produce the expected output as below. Also you don't need to do the sleep statements child.expect will block until the expected output has been received.
questions.py
answers = {}
for i in range(1, 4):
answers[i] = raw_input('Question %s:' % i)
print 'resuls:'
print answers
answers.py
import pexpect
child=pexpect.spawn('./questions.py', timeout=140)
for i in range(1, 4):
index = child.expect('Question %s:' % i)
child.sendline('answer%s' % i)
child.expect('resuls:')
print child.read()
output
{1: 'answer1', 2: 'answer2', 3: 'answer3'}