Having issue when reading binary data of float - python

I want to use os module specifically to handle read/write binary files. I have an issue when reading values of data type that takes more than 1 byte such as int64, float32, ... etc. To illustrate my issue, let's see the following example I wrote. I generate random values of type np.float64 which is 8 byte each:
# Write
n = 10
dim = 2
fd = os.open('test.dat', os.O_CREAT | os.O_WRONLY)
data_w = np.random.uniform(low=0.5, high=13.3, size=(n,dim)).astype(np.float64)
print("Written Data are:\n%s\n" % data_w)
os.write(fd, data_w.tobytes())
os.close(fd)
print("------------------ \n")
# Read
start_read = 0 # 0 for now. Later I can read from any row!
total_num_to_read = n*dim
fd = os.open('test.dat', os.O_RDONLY)
os.lseek(fd, start_read, 0) # start_read from the beginning 0
raw_data = os.read(fd, total_num_to_read) # How many values to be read
data_r = np.fromiter(raw_data, dtype=np.float64).reshape(-1, dim)
print("Data Read are:\n%s\n" % data_r)
os.close(fd)
The reading is not correct. Look how it is returned:
Written Data are:
[[ 2.75763292 9.87883101]
[ 1.73752327 9.9633879 ]
[ 1.01616811 1.81174597]
[ 9.93904659 10.6757686 ]
[ 7.02452029 2.68652109]
[ 5.29766028 11.15384409]
[ 4.12499766 10.37214532]
[11.75811252 3.30378401]
[ 1.72738203 2.11228277]
[ 7.7321937 11.64298051]]
------------------
Data Read are:
[[250. 87.]
[227. 216.]
[161. 15.]
[ 6. 64.]
[162. 178.]
[ 59. 35.]
[246. 193.]
[ 35. 64.]
[218. 97.]
[ 81. 50.]]
I cannot retrieve it correctly! I thought np.fromiter(raw_data, dtype=np.float64).reshape(-1, dim) is supposed to take care of it but I don't know where the issue is. How can I read binary data in this case given that I know it is of particular data type (i.e., np.float64)?


You should use np.fromstring(raw_data) instead of fromiter(). Check documentation for the purpose of each function. In addition, when reading from file, read the correct number of bytes!!!: 8* total_num_to_read.
In [103]: # Write
...: n = 10
...: dim = 2
...: fd = os.open('test.dat', os.O_CREAT | os.O_WRONLY)
...: data_w = np.random.uniform(low=0.5, high=13.3, size=(n,dim)).astype(np.float64)
...: print("Written Data are:\n%s\n" % data_w)
...: os.write(fd, data_w.tobytes())
...: os.close(fd)
...: print("------------------ \n")
...:
...: # Read
...: start_read = 0 # 0 for now. Later I can read from any row!
...: total_num_to_read = n*dim
...: fd = os.open('test.dat', os.O_RDONLY)
...: os.lseek(fd, start_read, 0) # start_read from the beginning 0
...: raw_data = os.read(fd, 8*total_num_to_read) # How many values to be read
...: data_r = np.fromstring(raw_data, dtype=np.float64).reshape(-1, dim)
...: print("Data Read are:\n%s\n" % data_r)
...: os.close(fd)
...:
...:
Written Data are:
[[ 11.2465988 5.45304778]
[ 12.06466331 9.95717255]
[ 7.35402895 1.68972606]
[ 0.7259652 1.01265826]
[ 3.11340311 2.44725153]
[ 2.82109715 5.02768335]
[ 12.69054614 9.26028537]
[ 5.13785639 2.0780649 ]
[ 4.6796513 4.24710598]
[ 2.34859141 8.87224674]]
------------------
Data Read are:
[[ 11.2465988 5.45304778]
[ 12.06466331 9.95717255]
[ 7.35402895 1.68972606]
[ 0.7259652 1.01265826]
[ 3.11340311 2.44725153]
[ 2.82109715 5.02768335]
[ 12.69054614 9.26028537]
[ 5.13785639 2.0780649 ]
[ 4.6796513 4.24710598]
[ 2.34859141 8.87224674]]

Related

Gekko seems to ignore the single equation

I've been using Gekko for the last three years for optimization purposes. As a matter of fact, I'm using Gekko version 1.0.4. Recently, I've been trying to solve an ESG-MVP problem developed by (Vo, He et al. 2019):
Considering sum(wesg) should be equal to 1 and 0 ≤ wesg_i ≤ 1 for iε[0, 1, 2,..., 23]. In this model, I'm trying to
find the best values of wesg_i for the model above. Hence, according to the information provided, I have to find the value of 24 variables through an optimization model. My code is available for you to test on your computer with all the vital data needed.
Libraries
from gekko import GEKKO
import numpy as np
import pandas as pd
Data Section
Make sure copy it all.
esg_scores_df = pd.DataFrame({'Score 2019': {'MSFT': 7,'PEP': 7,'TSLA': 4,'AMZN': 6,'LKQ': 3,'ABMD': 3,'MSI': 10,'PH': 8,'NKE': 6,'TM': 7,'EOG': 7,'GOOGL': 5,'NFLX': 4,'GS': 6,'EQIX': 9,'EA': 7,'AAP': 6,'TEL': 9,'DG': 6,'EXR': 5,'MDLZ': 6,'FIS': 8,'CRL': 8,'RCL': 9},
'Score 2020': {'MSFT': 6,'PEP': 11,'TSLA': 4,'AMZN': 6,'LKQ': 4,'ABMD': 4,'MSI': 8,'PH': 8,'NKE': 6,'TM': 7,'EOG': 7,'GOOGL': 5,'NFLX': 3,'GS': 6,'EQIX': 9,'EA': 6,'AAP': 7,'TEL': 9,'DG': 6,'EXR': 4,'MDLZ': 6,'FIS': 8,'CRL': 8,'RCL': 7}})
predicted_return_dataframe = pd.DataFrame({'MSFT': 1.0982472257593677e-15,'PEP': 4.567069595849647e-09,'TSLA': 7.439258841202596e-10,'AMZN': 3.176883309676764e-07,'LKQ': 4.825709334830293e-05,'ABMD': 2.0608058642685837e-05,'MSI': -3.1789250961959136e-12,'PH': -2.257237871892785e-07,'NKE': -4.737530217609426e-07,'TM': 9.951932427172896e-07,'EOG': 1.184074639824261e-08,'GOOGL': -1.7027206923083418e-10,'NFLX': 2.948344729885545e-05,'GS': 1.458862302453713e-07,'EQIX': 4.620207503091301e-07,'EA': -1.297137640100873e-06,'AAP': -3.493153382990658e-07,'TEL': 8.202463568291799e-08,'DG': 2.743802312024829e-05,'EXR': 1.971994818465128e-11,'MDLZ': 1.578772827915881e-08,'FIS': 1.3314663987166631e-08,'CRL': 4.112083587242872e-05,'RCL': -1.470829206425942e-09} , index = [0])
r = pd.DataFrame({'MSFT': -0.029838842874655123,'PEP': -0.012265555866403165,'TSLA': -0.053599428210690636,'AMZN': -0.02614876342440773,'LKQ': -0.026658034482507052,'ABMD': -0.017274321994365294,'MSI': -0.018624198080929168,'PH': -0.02871952268633362,'NKE': -0.028820168231794074,'TM': -0.011208781085200492,'EOG': -0.04606883798249617,'GOOGL': -0.027983366801701926,'NFLX': -0.0070609860042608165,'GS': -0.02429261616692462,'EQIX': -0.015181566244607758,'EA': -0.012009295653284258,'AAP': -0.011979871239295382,'TEL': -0.021782414925879602,'DG': -0.009820085124020328,'EXR': -0.012233190959318829,'MDLZ': -0.015917631061436933,'FIS': -0.023877815976094417,'CRL': -0.020169902356185498,'RCL': -0.06831607178882856} , index = [0])
sigma = pd.DataFrame(np.array([[ 1.02188994e-04, 3.16399586e-05, 8.54243667e-05,
7.14148287e-05, 3.55313650e-05, 5.44796564e-05,
3.39128448e-05, 6.65003271e-05, 5.42932808e-05,
2.29333906e-05, 4.26407605e-05, 7.92791756e-05,
9.28910574e-05, 6.36770764e-05, 3.75024646e-05,
5.13720672e-05, 1.33646720e-05, 6.21865710e-05,
4.72888751e-05, 2.35546749e-06, 2.49700413e-05,
6.74503236e-05, 7.08948785e-05, 5.55096135e-05],
[ 3.16399586e-05, 6.47824823e-05, -2.57461009e-05,
1.81931187e-05, 2.11795049e-05, 1.01708683e-05,
3.33739243e-05, 1.23893822e-05, 2.74825418e-05,
1.35954411e-05, 2.37108310e-06, 2.34208728e-05,
1.96972793e-05, 1.09335123e-05, 4.26042775e-05,
2.76849308e-05, 6.01821651e-06, 8.04387109e-06,
3.10892437e-05, 2.79970364e-05, 3.70572613e-05,
3.08215527e-05, 2.57244480e-05, 8.72002781e-06],
[ 8.54243667e-05, -2.57461009e-05, 8.84138653e-04,
6.16478906e-05, -1.78063009e-05, 3.62712388e-05,
-6.15200573e-05, 8.63597848e-05, 2.12571109e-05,
3.21749681e-05, 6.54693912e-05, 7.95298035e-05,
1.53917874e-04, 8.06306277e-05, -5.27893372e-06,
2.60899474e-05, 4.29017379e-05, 6.13689353e-05,
2.30097086e-06, -4.45963115e-05, -2.83669072e-06,
2.95395217e-05, 4.53889239e-05, 3.11615533e-05],
[ 7.14148287e-05, 1.81931187e-05, 6.16478906e-05,
1.19357840e-04, 5.48763301e-05, 5.55161292e-05,
2.77288469e-05, 6.25231790e-05, 5.19494452e-05,
1.93193913e-05, 4.28946368e-05, 7.00210371e-05,
9.92756795e-05, 6.54630027e-05, 1.73935405e-05,
4.46819063e-05, 2.94199398e-05, 5.76749375e-05,
3.75634244e-05, -5.53847280e-06, 9.72193169e-06,
4.42756733e-05, 4.21469376e-05, 4.35344231e-05],
[ 3.55313650e-05, 2.11795049e-05, -1.78063009e-05,
5.48763301e-05, 3.49607174e-04, 1.58156275e-04,
-1.64565378e-05, 5.95602984e-05, 3.36138721e-05,
2.25095973e-05, 4.90144491e-05, 5.60207136e-05,
3.73119885e-05, 8.65010497e-05, -1.68774498e-05,
1.06928038e-05, 3.53732120e-05, 8.46071106e-05,
3.89200328e-05, -1.10740797e-05, 4.20238479e-06,
2.07371361e-05, 2.27640364e-05, 7.39017715e-05],
[ 5.44796564e-05, 1.01708683e-05, 3.62712388e-05,
5.55161292e-05, 1.58156275e-04, 1.12266763e-03,
2.49786917e-05, 5.60053839e-05, -2.57686022e-05,
3.00855419e-05, 2.09079416e-04, 1.21797841e-04,
3.80689080e-06, 1.17305259e-04, -4.95192897e-05,
5.56059321e-05, 2.87765760e-06, 1.13640096e-04,
-2.73690072e-05, 2.17458024e-05, -1.24009254e-05,
6.11287657e-05, 1.12849096e-04, 1.17234141e-04],
[ 3.39128448e-05, 3.33739243e-05, -6.15200573e-05,
2.77288469e-05, -1.64565378e-05, 2.49786917e-05,
1.84271573e-04, -2.99455473e-06, 3.56885701e-05,
1.53451236e-05, 3.20222217e-06, 3.00619247e-05,
5.17902613e-05, 1.69851832e-05, 7.75007583e-05,
7.49839347e-06, -3.78485373e-05, -1.28942631e-05,
4.36657573e-05, 3.97590833e-05, 4.04704969e-05,
4.05201790e-05, 4.01384618e-05, 4.82818631e-06],
[ 6.65003271e-05, 1.23893822e-05, 8.63597848e-05,
6.25231790e-05, 5.95602984e-05, 5.60053839e-05,
-2.99455473e-06, 2.86568704e-04, 6.55278686e-05,
4.08938883e-05, 1.18822482e-04, 9.32577766e-05,
7.92498191e-05, 1.38967662e-04, -1.64128182e-05,
4.16393158e-05, 1.06300676e-04, 1.31672888e-04,
6.64839109e-05, -5.14108669e-05, -1.03044572e-05,
1.59107041e-05, 7.65457600e-05, 1.31611693e-04],
[ 5.42932808e-05, 2.74825418e-05, 2.12571109e-05,
5.19494452e-05, 3.36138721e-05, -2.57686022e-05,
3.56885701e-05, 6.55278686e-05, 1.36374334e-04,
1.36781102e-05, 2.38544176e-05, 5.90104054e-05,
6.20754315e-05, 5.84082445e-05, 3.96422393e-05,
2.96064418e-05, 5.54456912e-05, 5.69396332e-05,
7.28867485e-05, 7.92870807e-06, 3.79528145e-05,
3.38036564e-05, 3.47015652e-05, 3.86545249e-05],
[ 2.29333906e-05, 1.35954411e-05, 3.21749681e-05,
1.93193913e-05, 2.25095973e-05, 3.00855419e-05,
1.53451236e-05, 4.08938883e-05, 1.36781102e-05,
5.30619573e-05, 4.42693706e-05, 2.92937393e-05,
1.15184525e-05, 3.04141154e-05, 1.11981894e-06,
1.29388215e-05, 1.41913521e-05, 2.37704063e-05,
1.27983719e-05, -3.53963791e-06, 5.41505031e-06,
1.21874870e-05, 2.04099948e-05, 2.89151466e-05],
[ 4.26407605e-05, 2.37108310e-06, 6.54693912e-05,
4.28946368e-05, 4.90144491e-05, 2.09079416e-04,
3.20222217e-06, 1.18822482e-04, 2.38544176e-05,
4.42693706e-05, 5.01946804e-04, 7.79572195e-05,
7.61563242e-05, 1.31597522e-04, -2.43789107e-05,
6.77962026e-05, 5.39745318e-05, 9.05223133e-05,
-1.88433584e-06, -1.63784490e-05, -1.04001201e-05,
1.79450881e-05, 5.33951974e-05, 1.07102393e-04],
[ 7.92791756e-05, 2.34208728e-05, 7.95298035e-05,
7.00210371e-05, 5.60207136e-05, 1.21797841e-04,
3.00619247e-05, 9.32577766e-05, 5.90104054e-05,
2.92937393e-05, 7.79572195e-05, 1.10688236e-04,
8.13294795e-05, 8.72981821e-05, 2.13737337e-05,
4.33439061e-05, 3.06090710e-05, 7.60853329e-05,
4.44363057e-05, 3.33458671e-06, 1.60908360e-05,
4.09067616e-05, 5.72051871e-05, 6.05239478e-05],
[ 9.28910574e-05, 1.96972793e-05, 1.53917874e-04,
9.92756795e-05, 3.73119885e-05, 3.80689080e-06,
5.17902613e-05, 7.92498191e-05, 6.20754315e-05,
1.15184525e-05, 7.61563242e-05, 8.13294795e-05,
3.77348891e-04, 8.99709029e-05, 2.58732673e-05,
8.24015922e-05, 2.77333081e-05, 4.74146019e-05,
3.76467363e-05, -5.60031564e-06, 2.25653918e-05,
6.22679966e-05, 3.92394407e-05, 6.47067147e-05],
[ 6.36770764e-05, 1.09335123e-05, 8.06306277e-05,
6.54630027e-05, 8.65010497e-05, 1.17305259e-04,
1.69851832e-05, 1.38967662e-04, 5.84082445e-05,
3.04141154e-05, 1.31597522e-04, 8.72981821e-05,
8.99709029e-05, 1.54802097e-04, -1.57072336e-05,
3.58129141e-05, 5.52667399e-05, 1.02065825e-04,
4.96942712e-05, -2.44193897e-05, -5.66188177e-06,
2.35884764e-05, 5.15584350e-05, 9.98376706e-05],
[ 3.75024646e-05, 4.26042775e-05, -5.27893372e-06,
1.73935405e-05, -1.68774498e-05, -4.95192897e-05,
7.75007583e-05, -1.64128182e-05, 3.96422393e-05,
1.11981894e-06, -2.43789107e-05, 2.13737337e-05,
2.58732673e-05, -1.57072336e-05, 1.54605175e-04,
1.07085192e-05, -1.14274871e-05, -1.26204468e-05,
2.50810750e-05, 5.60340845e-05, 4.60520336e-05,
4.29696767e-05, 5.33004299e-05, -1.56063893e-05],
[ 5.13720672e-05, 2.76849308e-05, 2.60899474e-05,
4.46819063e-05, 1.06928038e-05, 5.56059321e-05,
7.49839347e-06, 4.16393158e-05, 2.96064418e-05,
1.29388215e-05, 6.77962026e-05, 4.33439061e-05,
8.24015922e-05, 3.58129141e-05, 1.07085192e-05,
1.32544808e-04, 3.15561503e-05, 3.62681876e-05,
1.76503394e-05, 1.90982407e-07, 1.02992114e-05,
2.81341342e-05, 2.81970597e-05, 3.94413355e-05],
[ 1.33646720e-05, 6.01821651e-06, 4.29017379e-05,
2.94199398e-05, 3.53732120e-05, 2.87765760e-06,
-3.78485373e-05, 1.06300676e-04, 5.54456912e-05,
1.41913521e-05, 5.39745318e-05, 3.06090710e-05,
2.77333081e-05, 5.52667399e-05, -1.14274871e-05,
3.15561503e-05, 2.71861487e-04, 6.04272650e-05,
3.55256428e-05, -2.37123843e-05, -1.99844011e-05,
-2.02943319e-07, -4.12746174e-07, 6.80701129e-05],
[ 6.21865710e-05, 8.04387109e-06, 6.13689353e-05,
5.76749375e-05, 8.46071106e-05, 1.13640096e-04,
-1.28942631e-05, 1.31672888e-04, 5.69396332e-05,
2.37704063e-05, 9.05223133e-05, 7.60853329e-05,
4.74146019e-05, 1.02065825e-04, -1.26204468e-05,
3.62681876e-05, 6.04272650e-05, 1.67171047e-04,
5.06392219e-05, -1.02673299e-05, 1.64715372e-05,
2.50897861e-05, 5.81836032e-05, 1.00294572e-04],
[ 4.72888751e-05, 3.10892437e-05, 2.30097086e-06,
3.75634244e-05, 3.89200328e-05, -2.73690072e-05,
4.36657573e-05, 6.64839109e-05, 7.28867485e-05,
1.27983719e-05, -1.88433584e-06, 4.44363057e-05,
3.76467363e-05, 4.96942712e-05, 2.50810750e-05,
1.76503394e-05, 3.55256428e-05, 5.06392219e-05,
1.99771178e-04, 4.70710752e-06, 3.76160380e-05,
2.30014580e-05, 3.12554634e-05, 5.94948441e-06],
[ 2.35546749e-06, 2.79970364e-05, -4.45963115e-05,
-5.53847280e-06, -1.10740797e-05, 2.17458024e-05,
3.97590833e-05, -5.14108669e-05, 7.92870807e-06,
-3.53963791e-06, -1.63784490e-05, 3.33458671e-06,
-5.60031564e-06, -2.44193897e-05, 5.60340845e-05,
1.90982407e-07, -2.37123843e-05, -1.02673299e-05,
4.70710752e-06, 8.86508330e-05, 4.02039134e-05,
2.39025489e-05, -5.94660474e-06, -1.92301274e-05],
[ 2.49700413e-05, 3.70572613e-05, -2.83669072e-06,
9.72193169e-06, 4.20238479e-06, -1.24009254e-05,
4.04704969e-05, -1.03044572e-05, 3.79528145e-05,
5.41505031e-06, -1.04001201e-05, 1.60908360e-05,
2.25653918e-05, -5.66188177e-06, 4.60520336e-05,
1.02992114e-05, -1.99844011e-05, 1.64715372e-05,
3.76160380e-05, 4.02039134e-05, 8.23927778e-05,
3.21948208e-05, 3.50482953e-05, -1.97971783e-07],
[ 6.74503236e-05, 3.08215527e-05, 2.95395217e-05,
4.42756733e-05, 2.07371361e-05, 6.11287657e-05,
4.05201790e-05, 1.59107041e-05, 3.38036564e-05,
1.21874870e-05, 1.79450881e-05, 4.09067616e-05,
6.22679966e-05, 2.35884764e-05, 4.29696767e-05,
2.81341342e-05, -2.02943319e-07, 2.50897861e-05,
2.30014580e-05, 2.39025489e-05, 3.21948208e-05,
1.20680211e-04, 5.96710279e-05, 3.14411495e-05],
[ 7.08948785e-05, 2.57244480e-05, 4.53889239e-05,
4.21469376e-05, 2.27640364e-05, 1.12849096e-04,
4.01384618e-05, 7.65457600e-05, 3.47015652e-05,
2.04099948e-05, 5.33951974e-05, 5.72051871e-05,
3.92394407e-05, 5.15584350e-05, 5.33004299e-05,
2.81970597e-05, -4.12746174e-07, 5.81836032e-05,
3.12554634e-05, -5.94660474e-06, 3.50482953e-05,
5.96710279e-05, 1.65974972e-04, 5.42453625e-05],
[ 5.55096135e-05, 8.72002781e-06, 3.11615533e-05,
4.35344231e-05, 7.39017715e-05, 1.17234141e-04,
4.82818631e-06, 1.31611693e-04, 3.86545249e-05,
2.89151466e-05, 1.07102393e-04, 6.05239478e-05,
6.47067147e-05, 9.98376706e-05, -1.56063893e-05,
3.94413355e-05, 6.80701129e-05, 1.00294572e-04,
5.94948441e-06, -1.92301274e-05, -1.97971783e-07,
3.14411495e-05, 5.42453625e-05, 1.92061080e-04]]))
sigma_bar = pd.DataFrame(np.array([[ 1.26820873e-09, 9.99357360e-01, 9.96834344e-01,
9.99572167e-01, 9.42281243e-01, 9.97900189e-01,
9.99540337e-01, 9.98601652e-01, 9.99273813e-01,
9.97018616e-01, -9.73801139e-01, 9.99572102e-01,
9.99493833e-01, 9.98516049e-01, 9.99016459e-01,
9.99032878e-01, -9.44718270e-01, 9.99102498e-01,
9.99273586e-01, 9.97964072e-01, 9.98809893e-01,
9.99306302e-01, 9.98670142e-01, 9.89075401e-01],
[ 9.99357360e-01, 8.85756152e-04, 9.96279483e-01,
9.99197947e-01, 9.41407791e-01, 9.97899466e-01,
9.99417552e-01, 9.98836192e-01, 9.99109838e-01,
9.97212584e-01, -9.74070678e-01, 9.99506469e-01,
9.99304970e-01, 9.98532839e-01, 9.99435799e-01,
9.99191149e-01, -9.45834544e-01, 9.99087732e-01,
9.99237110e-01, 9.98261304e-01, 9.98920402e-01,
9.99254574e-01, 9.98828941e-01, 9.88935866e-01],
[ 9.96834344e-01, 9.96279483e-01, 4.28636430e-07,
9.96600181e-01, 9.47870932e-01, 9.95399154e-01,
9.96381458e-01, 9.96592822e-01, 9.96914826e-01,
9.93862938e-01, -9.68327765e-01, 9.96760950e-01,
9.96379221e-01, 9.96672585e-01, 9.96674621e-01,
9.95069722e-01, -9.40676110e-01, 9.96340223e-01,
9.96354345e-01, 9.94003174e-01, 9.96112855e-01,
9.96745892e-01, 9.95997518e-01, 9.89905446e-01],
[ 9.99572167e-01, 9.99197947e-01, 9.96600181e-01,
1.21966822e-09, 9.43053169e-01, 9.97880532e-01,
9.99500369e-01, 9.98410401e-01, 9.99173244e-01,
9.96775797e-01, -9.74367127e-01, 9.99362203e-01,
9.99367414e-01, 9.98297331e-01, 9.98880383e-01,
9.98663753e-01, -9.44650667e-01, 9.98960337e-01,
9.99311633e-01, 9.97876202e-01, 9.98462419e-01,
9.99263944e-01, 9.98810150e-01, 9.89492504e-01],
[ 9.42281243e-01, 9.41407791e-01, 9.47870932e-01,
9.43053169e-01, 5.87433525e-09, 9.47101615e-01,
9.43035090e-01, 9.51819965e-01, 9.46546373e-01,
9.35729856e-01, -8.83958006e-01, 9.43231419e-01,
9.39614529e-01, 9.50658526e-01, 9.44337124e-01,
9.41768971e-01, -8.57348147e-01, 9.46361648e-01,
9.43788045e-01, 9.35138095e-01, 9.43924294e-01,
9.46696871e-01, 9.46932752e-01, 9.74895278e-01],
[ 9.97900189e-01, 9.97899466e-01, 9.95399154e-01,
9.97880532e-01, 9.47101615e-01, 8.52516849e-08,
9.98162029e-01, 9.97693085e-01, 9.97494890e-01,
9.94870487e-01, -9.71281207e-01, 9.97825507e-01,
9.97609029e-01, 9.97078699e-01, 9.97937996e-01,
9.97886353e-01, -9.39807793e-01, 9.97323488e-01,
9.98086971e-01, 9.96890683e-01, 9.97306126e-01,
9.98328683e-01, 9.98019225e-01, 9.90401184e-01],
[ 9.99540337e-01, 9.99417552e-01, 9.96381458e-01,
9.99500369e-01, 9.43035090e-01, 9.98162029e-01,
1.97026096e-09, 9.98697236e-01, 9.99167466e-01,
9.96534267e-01, -9.74797106e-01, 9.99405117e-01,
9.99312797e-01, 9.98338095e-01, 9.99222108e-01,
9.99012700e-01, -9.45433385e-01, 9.99002384e-01,
9.99391574e-01, 9.98227196e-01, 9.98655866e-01,
9.99450287e-01, 9.98871183e-01, 9.89348747e-01],
[ 9.98601652e-01, 9.98836192e-01, 9.96592822e-01,
9.98410401e-01, 9.51819965e-01, 9.97693085e-01,
9.98697236e-01, 3.81563526e-10, 9.98687080e-01,
9.97121814e-01, -9.69119242e-01, 9.98787443e-01,
9.98289054e-01, 9.98866436e-01, 9.98833399e-01,
9.98412389e-01, -9.40737501e-01, 9.99081172e-01,
9.98609560e-01, 9.96732620e-01, 9.98290946e-01,
9.98808945e-01, 9.98467379e-01, 9.92893798e-01],
[ 9.99273813e-01, 9.99109838e-01, 9.96914826e-01,
9.99173244e-01, 9.46546373e-01, 9.97494890e-01,
9.99167466e-01, 9.98687080e-01, 1.63952743e-09,
9.97076275e-01, -9.71833154e-01, 9.99346179e-01,
9.99129781e-01, 9.98683650e-01, 9.99143749e-01,
9.98628992e-01, -9.42716179e-01, 9.99193054e-01,
9.99051762e-01, 9.97592295e-01, 9.98279757e-01,
9.99206541e-01, 9.98709951e-01, 9.91294961e-01],
[ 9.97018616e-01, 9.97212584e-01, 9.93862938e-01,
9.96775797e-01, 9.35729856e-01, 9.94870487e-01,
9.96534267e-01, 9.97121814e-01, 9.97076275e-01,
4.14508406e-09, -9.69602110e-01, 9.97209084e-01,
9.97269710e-01, 9.96130871e-01, 9.96598901e-01,
9.96945711e-01, -9.41737281e-01, 9.97547921e-01,
9.96623630e-01, 9.95604803e-01, 9.96053414e-01,
9.96252977e-01, 9.96293594e-01, 9.87014134e-01],
[-9.73801139e-01, -9.74070678e-01, -9.68327765e-01,
-9.74367127e-01, -8.83958006e-01, -9.71281207e-01,
-9.74797106e-01, -9.69119242e-01, -9.71833154e-01,
-9.69602110e-01, 7.91579754e-08, -9.72627284e-01,
-9.74397877e-01, -9.66816215e-01, -9.74055921e-01,
-9.72152413e-01, 9.49931280e-01, -9.70349967e-01,
-9.74543996e-01, -9.77042256e-01, -9.72510826e-01,
-9.72753685e-01, -9.70977647e-01, -9.49332577e-01],
[ 9.99572102e-01, 9.99506469e-01, 9.96760950e-01,
9.99362203e-01, 9.43231419e-01, 9.97825507e-01,
9.99405117e-01, 9.98787443e-01, 9.99346179e-01,
9.97209084e-01, -9.72627284e-01, 9.49759898e-10,
9.99396168e-01, 9.98616747e-01, 9.99231469e-01,
9.99092707e-01, -9.43690714e-01, 9.99236560e-01,
9.99125265e-01, 9.97720253e-01, 9.98657360e-01,
9.99322811e-01, 9.98924819e-01, 9.89982094e-01],
[ 9.99493833e-01, 9.99304970e-01, 9.96379221e-01,
9.99367414e-01, 9.39614529e-01, 9.97609029e-01,
9.99312797e-01, 9.98289054e-01, 9.99129781e-01,
9.97269710e-01, -9.74397877e-01, 9.99396168e-01,
6.10314496e-08, 9.98282510e-01, 9.98908676e-01,
9.99022688e-01, -9.44584256e-01, 9.99076968e-01,
9.99242844e-01, 9.98014936e-01, 9.98393894e-01,
9.99093343e-01, 9.98535127e-01, 9.88011557e-01],
[ 9.98516049e-01, 9.98532839e-01, 9.96672585e-01,
9.98297331e-01, 9.50658526e-01, 9.97078699e-01,
9.98338095e-01, 9.98866436e-01, 9.98683650e-01,
9.96130871e-01, -9.66816215e-01, 9.98616747e-01,
9.98282510e-01, 1.39566686e-09, 9.98517176e-01,
9.98377797e-01, -9.41181012e-01, 9.99085880e-01,
9.98417332e-01, 9.95895305e-01, 9.98089538e-01,
9.98786233e-01, 9.98273278e-01, 9.92133050e-01],
[ 9.99016459e-01, 9.99435799e-01, 9.96674621e-01,
9.98880383e-01, 9.44337124e-01, 9.97937996e-01,
9.99222108e-01, 9.98833399e-01, 9.99143749e-01,
9.96598901e-01, -9.74055921e-01, 9.99231469e-01,
9.98908676e-01, 9.98517176e-01, 3.91145914e-09,
9.98762801e-01, -9.44712685e-01, 9.98882295e-01,
9.99167722e-01, 9.98365735e-01, 9.98539526e-01,
9.99349674e-01, 9.98682774e-01, 9.90282263e-01],
[ 9.99032878e-01, 9.99191149e-01, 9.95069722e-01,
9.98663753e-01, 9.41768971e-01, 9.97886353e-01,
9.99012700e-01, 9.98412389e-01, 9.98628992e-01,
9.96945711e-01, -9.72152413e-01, 9.99092707e-01,
9.99022688e-01, 9.98377797e-01, 9.98762801e-01,
6.12438062e-09, -9.43278583e-01, 9.98714615e-01,
9.98765094e-01, 9.97518255e-01, 9.98234814e-01,
9.98833239e-01, 9.98556185e-01, 9.88600916e-01],
[-9.44718270e-01, -9.45834544e-01, -9.40676110e-01,
-9.44650667e-01, -8.57348147e-01, -9.39807793e-01,
-9.45433385e-01, -9.40737501e-01, -9.42716179e-01,
-9.41737281e-01, 9.49931280e-01, -9.43690714e-01,
-9.44584256e-01, -9.41181012e-01, -9.44712685e-01,
-9.43278583e-01, 1.85097555e-08, -9.42274581e-01,
-9.43930225e-01, -9.44416759e-01, -9.44123084e-01,
-9.43611879e-01, -9.43058469e-01, -9.20277725e-01],
[ 9.99102498e-01, 9.99087732e-01, 9.96340223e-01,
9.98960337e-01, 9.46361648e-01, 9.97323488e-01,
9.99002384e-01, 9.99081172e-01, 9.99193054e-01,
9.97547921e-01, -9.70349967e-01, 9.99236560e-01,
9.99076968e-01, 9.99085880e-01, 9.98882295e-01,
9.98714615e-01, -9.42274581e-01, 4.63172335e-10,
9.99055447e-01, 9.96678367e-01, 9.97947641e-01,
9.99023756e-01, 9.98655609e-01, 9.90680163e-01],
[ 9.99273586e-01, 9.99237110e-01, 9.96354345e-01,
9.99311633e-01, 9.43788045e-01, 9.98086971e-01,
9.99391574e-01, 9.98609560e-01, 9.99051762e-01,
9.96623630e-01, -9.74543996e-01, 9.99125265e-01,
9.99242844e-01, 9.98417332e-01, 9.99167722e-01,
9.98765094e-01, -9.43930225e-01, 9.99055447e-01,
1.49968419e-08, 9.98019738e-01, 9.98407433e-01,
9.99222857e-01, 9.98652394e-01, 9.89226980e-01],
[ 9.97964072e-01, 9.98261304e-01, 9.94003174e-01,
9.97876202e-01, 9.35138095e-01, 9.96890683e-01,
9.98227196e-01, 9.96732620e-01, 9.97592295e-01,
9.95604803e-01, -9.77042256e-01, 9.97720253e-01,
9.98014936e-01, 9.95895305e-01, 9.98365735e-01,
9.97518255e-01, -9.44416759e-01, 9.96678367e-01,
9.98019738e-01, 7.47650147e-10, 9.97617693e-01,
9.97870740e-01, 9.97367170e-01, 9.86577350e-01],
[ 9.98809893e-01, 9.98920402e-01, 9.96112855e-01,
9.98462419e-01, 9.43924294e-01, 9.97306126e-01,
9.98655866e-01, 9.98290946e-01, 9.98279757e-01,
9.96053414e-01, -9.72510826e-01, 9.98657360e-01,
9.98393894e-01, 9.98089538e-01, 9.98539526e-01,
9.98234814e-01, -9.44123084e-01, 9.97947641e-01,
9.98407433e-01, 9.97617693e-01, 9.94482237e-10,
9.98596112e-01, 9.98000736e-01, 9.89489744e-01],
[ 9.99306302e-01, 9.99254574e-01, 9.96745892e-01,
9.99263944e-01, 9.46696871e-01, 9.98328683e-01,
9.99450287e-01, 9.98808945e-01, 9.99206541e-01,
9.96252977e-01, -9.72753685e-01, 9.99322811e-01,
9.99093343e-01, 9.98786233e-01, 9.99349674e-01,
9.98833239e-01, -9.43611879e-01, 9.99023756e-01,
9.99222857e-01, 9.97870740e-01, 9.98596112e-01,
7.44695142e-09, 9.99016834e-01, 9.90734007e-01],
[ 9.98670142e-01, 9.98828941e-01, 9.95997518e-01,
9.98810150e-01, 9.46932752e-01, 9.98019225e-01,
9.98871183e-01, 9.98467379e-01, 9.98709951e-01,
9.96293594e-01, -9.70977647e-01, 9.98924819e-01,
9.98535127e-01, 9.98273278e-01, 9.98682774e-01,
9.98556185e-01, -9.43058469e-01, 9.98655609e-01,
9.98652394e-01, 9.97367170e-01, 9.98000736e-01,
9.99016834e-01, 1.79765233e-09, 9.91199840e-01],
[ 9.89075401e-01, 9.88935866e-01, 9.89905446e-01,
9.89492504e-01, 9.74895278e-01, 9.90401184e-01,
9.89348747e-01, 9.92893798e-01, 9.91294961e-01,
9.87014134e-01, -9.49332577e-01, 9.89982094e-01,
9.88011557e-01, 9.92133050e-01, 9.90282263e-01,
9.88600916e-01, -9.20277725e-01, 9.90680163e-01,
9.89226980e-01, 9.86577350e-01, 9.89489744e-01,
9.90734007e-01, 9.91199840e-01, 9.41720628e-08]]))
Algorithm Section
N = 24
model = GEKKO(remote=False)
wesg = model.Array(model.Var , N , value = 1/N , lb=0 , ub=1)
model.Equation = (sum(wesg) == 1)
def obj(rf = 0.02):
esg_portfo = 0
for i in range(N):
esg_portfo += model.Intermediate(wesg[i] * ((esg_scores_df["Score 2019"][i] + esg_scores_df["Score 2020"][i])/2))
return_portfo = 0
for i in range(N):
return_portfo += model.Intermediate(wesg[i]*((predicted_return_dataframe.iloc[0 , i] + r.iloc[0 ,i])/2))
sigma_portfo = 0
for i in range(N):
for j in range(N):
sigma_portfo += model.Intermediate(wesg[i] * ((sigma.iloc[i,j] + sigma_bar.iloc[i,j])/2) * wesg[j])
return -esg_portfo * ((return_portfo - rf)/sigma_portfo)
model.Minimize(obj())
model.solve(disp=False)
wesg = np.array( [item[0] for item in wesg], dtype=float)
print(wesg)
Result
The code results in:
[0.00000000e+00 1.31172076e-07 0.00000000e+00 0.00000000e+00
1.88847768e-01 1.00000000e+00 0.00000000e+00 0.00000000e+00
0.00000000e+00 1.00000000e+00 0.00000000e+00 0.00000000e+00
1.00000000e+00 0.00000000e+00 0.00000000e+00 1.00000000e+00
0.00000000e+00 0.00000000e+00 1.00000000e+00 1.00000000e+00
9.07168980e-01 0.00000000e+00 0.00000000e+00 0.00000000e+00]
But, there are many ones in the result!! I have a constraint of sum(wesg) == 1! But the result indicates that somehow, Gekko doesn't consider the constraint. If you try print(wesg.sum()), then it will result:
7.096016879722077
Which I expected to be exact 1!
How can I fix this issue, and where is the problem? Any help would be appreciated.

The error is with model.Equation = (sum(wesg) == 1) that redefines the model.Equation function as False. Use this instead:
model.Equation(sum(wesg) == 1)
The new result enforces the constraint to add to one with sum(wesg)=1.000000000001.
[0. 0. 0. 0. 0.1425506 0.24959448
0. 0. 0. 0. 0. 0.0281217
0.32053308 0. 0. 0. 0. 0.
0.05860444 0.18554967 0.01504603 0. 0. 0. ]
Here are a couple additional suggestions and options:
For larger portfolios use model.sum() instead for better performance.
To select the top 3 stocks as integers (purchase or don't purchase) then use integer=True with the APOPT solver.
wesg = model.Array(model.Var, N, value = 1/N, lb=0, ub=1, integer=True)
model.Equation(model.sum(wesg) == 3)
model.options.SOLVER=1
Here is the complete code.
from gekko import GEKKO
import numpy as np
import pandas as pd
esg_scores_df = pd.DataFrame({'Score 2019': {'MSFT': 7,'PEP': 7,'TSLA': 4,'AMZN': 6,'LKQ': 3,'ABMD': 3,'MSI': 10,'PH': 8,'NKE': 6,'TM': 7,'EOG': 7,'GOOGL': 5,'NFLX': 4,'GS': 6,'EQIX': 9,'EA': 7,'AAP': 6,'TEL': 9,'DG': 6,'EXR': 5,'MDLZ': 6,'FIS': 8,'CRL': 8,'RCL': 9},
'Score 2020': {'MSFT': 6,'PEP': 11,'TSLA': 4,'AMZN': 6,'LKQ': 4,'ABMD': 4,'MSI': 8,'PH': 8,'NKE': 6,'TM': 7,'EOG': 7,'GOOGL': 5,'NFLX': 3,'GS': 6,'EQIX': 9,'EA': 6,'AAP': 7,'TEL': 9,'DG': 6,'EXR': 4,'MDLZ': 6,'FIS': 8,'CRL': 8,'RCL': 7}})
predicted_return_dataframe = pd.DataFrame({'MSFT': 1.0982472257593677e-15,'PEP': 4.567069595849647e-09,'TSLA': 7.439258841202596e-10,'AMZN': 3.176883309676764e-07,'LKQ': 4.825709334830293e-05,'ABMD': 2.0608058642685837e-05,'MSI': -3.1789250961959136e-12,'PH': -2.257237871892785e-07,'NKE': -4.737530217609426e-07,'TM': 9.951932427172896e-07,'EOG': 1.184074639824261e-08,'GOOGL': -1.7027206923083418e-10,'NFLX': 2.948344729885545e-05,'GS': 1.458862302453713e-07,'EQIX': 4.620207503091301e-07,'EA': -1.297137640100873e-06,'AAP': -3.493153382990658e-07,'TEL': 8.202463568291799e-08,'DG': 2.743802312024829e-05,'EXR': 1.971994818465128e-11,'MDLZ': 1.578772827915881e-08,'FIS': 1.3314663987166631e-08,'CRL': 4.112083587242872e-05,'RCL': -1.470829206425942e-09} , index = [0])
r = pd.DataFrame({'MSFT': -0.029838842874655123,'PEP': -0.012265555866403165,'TSLA': -0.053599428210690636,'AMZN': -0.02614876342440773,'LKQ': -0.026658034482507052,'ABMD': -0.017274321994365294,'MSI': -0.018624198080929168,'PH': -0.02871952268633362,'NKE': -0.028820168231794074,'TM': -0.011208781085200492,'EOG': -0.04606883798249617,'GOOGL': -0.027983366801701926,'NFLX': -0.0070609860042608165,'GS': -0.02429261616692462,'EQIX': -0.015181566244607758,'EA': -0.012009295653284258,'AAP': -0.011979871239295382,'TEL': -0.021782414925879602,'DG': -0.009820085124020328,'EXR': -0.012233190959318829,'MDLZ': -0.015917631061436933,'FIS': -0.023877815976094417,'CRL': -0.020169902356185498,'RCL': -0.06831607178882856} , index = [0])
sigma = pd.DataFrame(np.array([[ 1.02188994e-04, 3.16399586e-05, 8.54243667e-05,
7.14148287e-05, 3.55313650e-05, 5.44796564e-05,
3.39128448e-05, 6.65003271e-05, 5.42932808e-05,
2.29333906e-05, 4.26407605e-05, 7.92791756e-05,
9.28910574e-05, 6.36770764e-05, 3.75024646e-05,
5.13720672e-05, 1.33646720e-05, 6.21865710e-05,
4.72888751e-05, 2.35546749e-06, 2.49700413e-05,
6.74503236e-05, 7.08948785e-05, 5.55096135e-05],
[ 3.16399586e-05, 6.47824823e-05, -2.57461009e-05,
1.81931187e-05, 2.11795049e-05, 1.01708683e-05,
3.33739243e-05, 1.23893822e-05, 2.74825418e-05,
1.35954411e-05, 2.37108310e-06, 2.34208728e-05,
1.96972793e-05, 1.09335123e-05, 4.26042775e-05,
2.76849308e-05, 6.01821651e-06, 8.04387109e-06,
3.10892437e-05, 2.79970364e-05, 3.70572613e-05,
3.08215527e-05, 2.57244480e-05, 8.72002781e-06],
[ 8.54243667e-05, -2.57461009e-05, 8.84138653e-04,
6.16478906e-05, -1.78063009e-05, 3.62712388e-05,
-6.15200573e-05, 8.63597848e-05, 2.12571109e-05,
3.21749681e-05, 6.54693912e-05, 7.95298035e-05,
1.53917874e-04, 8.06306277e-05, -5.27893372e-06,
2.60899474e-05, 4.29017379e-05, 6.13689353e-05,
2.30097086e-06, -4.45963115e-05, -2.83669072e-06,
2.95395217e-05, 4.53889239e-05, 3.11615533e-05],
[ 7.14148287e-05, 1.81931187e-05, 6.16478906e-05,
1.19357840e-04, 5.48763301e-05, 5.55161292e-05,
2.77288469e-05, 6.25231790e-05, 5.19494452e-05,
1.93193913e-05, 4.28946368e-05, 7.00210371e-05,
9.92756795e-05, 6.54630027e-05, 1.73935405e-05,
4.46819063e-05, 2.94199398e-05, 5.76749375e-05,
3.75634244e-05, -5.53847280e-06, 9.72193169e-06,
4.42756733e-05, 4.21469376e-05, 4.35344231e-05],
[ 3.55313650e-05, 2.11795049e-05, -1.78063009e-05,
5.48763301e-05, 3.49607174e-04, 1.58156275e-04,
-1.64565378e-05, 5.95602984e-05, 3.36138721e-05,
2.25095973e-05, 4.90144491e-05, 5.60207136e-05,
3.73119885e-05, 8.65010497e-05, -1.68774498e-05,
1.06928038e-05, 3.53732120e-05, 8.46071106e-05,
3.89200328e-05, -1.10740797e-05, 4.20238479e-06,
2.07371361e-05, 2.27640364e-05, 7.39017715e-05],
[ 5.44796564e-05, 1.01708683e-05, 3.62712388e-05,
5.55161292e-05, 1.58156275e-04, 1.12266763e-03,
2.49786917e-05, 5.60053839e-05, -2.57686022e-05,
3.00855419e-05, 2.09079416e-04, 1.21797841e-04,
3.80689080e-06, 1.17305259e-04, -4.95192897e-05,
5.56059321e-05, 2.87765760e-06, 1.13640096e-04,
-2.73690072e-05, 2.17458024e-05, -1.24009254e-05,
6.11287657e-05, 1.12849096e-04, 1.17234141e-04],
[ 3.39128448e-05, 3.33739243e-05, -6.15200573e-05,
2.77288469e-05, -1.64565378e-05, 2.49786917e-05,
1.84271573e-04, -2.99455473e-06, 3.56885701e-05,
1.53451236e-05, 3.20222217e-06, 3.00619247e-05,
5.17902613e-05, 1.69851832e-05, 7.75007583e-05,
7.49839347e-06, -3.78485373e-05, -1.28942631e-05,
4.36657573e-05, 3.97590833e-05, 4.04704969e-05,
4.05201790e-05, 4.01384618e-05, 4.82818631e-06],
[ 6.65003271e-05, 1.23893822e-05, 8.63597848e-05,
6.25231790e-05, 5.95602984e-05, 5.60053839e-05,
-2.99455473e-06, 2.86568704e-04, 6.55278686e-05,
4.08938883e-05, 1.18822482e-04, 9.32577766e-05,
7.92498191e-05, 1.38967662e-04, -1.64128182e-05,
4.16393158e-05, 1.06300676e-04, 1.31672888e-04,
6.64839109e-05, -5.14108669e-05, -1.03044572e-05,
1.59107041e-05, 7.65457600e-05, 1.31611693e-04],
[ 5.42932808e-05, 2.74825418e-05, 2.12571109e-05,
5.19494452e-05, 3.36138721e-05, -2.57686022e-05,
3.56885701e-05, 6.55278686e-05, 1.36374334e-04,
1.36781102e-05, 2.38544176e-05, 5.90104054e-05,
6.20754315e-05, 5.84082445e-05, 3.96422393e-05,
2.96064418e-05, 5.54456912e-05, 5.69396332e-05,
7.28867485e-05, 7.92870807e-06, 3.79528145e-05,
3.38036564e-05, 3.47015652e-05, 3.86545249e-05],
[ 2.29333906e-05, 1.35954411e-05, 3.21749681e-05,
1.93193913e-05, 2.25095973e-05, 3.00855419e-05,
1.53451236e-05, 4.08938883e-05, 1.36781102e-05,
5.30619573e-05, 4.42693706e-05, 2.92937393e-05,
1.15184525e-05, 3.04141154e-05, 1.11981894e-06,
1.29388215e-05, 1.41913521e-05, 2.37704063e-05,
1.27983719e-05, -3.53963791e-06, 5.41505031e-06,
1.21874870e-05, 2.04099948e-05, 2.89151466e-05],
[ 4.26407605e-05, 2.37108310e-06, 6.54693912e-05,
4.28946368e-05, 4.90144491e-05, 2.09079416e-04,
3.20222217e-06, 1.18822482e-04, 2.38544176e-05,
4.42693706e-05, 5.01946804e-04, 7.79572195e-05,
7.61563242e-05, 1.31597522e-04, -2.43789107e-05,
6.77962026e-05, 5.39745318e-05, 9.05223133e-05,
-1.88433584e-06, -1.63784490e-05, -1.04001201e-05,
1.79450881e-05, 5.33951974e-05, 1.07102393e-04],
[ 7.92791756e-05, 2.34208728e-05, 7.95298035e-05,
7.00210371e-05, 5.60207136e-05, 1.21797841e-04,
3.00619247e-05, 9.32577766e-05, 5.90104054e-05,
2.92937393e-05, 7.79572195e-05, 1.10688236e-04,
8.13294795e-05, 8.72981821e-05, 2.13737337e-05,
4.33439061e-05, 3.06090710e-05, 7.60853329e-05,
4.44363057e-05, 3.33458671e-06, 1.60908360e-05,
4.09067616e-05, 5.72051871e-05, 6.05239478e-05],
[ 9.28910574e-05, 1.96972793e-05, 1.53917874e-04,
9.92756795e-05, 3.73119885e-05, 3.80689080e-06,
5.17902613e-05, 7.92498191e-05, 6.20754315e-05,
1.15184525e-05, 7.61563242e-05, 8.13294795e-05,
3.77348891e-04, 8.99709029e-05, 2.58732673e-05,
8.24015922e-05, 2.77333081e-05, 4.74146019e-05,
3.76467363e-05, -5.60031564e-06, 2.25653918e-05,
6.22679966e-05, 3.92394407e-05, 6.47067147e-05],
[ 6.36770764e-05, 1.09335123e-05, 8.06306277e-05,
6.54630027e-05, 8.65010497e-05, 1.17305259e-04,
1.69851832e-05, 1.38967662e-04, 5.84082445e-05,
3.04141154e-05, 1.31597522e-04, 8.72981821e-05,
8.99709029e-05, 1.54802097e-04, -1.57072336e-05,
3.58129141e-05, 5.52667399e-05, 1.02065825e-04,
4.96942712e-05, -2.44193897e-05, -5.66188177e-06,
2.35884764e-05, 5.15584350e-05, 9.98376706e-05],
[ 3.75024646e-05, 4.26042775e-05, -5.27893372e-06,
1.73935405e-05, -1.68774498e-05, -4.95192897e-05,
7.75007583e-05, -1.64128182e-05, 3.96422393e-05,
1.11981894e-06, -2.43789107e-05, 2.13737337e-05,
2.58732673e-05, -1.57072336e-05, 1.54605175e-04,
1.07085192e-05, -1.14274871e-05, -1.26204468e-05,
2.50810750e-05, 5.60340845e-05, 4.60520336e-05,
4.29696767e-05, 5.33004299e-05, -1.56063893e-05],
[ 5.13720672e-05, 2.76849308e-05, 2.60899474e-05,
4.46819063e-05, 1.06928038e-05, 5.56059321e-05,
7.49839347e-06, 4.16393158e-05, 2.96064418e-05,
1.29388215e-05, 6.77962026e-05, 4.33439061e-05,
8.24015922e-05, 3.58129141e-05, 1.07085192e-05,
1.32544808e-04, 3.15561503e-05, 3.62681876e-05,
1.76503394e-05, 1.90982407e-07, 1.02992114e-05,
2.81341342e-05, 2.81970597e-05, 3.94413355e-05],
[ 1.33646720e-05, 6.01821651e-06, 4.29017379e-05,
2.94199398e-05, 3.53732120e-05, 2.87765760e-06,
-3.78485373e-05, 1.06300676e-04, 5.54456912e-05,
1.41913521e-05, 5.39745318e-05, 3.06090710e-05,
2.77333081e-05, 5.52667399e-05, -1.14274871e-05,
3.15561503e-05, 2.71861487e-04, 6.04272650e-05,
3.55256428e-05, -2.37123843e-05, -1.99844011e-05,
-2.02943319e-07, -4.12746174e-07, 6.80701129e-05],
[ 6.21865710e-05, 8.04387109e-06, 6.13689353e-05,
5.76749375e-05, 8.46071106e-05, 1.13640096e-04,
-1.28942631e-05, 1.31672888e-04, 5.69396332e-05,
2.37704063e-05, 9.05223133e-05, 7.60853329e-05,
4.74146019e-05, 1.02065825e-04, -1.26204468e-05,
3.62681876e-05, 6.04272650e-05, 1.67171047e-04,
5.06392219e-05, -1.02673299e-05, 1.64715372e-05,
2.50897861e-05, 5.81836032e-05, 1.00294572e-04],
[ 4.72888751e-05, 3.10892437e-05, 2.30097086e-06,
3.75634244e-05, 3.89200328e-05, -2.73690072e-05,
4.36657573e-05, 6.64839109e-05, 7.28867485e-05,
1.27983719e-05, -1.88433584e-06, 4.44363057e-05,
3.76467363e-05, 4.96942712e-05, 2.50810750e-05,
1.76503394e-05, 3.55256428e-05, 5.06392219e-05,
1.99771178e-04, 4.70710752e-06, 3.76160380e-05,
2.30014580e-05, 3.12554634e-05, 5.94948441e-06],
[ 2.35546749e-06, 2.79970364e-05, -4.45963115e-05,
-5.53847280e-06, -1.10740797e-05, 2.17458024e-05,
3.97590833e-05, -5.14108669e-05, 7.92870807e-06,
-3.53963791e-06, -1.63784490e-05, 3.33458671e-06,
-5.60031564e-06, -2.44193897e-05, 5.60340845e-05,
1.90982407e-07, -2.37123843e-05, -1.02673299e-05,
4.70710752e-06, 8.86508330e-05, 4.02039134e-05,
2.39025489e-05, -5.94660474e-06, -1.92301274e-05],
[ 2.49700413e-05, 3.70572613e-05, -2.83669072e-06,
9.72193169e-06, 4.20238479e-06, -1.24009254e-05,
4.04704969e-05, -1.03044572e-05, 3.79528145e-05,
5.41505031e-06, -1.04001201e-05, 1.60908360e-05,
2.25653918e-05, -5.66188177e-06, 4.60520336e-05,
1.02992114e-05, -1.99844011e-05, 1.64715372e-05,
3.76160380e-05, 4.02039134e-05, 8.23927778e-05,
3.21948208e-05, 3.50482953e-05, -1.97971783e-07],
[ 6.74503236e-05, 3.08215527e-05, 2.95395217e-05,
4.42756733e-05, 2.07371361e-05, 6.11287657e-05,
4.05201790e-05, 1.59107041e-05, 3.38036564e-05,
1.21874870e-05, 1.79450881e-05, 4.09067616e-05,
6.22679966e-05, 2.35884764e-05, 4.29696767e-05,
2.81341342e-05, -2.02943319e-07, 2.50897861e-05,
2.30014580e-05, 2.39025489e-05, 3.21948208e-05,
1.20680211e-04, 5.96710279e-05, 3.14411495e-05],
[ 7.08948785e-05, 2.57244480e-05, 4.53889239e-05,
4.21469376e-05, 2.27640364e-05, 1.12849096e-04,
4.01384618e-05, 7.65457600e-05, 3.47015652e-05,
2.04099948e-05, 5.33951974e-05, 5.72051871e-05,
3.92394407e-05, 5.15584350e-05, 5.33004299e-05,
2.81970597e-05, -4.12746174e-07, 5.81836032e-05,
3.12554634e-05, -5.94660474e-06, 3.50482953e-05,
5.96710279e-05, 1.65974972e-04, 5.42453625e-05],
[ 5.55096135e-05, 8.72002781e-06, 3.11615533e-05,
4.35344231e-05, 7.39017715e-05, 1.17234141e-04,
4.82818631e-06, 1.31611693e-04, 3.86545249e-05,
2.89151466e-05, 1.07102393e-04, 6.05239478e-05,
6.47067147e-05, 9.98376706e-05, -1.56063893e-05,
3.94413355e-05, 6.80701129e-05, 1.00294572e-04,
5.94948441e-06, -1.92301274e-05, -1.97971783e-07,
3.14411495e-05, 5.42453625e-05, 1.92061080e-04]]))
sigma_bar = pd.DataFrame(np.array([[ 1.26820873e-09, 9.99357360e-01, 9.96834344e-01,
9.99572167e-01, 9.42281243e-01, 9.97900189e-01,
9.99540337e-01, 9.98601652e-01, 9.99273813e-01,
9.97018616e-01, -9.73801139e-01, 9.99572102e-01,
9.99493833e-01, 9.98516049e-01, 9.99016459e-01,
9.99032878e-01, -9.44718270e-01, 9.99102498e-01,
9.99273586e-01, 9.97964072e-01, 9.98809893e-01,
9.99306302e-01, 9.98670142e-01, 9.89075401e-01],
[ 9.99357360e-01, 8.85756152e-04, 9.96279483e-01,
9.99197947e-01, 9.41407791e-01, 9.97899466e-01,
9.99417552e-01, 9.98836192e-01, 9.99109838e-01,
9.97212584e-01, -9.74070678e-01, 9.99506469e-01,
9.99304970e-01, 9.98532839e-01, 9.99435799e-01,
9.99191149e-01, -9.45834544e-01, 9.99087732e-01,
9.99237110e-01, 9.98261304e-01, 9.98920402e-01,
9.99254574e-01, 9.98828941e-01, 9.88935866e-01],
[ 9.96834344e-01, 9.96279483e-01, 4.28636430e-07,
9.96600181e-01, 9.47870932e-01, 9.95399154e-01,
9.96381458e-01, 9.96592822e-01, 9.96914826e-01,
9.93862938e-01, -9.68327765e-01, 9.96760950e-01,
9.96379221e-01, 9.96672585e-01, 9.96674621e-01,
9.95069722e-01, -9.40676110e-01, 9.96340223e-01,
9.96354345e-01, 9.94003174e-01, 9.96112855e-01,
9.96745892e-01, 9.95997518e-01, 9.89905446e-01],
[ 9.99572167e-01, 9.99197947e-01, 9.96600181e-01,
1.21966822e-09, 9.43053169e-01, 9.97880532e-01,
9.99500369e-01, 9.98410401e-01, 9.99173244e-01,
9.96775797e-01, -9.74367127e-01, 9.99362203e-01,
9.99367414e-01, 9.98297331e-01, 9.98880383e-01,
9.98663753e-01, -9.44650667e-01, 9.98960337e-01,
9.99311633e-01, 9.97876202e-01, 9.98462419e-01,
9.99263944e-01, 9.98810150e-01, 9.89492504e-01],
[ 9.42281243e-01, 9.41407791e-01, 9.47870932e-01,
9.43053169e-01, 5.87433525e-09, 9.47101615e-01,
9.43035090e-01, 9.51819965e-01, 9.46546373e-01,
9.35729856e-01, -8.83958006e-01, 9.43231419e-01,
9.39614529e-01, 9.50658526e-01, 9.44337124e-01,
9.41768971e-01, -8.57348147e-01, 9.46361648e-01,
9.43788045e-01, 9.35138095e-01, 9.43924294e-01,
9.46696871e-01, 9.46932752e-01, 9.74895278e-01],
[ 9.97900189e-01, 9.97899466e-01, 9.95399154e-01,
9.97880532e-01, 9.47101615e-01, 8.52516849e-08,
9.98162029e-01, 9.97693085e-01, 9.97494890e-01,
9.94870487e-01, -9.71281207e-01, 9.97825507e-01,
9.97609029e-01, 9.97078699e-01, 9.97937996e-01,
9.97886353e-01, -9.39807793e-01, 9.97323488e-01,
9.98086971e-01, 9.96890683e-01, 9.97306126e-01,
9.98328683e-01, 9.98019225e-01, 9.90401184e-01],
[ 9.99540337e-01, 9.99417552e-01, 9.96381458e-01,
9.99500369e-01, 9.43035090e-01, 9.98162029e-01,
1.97026096e-09, 9.98697236e-01, 9.99167466e-01,
9.96534267e-01, -9.74797106e-01, 9.99405117e-01,
9.99312797e-01, 9.98338095e-01, 9.99222108e-01,
9.99012700e-01, -9.45433385e-01, 9.99002384e-01,
9.99391574e-01, 9.98227196e-01, 9.98655866e-01,
9.99450287e-01, 9.98871183e-01, 9.89348747e-01],
[ 9.98601652e-01, 9.98836192e-01, 9.96592822e-01,
9.98410401e-01, 9.51819965e-01, 9.97693085e-01,
9.98697236e-01, 3.81563526e-10, 9.98687080e-01,
9.97121814e-01, -9.69119242e-01, 9.98787443e-01,
9.98289054e-01, 9.98866436e-01, 9.98833399e-01,
9.98412389e-01, -9.40737501e-01, 9.99081172e-01,
9.98609560e-01, 9.96732620e-01, 9.98290946e-01,
9.98808945e-01, 9.98467379e-01, 9.92893798e-01],
[ 9.99273813e-01, 9.99109838e-01, 9.96914826e-01,
9.99173244e-01, 9.46546373e-01, 9.97494890e-01,
9.99167466e-01, 9.98687080e-01, 1.63952743e-09,
9.97076275e-01, -9.71833154e-01, 9.99346179e-01,
9.99129781e-01, 9.98683650e-01, 9.99143749e-01,
9.98628992e-01, -9.42716179e-01, 9.99193054e-01,
9.99051762e-01, 9.97592295e-01, 9.98279757e-01,
9.99206541e-01, 9.98709951e-01, 9.91294961e-01],
[ 9.97018616e-01, 9.97212584e-01, 9.93862938e-01,
9.96775797e-01, 9.35729856e-01, 9.94870487e-01,
9.96534267e-01, 9.97121814e-01, 9.97076275e-01,
4.14508406e-09, -9.69602110e-01, 9.97209084e-01,
9.97269710e-01, 9.96130871e-01, 9.96598901e-01,
9.96945711e-01, -9.41737281e-01, 9.97547921e-01,
9.96623630e-01, 9.95604803e-01, 9.96053414e-01,
9.96252977e-01, 9.96293594e-01, 9.87014134e-01],
[-9.73801139e-01, -9.74070678e-01, -9.68327765e-01,
-9.74367127e-01, -8.83958006e-01, -9.71281207e-01,
-9.74797106e-01, -9.69119242e-01, -9.71833154e-01,
-9.69602110e-01, 7.91579754e-08, -9.72627284e-01,
-9.74397877e-01, -9.66816215e-01, -9.74055921e-01,
-9.72152413e-01, 9.49931280e-01, -9.70349967e-01,
-9.74543996e-01, -9.77042256e-01, -9.72510826e-01,
-9.72753685e-01, -9.70977647e-01, -9.49332577e-01],
[ 9.99572102e-01, 9.99506469e-01, 9.96760950e-01,
9.99362203e-01, 9.43231419e-01, 9.97825507e-01,
9.99405117e-01, 9.98787443e-01, 9.99346179e-01,
9.97209084e-01, -9.72627284e-01, 9.49759898e-10,
9.99396168e-01, 9.98616747e-01, 9.99231469e-01,
9.99092707e-01, -9.43690714e-01, 9.99236560e-01,
9.99125265e-01, 9.97720253e-01, 9.98657360e-01,
9.99322811e-01, 9.98924819e-01, 9.89982094e-01],
[ 9.99493833e-01, 9.99304970e-01, 9.96379221e-01,
9.99367414e-01, 9.39614529e-01, 9.97609029e-01,
9.99312797e-01, 9.98289054e-01, 9.99129781e-01,
9.97269710e-01, -9.74397877e-01, 9.99396168e-01,
6.10314496e-08, 9.98282510e-01, 9.98908676e-01,
9.99022688e-01, -9.44584256e-01, 9.99076968e-01,
9.99242844e-01, 9.98014936e-01, 9.98393894e-01,
9.99093343e-01, 9.98535127e-01, 9.88011557e-01],
[ 9.98516049e-01, 9.98532839e-01, 9.96672585e-01,
9.98297331e-01, 9.50658526e-01, 9.97078699e-01,
9.98338095e-01, 9.98866436e-01, 9.98683650e-01,
9.96130871e-01, -9.66816215e-01, 9.98616747e-01,
9.98282510e-01, 1.39566686e-09, 9.98517176e-01,
9.98377797e-01, -9.41181012e-01, 9.99085880e-01,
9.98417332e-01, 9.95895305e-01, 9.98089538e-01,
9.98786233e-01, 9.98273278e-01, 9.92133050e-01],
[ 9.99016459e-01, 9.99435799e-01, 9.96674621e-01,
9.98880383e-01, 9.44337124e-01, 9.97937996e-01,
9.99222108e-01, 9.98833399e-01, 9.99143749e-01,
9.96598901e-01, -9.74055921e-01, 9.99231469e-01,
9.98908676e-01, 9.98517176e-01, 3.91145914e-09,
9.98762801e-01, -9.44712685e-01, 9.98882295e-01,
9.99167722e-01, 9.98365735e-01, 9.98539526e-01,
9.99349674e-01, 9.98682774e-01, 9.90282263e-01],
[ 9.99032878e-01, 9.99191149e-01, 9.95069722e-01,
9.98663753e-01, 9.41768971e-01, 9.97886353e-01,
9.99012700e-01, 9.98412389e-01, 9.98628992e-01,
9.96945711e-01, -9.72152413e-01, 9.99092707e-01,
9.99022688e-01, 9.98377797e-01, 9.98762801e-01,
6.12438062e-09, -9.43278583e-01, 9.98714615e-01,
9.98765094e-01, 9.97518255e-01, 9.98234814e-01,
9.98833239e-01, 9.98556185e-01, 9.88600916e-01],
[-9.44718270e-01, -9.45834544e-01, -9.40676110e-01,
-9.44650667e-01, -8.57348147e-01, -9.39807793e-01,
-9.45433385e-01, -9.40737501e-01, -9.42716179e-01,
-9.41737281e-01, 9.49931280e-01, -9.43690714e-01,
-9.44584256e-01, -9.41181012e-01, -9.44712685e-01,
-9.43278583e-01, 1.85097555e-08, -9.42274581e-01,
-9.43930225e-01, -9.44416759e-01, -9.44123084e-01,
-9.43611879e-01, -9.43058469e-01, -9.20277725e-01],
[ 9.99102498e-01, 9.99087732e-01, 9.96340223e-01,
9.98960337e-01, 9.46361648e-01, 9.97323488e-01,
9.99002384e-01, 9.99081172e-01, 9.99193054e-01,
9.97547921e-01, -9.70349967e-01, 9.99236560e-01,
9.99076968e-01, 9.99085880e-01, 9.98882295e-01,
9.98714615e-01, -9.42274581e-01, 4.63172335e-10,
9.99055447e-01, 9.96678367e-01, 9.97947641e-01,
9.99023756e-01, 9.98655609e-01, 9.90680163e-01],
[ 9.99273586e-01, 9.99237110e-01, 9.96354345e-01,
9.99311633e-01, 9.43788045e-01, 9.98086971e-01,
9.99391574e-01, 9.98609560e-01, 9.99051762e-01,
9.96623630e-01, -9.74543996e-01, 9.99125265e-01,
9.99242844e-01, 9.98417332e-01, 9.99167722e-01,
9.98765094e-01, -9.43930225e-01, 9.99055447e-01,
1.49968419e-08, 9.98019738e-01, 9.98407433e-01,
9.99222857e-01, 9.98652394e-01, 9.89226980e-01],
[ 9.97964072e-01, 9.98261304e-01, 9.94003174e-01,
9.97876202e-01, 9.35138095e-01, 9.96890683e-01,
9.98227196e-01, 9.96732620e-01, 9.97592295e-01,
9.95604803e-01, -9.77042256e-01, 9.97720253e-01,
9.98014936e-01, 9.95895305e-01, 9.98365735e-01,
9.97518255e-01, -9.44416759e-01, 9.96678367e-01,
9.98019738e-01, 7.47650147e-10, 9.97617693e-01,
9.97870740e-01, 9.97367170e-01, 9.86577350e-01],
[ 9.98809893e-01, 9.98920402e-01, 9.96112855e-01,
9.98462419e-01, 9.43924294e-01, 9.97306126e-01,
9.98655866e-01, 9.98290946e-01, 9.98279757e-01,
9.96053414e-01, -9.72510826e-01, 9.98657360e-01,
9.98393894e-01, 9.98089538e-01, 9.98539526e-01,
9.98234814e-01, -9.44123084e-01, 9.97947641e-01,
9.98407433e-01, 9.97617693e-01, 9.94482237e-10,
9.98596112e-01, 9.98000736e-01, 9.89489744e-01],
[ 9.99306302e-01, 9.99254574e-01, 9.96745892e-01,
9.99263944e-01, 9.46696871e-01, 9.98328683e-01,
9.99450287e-01, 9.98808945e-01, 9.99206541e-01,
9.96252977e-01, -9.72753685e-01, 9.99322811e-01,
9.99093343e-01, 9.98786233e-01, 9.99349674e-01,
9.98833239e-01, -9.43611879e-01, 9.99023756e-01,
9.99222857e-01, 9.97870740e-01, 9.98596112e-01,
7.44695142e-09, 9.99016834e-01, 9.90734007e-01],
[ 9.98670142e-01, 9.98828941e-01, 9.95997518e-01,
9.98810150e-01, 9.46932752e-01, 9.98019225e-01,
9.98871183e-01, 9.98467379e-01, 9.98709951e-01,
9.96293594e-01, -9.70977647e-01, 9.98924819e-01,
9.98535127e-01, 9.98273278e-01, 9.98682774e-01,
9.98556185e-01, -9.43058469e-01, 9.98655609e-01,
9.98652394e-01, 9.97367170e-01, 9.98000736e-01,
9.99016834e-01, 1.79765233e-09, 9.91199840e-01],
[ 9.89075401e-01, 9.88935866e-01, 9.89905446e-01,
9.89492504e-01, 9.74895278e-01, 9.90401184e-01,
9.89348747e-01, 9.92893798e-01, 9.91294961e-01,
9.87014134e-01, -9.49332577e-01, 9.89982094e-01,
9.88011557e-01, 9.92133050e-01, 9.90282263e-01,
9.88600916e-01, -9.20277725e-01, 9.90680163e-01,
9.89226980e-01, 9.86577350e-01, 9.89489744e-01,
9.90734007e-01, 9.91199840e-01, 9.41720628e-08]]))
N = 24
model = GEKKO(remote=False)
wesg = model.Array(model.Var, N, value = 1/N, lb=0, ub=1)
model.Equation(m.sum(wesg) == 1)
def obj(rf = 0.02):
esg_portfo = 0
for i in range(N):
esg_portfo += model.Intermediate(wesg[i] * ((esg_scores_df["Score 2019"][i] + esg_scores_df["Score 2020"][i])/2))
return_portfo = 0
for i in range(N):
return_portfo += model.Intermediate(wesg[i]*((predicted_return_dataframe.iloc[0 , i] + r.iloc[0 ,i])/2))
sigma_portfo = 0
for i in range(N):
for j in range(N):
sigma_portfo += model.Intermediate(wesg[i] * ((sigma.iloc[i,j] + sigma_bar.iloc[i,j])/2) * wesg[j])
return -esg_portfo * ((return_portfo - rf)/sigma_portfo)
model.Minimize(obj())
model.solve(disp=True)
wesg = np.array( [item[0] for item in wesg], dtype=float)
print(wesg)
print(sum(wesg))
Thanks for the well-written question and for sharing the application.

How to combine two python matrices numpy

I have this CSV "TEMP2" full of data showed below.
1376460059,4,33.29,33.23,33.23,33.29,33.23,33.29,33.29,33.29,33.33,33.29,33.33,33.29,33.33,33.33,33.37,33.33,33.33,33.33,33.33,33.37,33.37,33.37,33.37
My work so far is this:
import csv
import numpy as np
import datetime
data = np.genfromtxt('TEMP2.csv', delimiter=',')[1:]
limite=data[0]
COLUMN_NUM = int(limite)
data = np.genfromtxt('TEMP2.csv', delimiter=',')[2:]
for x in range(0, len(data)):
tiempo = (((x*1.)/COLUMN_NUM) + 1376460059)
tiempo = np.array(tiempo)
print tiempo
results = []
for i in range(0, len(data)):
tiempo = (((x*1.)/COLUMN_NUM) + 1376460059)
results.append(tiempo)
print np.hstack(results)
if data.shape[0] % 4 == 0:
print data.reshape((-1, 4))
else:
data = np.pad(data, (0, COLUMN_NUM - len(data) % COLUMN_NUM), 'constant')
print (data)
print data.reshape((-1, COLUMN_NUM))
This part of the code will give me the time and miliseconds when each data was generated.
for x in range(0, len(data)):
tiempo = (((x*1.)/COLUMN_NUM) + 1376460059)
tiempo = np.array(tiempo)
print tiempo
My question is, how can I set this results into a matrix looking something like this:
[[1376460059.0 1376460059.25 1376460059.5 1376460059.75]
[1376460060.0 1376460060.25 1376460060.5 1376460060.75]
[1376460061.0 1376460061.25 1376460061.5 1376460061.75]
[. . . . . . . . . . . . . . . . . . . and so on . .]
[1376460063.75 1376460064.0 1376460064.25 1376460064.5]]
Also, once I have it, is it possible to combine it with my other matrix to get a result like this:
[[33.29 33.23 33.23 33.29]
[1376460059.0 1376460059.25 1376460059.5 1376460059.75]
[33.23 33.29 33.29 33.29]
[1376460060.0 1376460060.25 1376460060.5 1376460060.75]
[33.33 33.29 33.33 33.29]
[1376460061.0 1376460061.25 1376460061.5 1376460061.75]
[ and so on. . . . .. . . . . .. . . . . . . .]]
I am really asking this because I have other source of help. But desperately to know. I have searched but Im not getting no where now. Thanks.

for the first part of your question, you just have to create a tab before your first iteration.
res = []
for x in range(0, len(data)):
tiempo = (((x*1.)/COLUMN_NUM) + 1376460059)
tiempo = np.array(tiempo)
print tiempo
res.append(tiempo)
for the second part of your question, to combine the two matrix, you can use numpy.concatenate:
import numpy as np
np.concatenate((A, B))
matrix([[ 1., 2.],
[ 3., 4.],
[ 5., 6.]])
I have not test my code, so I hope it will help you
I have work a little on the code, and now I have that :
import csv
import numpy as np
import datetime
data = np.genfromtxt('TEMP2.csv', delimiter=',')[1:]
limite=data[0]
COLUMN_NUM = int(limite)
data = np.genfromtxt('TEMP2.csv', delimiter=',')[2:]
results = np.zeros(shape=(len(data)))
for x in range(0, len(data)):
tiempo = (((x*1.)/COLUMN_NUM) + 1376460059)
results[x] = tiempo
if data.shape[0] % 4 == 0:
print(data.reshape((-1, 4)))
else:
data = np.pad(data, (0, COLUMN_NUM - len(data) % COLUMN_NUM), 'constant')
if results.shape[0] % 4 == 0:
print(results.reshape((-1, 4)))
else:
results = np.pad(results, (0, COLUMN_NUM - len(results) % COLUMN_NUM),
'constant')
data = data.reshape((-1, COLUMN_NUM))
results = results.reshape((-1, COLUMN_NUM))
final_matrix = np.concatenate((data, results))
final_matrix2 = []
for i in range(len(data)):
final_matrix2.append(list(data[i]))
final_matrix2.append(list(results[i]))
print(final_matrix)
print(final_matrix2)
and I have this result
[ 3.32900000e+01 3.32300000e+01 3.32300000e+01 3.32900000e+01]
[ 3.32300000e+01 3.32900000e+01 3.32900000e+01 3.32900000e+01]
[ 3.33300000e+01 3.32900000e+01 3.33300000e+01 3.32900000e+01]
[ 3.33300000e+01 3.33300000e+01 3.33700000e+01 3.33300000e+01]
[ 3.33300000e+01 3.33300000e+01 3.33300000e+01 3.33700000e+01]
[ 3.33700000e+01 3.33700000e+01 3.33700000e+01 0.00000000e+00]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 1.37646006e+09]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 1.37646006e+09]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 1.37646006e+09]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 1.37646006e+09]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 1.37646006e+09]
[ 1.37646006e+09 1.37646006e+09 1.37646006e+09 0.00000000e+00]]
[[33.289999999999999, 33.229999999999997, 33.229999999999997, 33.289999999999999], [1376460059.0, 1376460059.25, 1376460059.5, 1376460059.75], [33.229999999999997, 33.289999999999999, 33.289999999999999, 33.289999999999999], [1376460060.0, 1376460060.25, 1376460060.5, 1376460060.75], [33.329999999999998, 33.289999999999999, 33.329999999999998, 33.289999999999999], [1376460061.0, 1376460061.25, 1376460061.5, 1376460061.75], [33.329999999999998, 33.329999999999998, 33.369999999999997, 33.329999999999998], [1376460062.0, 1376460062.25, 1376460062.5, 1376460062.75], [33.329999999999998, 33.329999999999998, 33.329999999999998, 33.369999999999997], [1376460063.0, 1376460063.25, 1376460063.5, 1376460063.75], [33.369999999999997, 33.369999999999997, 33.369999999999997, 0.0], [1376460064.0, 1376460064.25, 1376460064.5, 0.0]]

Can JModelica print results directly to file?

I am running the following JModelica script:
#!/usr/local/jmodelica/bin/jm_python.sh
import pyjmi
op = pyjmi.transfer_optimization_problem("BatchReactor", "model.mop")
opt_opts = op.optimize_options()
opt_opts['n_e'] = 40 # Number of elements
opt_opts['IPOPT_options']['tol'] = 1e-10
opt_opts['IPOPT_options']['print_level'] = 8
opt_opts['IPOPT_options']['output_file'] = '/z/out'
res = op.optimize(options=opt_opts)
I had hoped that the results (e.g. time, x1, x2, &c.) would be printed to the file /z/out. But the file only contains IPOPT verbose debugging/status info.
Is there a way to print the information that would be stored in res directly to a file? Either by somehow writing res itself or, preferably, having IPOPT/JModelica write the results without having to go through Python?

There is a way to print the information directly to a file. The following accomplishes this. Note that result_file_name is the key to making this happen.
#!/usr/local/jmodelica/bin/jm_python.sh
import pyjmi
op = pyjmi.transfer_optimization_problem("BatchReactor", "model.mop")
opt_opts = op.optimize_options()
opt_opts['n_e'] = 40 # Number of elements
opt_opts['result_file_name'] = '/z/out'
opt_opts['IPOPT_options']['tol'] = 1e-10
opt_opts['IPOPT_options']['print_level'] = 0
res = op.optimize(options=opt_opts)
Unfortunately, the contents of the file are somewhat mysterious.

You may find that using result_file_name per another answer here results in an output file which is difficult to understand.
The following produces a nicer format:
import StringIO
import numpy as np
def PrintResToFile(filename,result):
def StripMX(x):
return str(x).replace('MX(','').replace(')','')
varstr = '#Variable Name={name: <10}, Unit={unit: <7}, Val={val: <10}, Col={col:< 5}, Comment="{comment}"\n'
with open(filename,'w') as fout:
#Print all variables at the top of the file, along with relevant information
#about them.
for var in result.model.getAllVariables():
if not result.is_variable(var.getName()):
val = result.initial(var.getName())
col = -1
else:
val = "Varies"
col = result.get_column(var.getName())
unit = StripMX(var.getUnit())
if not unit:
unit = "X"
fout.write(varstr.format(
name = var.getName(),
unit = unit,
val = val,
col = col,
comment = StripMX(var.getAttribute('comment'))
))
#Ensure that time variable is printed
fout.write(varstr.format(
name = 'time',
unit = 's',
val = 'Varies',
col = 0,
comment = 'None'
))
#The data matrix contains only time-varying variables. So fetch all of
#these, couple them in tuples with their column number, sort by column
#number, and then extract the name of the variable again. This results in a
#list of variable names which are guaranteed to be in the same order as the
#data matrix.
vkeys_in_order = map(lambda x: x[1], sorted([(result.get_column(x),x) for x in result.keys() if result.is_variable(x)]))
for vk in vkeys_in_order:
fout.write("{0:>13},".format(vk))
fout.write("\n")
sio = StringIO.StringIO()
np.savetxt(sio, result.data_matrix, delimiter=',', fmt='%13.5f')
fout.write(sio.getvalue())
which looks like this:
#Variable Name=S0 , Unit=kg , Val=2.0 , Col=-1 , Comment="Solid Mass"
#Variable Name=F0 , Unit=kg , Val=0.0 , Col=-1 , Comment="Fluid Mass"
#Variable Name=a , Unit=Hz , Val=0.2 , Col=-1 , Comment="None"
#Variable Name=b , Unit=kg/s , Val=1.0 , Col=-1 , Comment="None"
#Variable Name=f , Unit=kg/s , Val=0.05 , Col=-1 , Comment="None"
#Variable Name=h , Unit=1/g , Val=0.05 , Col=-1 , Comment="None"
#Variable Name=der(F) , Unit=X , Val=Varies , Col= 1 , Comment="None"
#Variable Name=F , Unit=kg , Val=Varies , Col= 3 , Comment="None"
#Variable Name=der(S) , Unit=X , Val=Varies , Col= 2 , Comment="None"
#Variable Name=S , Unit=kg , Val=Varies , Col= 4 , Comment="None"
#Variable Name=u , Unit=X , Val=Varies , Col= 5 , Comment="None"
#Variable Name=startTime , Unit=X , Val=0.0 , Col=-1 , Comment="None"
#Variable Name=finalTime , Unit=X , Val=100.0 , Col=-1 , Comment="None"
#Variable Name=time , Unit=s , Val=Varies , Col= 0 , Comment="None"
time, der(F), der(S), F, S, u,
0.00000, 0.97097, -0.97097, 0.00000, 2.00000, 0.97097
0.38763, 1.07704, -1.05814, 0.38519, 1.61698, 1.00000
1.61237, 0.88350, -0.80485, 1.70714, 0.35885, 0.65862
2.50000, 0.00000, 0.09688, 2.14545, 0.00000, 0.00000
2.88763, 0.09842, -0.00000, 2.18330, 0.00000, 0.06851
4.11237, 0.10342, 0.00000, 2.30688, 0.00000, 0.07077
5.00000, 0.10716, 0.00000, 2.40033, 0.00000, 0.07240
5.38763, 0.10882, -0.00000, 2.44219, 0.00000, 0.07311
6.61237, 0.11421, 0.00000, 2.57875, 0.00000, 0.07535

Array only filling last value

I'm having a problem with my array only filling the last space with the calculated value. My code is below:
c_volume = 4.45e-5
c_flow_rate = 1.67e-6
acr = c_flow_rate/c_volume
t1 = [3600.0,18000.0, 36000.0]
air_conc_t1 = [6.42404968e+02, 2.74977722e+02, 1.45282562e+02]
t2 = [7200.0, 21600.0, 39600.0]
air_conc_t2 = [4.53346985e+02, 2.41359268e+02, 1.28038071e+02]
===============================================================
n_calc = np.zeros((len(t1),1), dtype='f')
def n(t1, air_conc_t1, t2, air_conc_t2):
return (1/(t2-t1))*(np.log(air_conc_t1/air_conc_t2))
for i in range(len(t1)):
n_calc[i] = n(t1[i], air_conc_t1[i], t2[i], air_conc_t2[i])
===============================================================
calc_f1 = np.zeros((len(t1),1), dtype='f')
calc_f2 = np.zeros((len(t1),1), dtype='f')
calc_N = np.zeros((len(t1),1), dtype='f')
def f1(acr, n_calc):
return (acr+n_calc)/n_calc
calc_f1[i] = f1(acr, n_calc[i])
def f2(acr, n_calc):
return (acr-n_calc)/n_calc
calc_f2[i] = f2(acr, n_calc[i])
def N(R, calc_root, m_thickness, calc_f1, calc_f2):
return (2*R*np.tan(calc_root*m_thickness))/(calc_root*m_thickness*
(calc_f1+calc_f2*calc_root*m_thickness*((1/np.tan(calc_root\
*m_thickness))+np.tan(calc_root*m_thickness))))
for i in xrange(len(t1)):
calc_N[i] = N(R, calc_root[i], m_thickness, calc_f1[i], calc_f2[i])
print calc_f1
print calc_f2
print calc_N
I'm getting the following printed:
[[ 0. ]
[ 0. ]
[ 1070.23657227]]
[[ 0. ]
[ 0. ]
[ 1068.2364502]]
[[ inf]
[ inf]
[ 3.55326119e-06]]
I'm not sure why the first two values of the array are not being filled, but the last one is. I've calculated them by hand and get values.
I'm very new to programming so any help would be appreciated.
Thanks,
Shane

your calls to f1 and f2 are not in a loop. Note that everything at the same level of indentation is included in the loop.
try this instead:
for i in xrange(len(t1)):
calc_f1[i] = f1(acr, n_calc[i])
calc_f2[i] = f2(acr, n_calc[i])
calc_N[i] = N(R, calc_root[i], m_thickness, calc_f1[i], calc_f2[i])
print calc_f1
print calc_f2
print calc_N

Lines
calc_f1[i] = f1(acr, n_calc[i])
calc_f2[i] = f2(acr, n_calc[i])
are outside any for loop (maybe it is only problem with indentions).

Reading a binary file into a struct

I have a binary file with a known format/structure.
How do I read all the binary data in to an array of the structure?
Something like (in pseudo code)
bytes = read_file(filename)
struct = {'int','int','float','byte[255]'}
data = read_as_struct(bytes, struct)
data[1]
>>> 10,11,10.1,Arr[255]
My solution so far is:
data = []
fmt = '=iiiii256i'
fmt_s = '=iiiii'
fmt_spec = '256i'
struct_size = struct.calcsize(fmt)
for i in range(struct_size, len(bytes)-struct_size, struct_size):
dat1= list(struct.unpack(fmt_s, bytes[i-struct_size:i-1024]))
dat2= list(struct.unpack(fmt_spec, bytes[i-1024:i]))
dat1.append(dat2)
data.append(dat1)

Actually it looks like you're trying to read a list (or array) of structures from the file. The idiomatic way to do this in Python is use the struct module and call struct.unpack() in a loop—either a fixed number of times if you know the number of them in advance, or until end-of-file is reached—and store the results in a list. Here's an example of the latter:
import struct
struct_fmt = '=5if255s' # int[5], float, byte[255]
struct_len = struct.calcsize(struct_fmt)
struct_unpack = struct.Struct(struct_fmt).unpack_from
results = []
with open(filename, "rb") as f:
while True:
data = f.read(struct_len)
if not data: break
s = struct_unpack(data)
results.append(s)
The same results can be also obtained slightly more concisely using a list comprehension along with a short generator function helper (i.e. read_chunks() below):
def read_chunks(f, length):
while True:
data = f.read(length)
if not data: break
yield data
with open(filename, "rb") as f:
results = [struct_unpack(chunk) for chunk in read_chunks(f, struct_len)]
Update
You don't, in fact, need to explicitly define a helper function as shown above because you can use Python's built-in iter() function to dynamically create the needed iterator object in the list comprehension itself like so:
from functools import partial
with open(filename, "rb") as f:
results = [struct_unpack(chunk) for chunk in iter(partial(f.read, struct_len), b'')]

Use the struct module; you need to define the types in a string format documented with that library:
struct.unpack('=HHf255s', bytes)
The above example expects native byte-order, two unsigned shorts, a float and a string of 255 characters.
To loop over an already fully read bytes string, I'd use itertools; there is a handy grouper recipe that I've adapted here:
from itertools import izip_longest, imap
from struct import unpack, calcsize
fmt_s = '=5i'
fmt_spec = '=256i'
size_s = calcsize(fmt_s)
size = size_s + calcsize(fmt_spec)
def chunked(iterable, n, fillvalue=''):
args = [iter(iterable)] * n
return imap(''.join, izip_longest(*args, fillvalue=fillvalue))
data = [unpack(fmt_s, section[:size_s]) + (unpack(fmt_spec, section[size_s:]),)
for section in chunked(bytes, size)]
This produces tuples rather than lists, but it's easy enough to adjust if you have to:
data = [list(unpack(fmt_s, section[:size_s])) + [list(unpack(fmt_spec, section[size_s:]))]
for section in chunked(bytes, size)]

Add comments
import struct
First just read the binary into an array
mbr = file('mbrcontent', 'rb').read()
So you can just fetch some piece of the the array
partition_table = mbr[446:510]
and then unpack it as an integer
signature = struct.unpack('<H', mbr[510:512])[0]
a more complex example
little_endian = (signature == 0xaa55) # should be True
print "Little endian:", little_endian
PART_FMT = (little_endian and '<' or '>') + (
"B" # status (0x80 = bootable (active), 0x00 = non-bootable)
# CHS of first block
"B" # Head
"B" # Sector is in bits 5; bits 9 of cylinder are in bits 7-6
"B" # bits 7-0 of cylinder
"B" # partition type
# CHS of last block
"B" # Head
"B" # Sector is in bits 5; bits 9 of cylinder are in bits 7-6
"B" # bits 7-0 of cylinder
"L" # LBA of first sector in the partition
"L" # number of blocks in partition, in little-endian format
)
PART_SIZE = 16
fmt_size = struct.calcsize(PART_FMT)
# sanity check expectations
assert fmt_size == PART_SIZE, "Partition format string is %i bytes, not %i" % (fmt_size, PART_SIZE)
def cyl_sector(sector_cyl, cylinder7_0):
sector = sector_cyl & 0x1F # bits 5-0
# bits 7-6 of sector_cyl contain bits 9-8 of the cylinder
cyl_high = (sector_cyl >> 5) & 0x03
cyl = (cyl_high << 8) | cylinder7_0
return sector, cyl
#I have corrected the indentation, but the change is refused because less than 6 characters, so I am adding this useful comment.
for partition in range(4):
print "Partition #%i" % partition,
offset = PART_SIZE * partition
(status, start_head, start_sector_cyl, start_cyl7_0, part_type, end_head, end_sector_cyl, end_cyl7_0,
lba, blocks ) = struct.unpack( PART_FMT,partition_table[offset:offset + PART_SIZE])
if status == 0x80:
print "Bootable",
elif status:
print "Unknown status [%s]" % hex(status),
print "Type=0x%x" % part_type
start = (start_head,) + cyl_sector(start_sector_cyl, start_cyl7_0)
end = (end_head,) + cyl_sector(end_sector_cyl, end_cyl7_0)
print " (Start: Heads:%i\tCyl:%i\tSect:%i)" % start
print " (End: Heads:%i\tCyl:%i\tSect:%i)" % end
print " LBA:", lba
print " Blocks:", blocks

import os, re
import functools
import ctypes
from ctypes import string_at, byref, sizeof, cast, POINTER, pointer, create_string_buffer, memmove
import numpy as np
import pandas as pd
class _StructBase(ctypes.Structure):
__type__ = 0
_fields_ = []
#classmethod
def Offsetof(cls, field):
pattern = '(?P<field>\w+)\[(?P<idx>\d+)\]'
mat = re.match(pattern, field)
if mat:
fields = dict(cls.Fields())
f = mat.groupdict()['field']
idx = mat.groupdict()['idx']
return cls.Offsetof(f) + int(idx) * ctypes.sizeof(fields[field])
else:
return getattr(cls, field).offset
#classmethod
def DType(cls):
map = {
ctypes.c_byte: np.byte,
ctypes.c_ubyte: np.ubyte,
ctypes.c_char: np.ubyte,
ctypes.c_int8: np.int8,
ctypes.c_int16: np.int16,
ctypes.c_int32: np.int32,
ctypes.c_int64: np.int64,
ctypes.c_uint8: np.uint8,
ctypes.c_uint16: np.uint16,
ctypes.c_uint32: np.uint32,
ctypes.c_uint64: np.uint64,
ctypes.c_float: np.float32,
ctypes.c_double: np.float64,
}
res = []
for k, v in cls.Fields():
if hasattr(v, '_length_'):
if v._type_ != ctypes.c_char:
for i in range(v._length):
res.append((k, map[v], cls.Offsetof(k)))
else:
res.append((k, 'S%d' % v._length_, cls.Offsetof(k)))
else:
res.append((k, map[v], cls.Offsetof(k)))
res = pd.DataFrame(res, columns=['name', 'format', 'offset'])
return np.dtype({
'names': res['name'],
'formats': res['format'],
'offsets': res['offset'],
})
#classmethod
def Attr(cls):
fields = cls._fields_
res = []
for attr, tp in fields:
if str(tp).find('_Array_') > 0 and str(tp).find('char_Array_') < 0:
for i in range(tp._length_):
res.append((attr + '[%s]' % str(i), tp._type_))
else:
res.append((attr, tp))
return res
#classmethod
def Fields(cls, notype=False):
res = [cls.Attr()]
cur_cls = cls
while True:
cur_cls = cur_cls.__bases__[0]
if cur_cls == ctypes.Structure:
break
res.append(cur_cls.Attr())
if notype:
return [k for k, v in functools.reduce(list.__add__, reversed(res), [])]
else:
return functools.reduce(list.__add__, reversed(res), [])
#classmethod
def size(cls):
return sizeof(cls)
#classmethod
def from_struct_binary(cls, path, max_count=2 ** 32, decode=True):
print(os.path.getsize(path), cls.size())
assert os.path.getsize(path) % cls.size() == 0
size = os.path.getsize(path) // cls.size()
size = min(size, max_count)
index = range(size)
array = np.fromfile(path, dtype=cls.DType(), count=size)
df = pd.DataFrame(array, index=index)
for attr, tp in eval(str(cls.DType())):
if re.match('S\d+', tp) is not None and decode:
try:
df[attr] = df[attr].map(lambda x: x.decode("utf-8"))
except:
df[attr] = df[attr].map(lambda x: x.decode("gbk"))
return df
class StructBase(_StructBase):
_fields_ = [
('Type', ctypes.c_uint32),
]
class IndexStruct(StructBase):
_fields_ = [
('Seq', ctypes.c_uint32),
('ExID', ctypes.c_char * 8),
('SecID', ctypes.c_char * 8),
('SecName', ctypes.c_char * 16),
('SourceID', ctypes.c_int32),
('Time', ctypes.c_uint32),
('PreClose', ctypes.c_uint32),
('Open', ctypes.c_uint32),
('High', ctypes.c_uint32),
('Low', ctypes.c_uint32),
('Match', ctypes.c_uint32),
]
df = IndexStruct.from_struct_binary('your path')
print(df)

Categories

Resources