How to solve warning message in Gekko due to m.connection? - python
I am using m.connection to estimate variables initial conditions but I am getting 12 warning messages like:
Moreover, the APM file shows:
I am not sure how to solve these messages.
I am following this explanation "If pos1 or pos2 is not None, the associated var must be a GEKKO variable and the position is the (0-indexed) time-discretized index of the variable" to write m.Connection(var1,var2,pos1=None,pos2=None,node1='end',node2='end').
https://gekko.readthedocs.io/en/latest/quick_start.html#connections
Thanks in advance.
from gekko import GEKKO
import numpy as np
import matplotlib.pyplot as plt
import math as math
import pandas as pd
tm1 = [0, 0.0667,0.5,1,4, 22.61667]
mca1 = [5.68, 3.48, 3.24, 3.36, 2.96, 1.96]
tm2 = [0, 0.08333,0.5,1,4.25 , 22.8167]
mca2 = [5.68, 4.20, 4.04, 4.00, 3.76, 2.88]
tm3 = [0,0.08333,0.5,1,4.33 , 22.9500]
mca3 = [5.68, 4.64, 4.52, 4.56, 4.24, 3.72]
tm4 = [0,0.08333,0.5,1,4.0833 , 23.0833]
mca4 =[18.90,15.4,14.3,15.10,13.50, 10.90]
tm5 = [0,0.08333,0.5,1,4.5, 23.2167]
mca5 =[18.90, 15.5, 16.30, 16, 14.70, 13.00]
tm6 = [0,0.08333,0.5,1,4.6667, 23.3333 ]
mca6 = [18.90, 15.8, 11.70,15.5,12, 9.5 ]
df1=pd.DataFrame({'time':tm1,'x1':mca1})
df2=pd.DataFrame({'time':tm2,'x2':mca2})
df3=pd.DataFrame({'time':tm3,'x3':mca3})
df4=pd.DataFrame({'time':tm4,'x4':mca4})
df5=pd.DataFrame({'time':tm5,'x5':mca5})
df6=pd.DataFrame({'time':tm6,'x6':mca6})
df1.set_index('time',inplace=True)
df2.set_index('time',inplace=True)
df3.set_index('time',inplace=True)
df4.set_index('time',inplace=True)
df5.set_index('time',inplace=True)
df6.set_index('time',inplace=True)
#simulation time points
dfx = pd.DataFrame({'time':np.linspace(0,25,101)})
dfx.set_index('time',inplace=True)
#merge dataframes
dfxx=dfx.join(df1,how='outer')
dfxxx=dfxx.join(df2,how='outer')
dfxxxx=dfxxx.join(df3,how='outer')
dfxxxxx=dfxxxx.join(df4,how='outer')
dfxxxxxx=dfxxxxx.join(df5,how='outer')
df=dfxxxxxx.join(df6,how='outer')
# get True (1) or False (0) for measurement
df['meas1']=(df['x1'].values==df['x1'].values).astype(int)
df['meas2']=(df['x2'].values==df['x2'].values).astype(int)
df['meas3']=(df['x3'].values==df['x3'].values).astype(int)
df['meas4']=(df['x4'].values==df['x4'].values).astype(int)
df['meas5']=(df['x5'].values==df['x5'].values).astype(int)
df['meas6']=(df['x6'].values==df['x6'].values).astype(int)
#replace NaN with zeros
df0=df.fillna(value=0)
m = GEKKO()
m.time = df0.index.values
meas1 = m.Param(df0['meas1'].values)
meas2 = m.Param(df0['meas2'].values)
meas3 = m.Param(df0['meas3'].values)
meas4 = m.Param(df0['meas4'].values)
meas5 = m.Param(df0['meas5'].values)
meas6 = m.Param(df0['meas6'].values)
#adjustable Parameters
kf=m.FV(1.3,lb=0.01,ub=10)
ks=m.FV(1.3,lb=0.01,ub=10)
cnf01=m.FV(1.3,lb=0.01,ub=10)
cns01=m.FV(1.3,lb=0.01,ub=10)
#constrains
cnf02=m.FV(value=cnf01*0.5,lb=cnf01*0.5, ub=cnf01*0.5)
cns02=m.FV(value=cns01*0.5,lb=cns01*0.5, ub=cns01*0.5)
cnf03=m.FV(value=cnf01*0.25,lb=cnf01*0.25, ub=cnf01*0.25)
cns03=m.FV(value=cns01*0.25,lb=cns01*0.25, ub=cns01*0.25)
cnf04=m.FV(value=cnf01,lb=cnf01, ub=cnf01)
cns04=m.FV(value=cns01,lb=cns01, ub=cns01)
cnf05=m.FV(value=cnf01*0.5,lb=cnf01*0.5, ub=cnf01*0.5)
cns05=m.FV(value=cns01*0.5,lb=cns01*0.5, ub=cns01*0.5)
cnf06=m.FV(value=cnf01*0.25,lb=cnf01*0.25, ub=cnf01*0.25)
cns06=m.FV(value=cns01*0.25,lb=cns01*0.25, ub=cns01*0.25)
#Variables
c1 = m.Var(value=mca1[0])
c2 = m.Var(value=mca2[0])
c3 = m.Var(value=mca3[0])
c4 = m.Var(value=mca4[0])
c5 = m.Var(value=mca5[0])
c6 = m.Var(value=mca6[0])
cm1 = m.Param(df0['x1'].values)
cm2 = m.Param(df0['x2'].values)
cm3 = m.Param(df0['x3'].values)
cm4 = m.Param(df0['x4'].values)
cm5 = m.Param(df0['x5'].values)
cm6 = m.Param(df0['x6'].values)
m.Minimize((meas1*(c1-cm1)**2)+(meas2*(c2-cm2)**2)\
+(meas3*(c3-cm3)**2)+(meas4*(c4-cm4)**2)\
+(meas5*(c5-cm5)**2)+(meas6*(c6-cm6)**2))
cnf1=m.Var(value=cnf01,fixed_initial=False)
cns1=m.Var(value=cns01,fixed_initial=False)
cnf2=m.Var(value=cnf02,fixed_initial=False)
cns2=m.Var(value=cns02,fixed_initial=False)
cnf3=m.Var(value=cnf03,fixed_initial=False)
cns3=m.Var(value=cns03,fixed_initial=False)
cnf4=m.Var(value=cnf04,fixed_initial=False)
cns4=m.Var(value=cns04,fixed_initial=False)
cnf5=m.Var(value=cnf05,fixed_initial=False)
cns5=m.Var(value=cns05,fixed_initial=False)
cnf6=m.Var(value=cnf06,fixed_initial=False)
cns6=m.Var(value=cns06,fixed_initial=False)
#Equations
t = m.Param(value=m.time)
m.Connection(cnf1,cnf01,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cnf2,cnf02,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cnf3,cnf03,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cnf4,cnf04,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cnf5,cnf05,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cnf6,cnf06,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns1,cns01,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns2,cns02,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns3,cns03,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns4,cns04,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns5,cns05,pos1=0,pos2=0,node1=1,node2=1)
m.Connection(cns6,cns06,pos1=0,pos2=0,node1=1,node2=1)
m.Equation(cnf1.dt()==-kf*c1*cnf1)
m.Equation(cns1.dt()==-ks*c1*cns1)
m.Equation(c1.dt()==cnf1.dt()+cns1.dt())
m.Equation(cnf2.dt()==-kf*c2*cnf2)
m.Equation(cns2.dt()==-ks*c2*cns2)
m.Equation(c2.dt()==cnf2.dt()+cns2.dt())
m.Equation(cnf3.dt()==-kf*c3*cnf3)
m.Equation(cns3.dt()==-ks*c3*cns3)
m.Equation(c3.dt()==cnf3.dt()+cns3.dt())
m.Equation(cnf4.dt()==-kf*c4*cnf4)
m.Equation(cns4.dt()==-ks*c4*cns4)
m.Equation(c4.dt()==cnf4.dt()+cns4.dt())
m.Equation(cnf5.dt()==-kf*c5*cnf5)
m.Equation(cns5.dt()==-ks*c5*cns5)
m.Equation(c5.dt()==cnf5.dt()+cns5.dt())
m.Equation(cnf6.dt()==-kf*c6*cnf6)
m.Equation(cns6.dt()==-ks*c6*cns6)
m.Equation(c6.dt()==cnf6.dt()+cns6.dt())
if True:
kf.STATUS=1
ks.STATUS=1
cnf01.STATUS=1
cns01.STATUS=1
cnf02.STATUS=1
cns02.STATUS=1
cnf03.STATUS=1
cns03.STATUS=1
cnf04.STATUS=1
cns04.STATUS=1
cnf05.STATUS=1
cns05.STATUS=1
cnf06.STATUS=1
cns06.STATUS=1
#Options
m.options.SOLVER = 1 #IPOPT solver
m.options.IMODE = 5 #Dynamic Simultaneous - estimation = MHE
m.options.EV_TYPE = 2 #absolute error
m.options.NODES = 3 #collocation nodes (2,5)
m.solve(disp=True)
m.open_folder()
print('Final SSE Objective: ' + str(m.options.objfcnval))
print('Solution')
print('cnf01 = ' + str(cnf01.value[0]))
print('cns01 = ' + str(cns01.value[0]))
print('kf = ' + str(kf.value[0]))
print('ks = ' + str(ks.value[0]))
print('cns02 = '+ str(cns02.value[0]))
print('cnf02 = '+ str(cnf02.value[0]))
print('cns03 = '+ str(cns03.value[0]))
print('cnf03 = '+ str(cnf03.value[0]))
print('cns04 = '+ str(cns04.value[0]))
print('cnf04 = '+ str(cnf04.value[0]))
print('cns05 = '+ str(cns05.value[0]))
print('cnf05 = '+ str(cnf05.value[0]))
print('cns06 = '+ str(cns06.value[0]))
print('cnf06 = '+ str(cnf06.value[0]))
plt.figure(1,figsize=(8,5))
plt.plot(m.time,c1.value,'r',label='Predicted c1')
plt.plot(m.time,c2.value,'y',label='Predicted c2')
plt.plot(m.time,c3.value,'c',label='Predicted c3')
plt.plot(m.time,c4.value,'g',label='Predicted c4')
plt.plot(m.time,c5.value,'b',label='Predicted c5')
plt.plot(m.time,c6.value,'m',label='Predicted c6')
plt.plot(tm1,mca1,'rx',label='Meas c1')
plt.plot(tm2,mca2,'yx',label='Meas c2')
plt.plot(tm3,mca3,'cx',label='Meas c3')
plt.plot(tm4,mca4,'go',label='Meas c4')
plt.plot(tm5,mca5,'bo',label='Meas c5')
plt.plot(tm6,mca6,'mo',label='Meas c6')
plt.xlabel('time (h)')
plt.ylabel('Concentration (mgCl2/L)')
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=2)
The underlying node structure has a 1-index instead of a 0-index that is common in Python. Using pos1=1 and pos2=1 resolves the warnings.
m.Connection(cnf1,cnf01,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf2,cnf02,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf3,cnf03,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf4,cnf04,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf5,cnf05,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf6,cnf06,pos1=1,pos2=1,node1=1,node2=1)
Another issue is that Gekko variables shouldn't generally be used to initialize other values. I recommend setting x0=1.3 and using that float to initialize the variables. Change m.Var() to m.SV() to avoid reclassification of m.Var() as an m.FV() during the connection. The m.SV() is a promoted type of variable that is at the same level of precedence as the m.FV(). Here is a complete script although the results don't look optimal.
from gekko import GEKKO
import numpy as np
import matplotlib.pyplot as plt
import math as math
import pandas as pd
tm1 = [0, 0.0667,0.5,1,4, 22.61667]
mca1 = [5.68, 3.48, 3.24, 3.36, 2.96, 1.96]
tm2 = [0, 0.08333,0.5,1,4.25 , 22.8167]
mca2 = [5.68, 4.20, 4.04, 4.00, 3.76, 2.88]
tm3 = [0,0.08333,0.5,1,4.33 , 22.9500]
mca3 = [5.68, 4.64, 4.52, 4.56, 4.24, 3.72]
tm4 = [0,0.08333,0.5,1,4.0833 , 23.0833]
mca4 =[18.90,15.4,14.3,15.10,13.50, 10.90]
tm5 = [0,0.08333,0.5,1,4.5, 23.2167]
mca5 =[18.90, 15.5, 16.30, 16, 14.70, 13.00]
tm6 = [0,0.08333,0.5,1,4.6667, 23.3333 ]
mca6 = [18.90, 15.8, 11.70,15.5,12, 9.5 ]
df1=pd.DataFrame({'time':tm1,'x1':mca1})
df2=pd.DataFrame({'time':tm2,'x2':mca2})
df3=pd.DataFrame({'time':tm3,'x3':mca3})
df4=pd.DataFrame({'time':tm4,'x4':mca4})
df5=pd.DataFrame({'time':tm5,'x5':mca5})
df6=pd.DataFrame({'time':tm6,'x6':mca6})
df1.set_index('time',inplace=True)
df2.set_index('time',inplace=True)
df3.set_index('time',inplace=True)
df4.set_index('time',inplace=True)
df5.set_index('time',inplace=True)
df6.set_index('time',inplace=True)
#simulation time points
dfx = pd.DataFrame({'time':np.linspace(0,25,101)})
dfx.set_index('time',inplace=True)
#merge dataframes
dfxx=dfx.join(df1,how='outer')
dfxxx=dfxx.join(df2,how='outer')
dfxxxx=dfxxx.join(df3,how='outer')
dfxxxxx=dfxxxx.join(df4,how='outer')
dfxxxxxx=dfxxxxx.join(df5,how='outer')
df=dfxxxxxx.join(df6,how='outer')
# get True (1) or False (0) for measurement
df['meas1']=(df['x1'].values==df['x1'].values).astype(int)
df['meas2']=(df['x2'].values==df['x2'].values).astype(int)
df['meas3']=(df['x3'].values==df['x3'].values).astype(int)
df['meas4']=(df['x4'].values==df['x4'].values).astype(int)
df['meas5']=(df['x5'].values==df['x5'].values).astype(int)
df['meas6']=(df['x6'].values==df['x6'].values).astype(int)
#replace NaN with zeros
df0=df.fillna(value=0)
m = GEKKO()
m.time = df0.index.values
meas1 = m.Param(df0['meas1'].values)
meas2 = m.Param(df0['meas2'].values)
meas3 = m.Param(df0['meas3'].values)
meas4 = m.Param(df0['meas4'].values)
meas5 = m.Param(df0['meas5'].values)
meas6 = m.Param(df0['meas6'].values)
#adjustable Parameters
kf=m.FV(1.3,lb=0.01,ub=10)
ks=m.FV(1.3,lb=0.01,ub=10)
x0 = 1.3
cnf01=m.FV(x0,lb=0.01,ub=10)
cns01=m.FV(x0,lb=0.01,ub=10)
#constrains
cnf02=m.FV(value=x0*0.5,lb=x0*0.5, ub=x0*0.5)
cns02=m.FV(value=x0*0.5,lb=x0*0.5, ub=x0*0.5)
cnf03=m.FV(value=x0*0.25,lb=x0*0.25, ub=x0*0.25)
cns03=m.FV(value=x0*0.25,lb=x0*0.25, ub=x0*0.25)
cnf04=m.FV(value=x0,lb=x0, ub=x0)
cns04=m.FV(value=x0,lb=x0, ub=x0)
cnf05=m.FV(value=x0*0.5,lb=x0*0.5, ub=x0*0.5)
cns05=m.FV(value=x0*0.5,lb=x0*0.5, ub=x0*0.5)
cnf06=m.FV(value=x0*0.25,lb=x0*0.25, ub=x0*0.25)
cns06=m.FV(value=x0*0.25,lb=x0*0.25, ub=x0*0.25)
#Variables
c1 = m.SV(value=mca1[0])
c2 = m.SV(value=mca2[0])
c3 = m.SV(value=mca3[0])
c4 = m.SV(value=mca4[0])
c5 = m.SV(value=mca5[0])
c6 = m.SV(value=mca6[0])
cm1 = m.Param(df0['x1'].values)
cm2 = m.Param(df0['x2'].values)
cm3 = m.Param(df0['x3'].values)
cm4 = m.Param(df0['x4'].values)
cm5 = m.Param(df0['x5'].values)
cm6 = m.Param(df0['x6'].values)
m.Minimize((meas1*(c1-cm1)**2)+(meas2*(c2-cm2)**2)\
+(meas3*(c3-cm3)**2)+(meas4*(c4-cm4)**2)\
+(meas5*(c5-cm5)**2)+(meas6*(c6-cm6)**2))
cnf1=m.SV(value=x0,fixed_initial=False)
cns1=m.SV(value=x0,fixed_initial=False)
cnf2=m.SV(value=x0,fixed_initial=False)
cns2=m.SV(value=x0,fixed_initial=False)
cnf3=m.SV(value=x0,fixed_initial=False)
cns3=m.SV(value=x0,fixed_initial=False)
cnf4=m.SV(value=x0,fixed_initial=False)
cns4=m.SV(value=x0,fixed_initial=False)
cnf5=m.SV(value=x0,fixed_initial=False)
cns5=m.SV(value=x0,fixed_initial=False)
cnf6=m.SV(value=x0,fixed_initial=False)
cns6=m.SV(value=x0,fixed_initial=False)
#Equations
t = m.Param(value=m.time)
m.Connection(cnf1,cnf01,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf2,cnf02,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf3,cnf03,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf4,cnf04,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf5,cnf05,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cnf6,cnf06,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns1,cns01,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns2,cns02,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns3,cns03,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns4,cns04,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns5,cns05,pos1=1,pos2=1,node1=1,node2=1)
m.Connection(cns6,cns06,pos1=1,pos2=1,node1=1,node2=1)
m.Equation(cnf1.dt()==-kf*c1*cnf1)
m.Equation(cns1.dt()==-ks*c1*cns1)
m.Equation(c1.dt()==cnf1.dt()+cns1.dt())
m.Equation(cnf2.dt()==-kf*c2*cnf2)
m.Equation(cns2.dt()==-ks*c2*cns2)
m.Equation(c2.dt()==cnf2.dt()+cns2.dt())
m.Equation(cnf3.dt()==-kf*c3*cnf3)
m.Equation(cns3.dt()==-ks*c3*cns3)
m.Equation(c3.dt()==cnf3.dt()+cns3.dt())
m.Equation(cnf4.dt()==-kf*c4*cnf4)
m.Equation(cns4.dt()==-ks*c4*cns4)
m.Equation(c4.dt()==cnf4.dt()+cns4.dt())
m.Equation(cnf5.dt()==-kf*c5*cnf5)
m.Equation(cns5.dt()==-ks*c5*cns5)
m.Equation(c5.dt()==cnf5.dt()+cns5.dt())
m.Equation(cnf6.dt()==-kf*c6*cnf6)
m.Equation(cns6.dt()==-ks*c6*cns6)
m.Equation(c6.dt()==cnf6.dt()+cns6.dt())
#Options
m.options.SOLVER = 1 # APOPT solver
m.options.IMODE = 5 # Dynamic Simultaneous - estimation = MHE
m.options.EV_TYPE = 2 # Squared error
m.options.NODES = 3 # Collocation nodes (2,5)
if True:
kf.STATUS=1
ks.STATUS=1
cnf01.STATUS=1
cns01.STATUS=1
cnf02.STATUS=1
cns02.STATUS=1
cnf03.STATUS=1
cns03.STATUS=1
cnf04.STATUS=1
cns04.STATUS=1
cnf05.STATUS=1
cns05.STATUS=1
cnf06.STATUS=1
cns06.STATUS=1
m.options.TIME_SHIFT = 0
try:
m.solve(disp=True)
except:
print("don't stop when not finding cnf01...cnf06")
#m.open_folder()
print('Final SSE Objective: ' + str(m.options.objfcnval))
print('Solution')
print('cnf01 = ' + str(cnf1.value[0]))
print('cns01 = ' + str(cns1.value[0]))
print('kf = ' + str(kf.value[0]))
print('ks = ' + str(ks.value[0]))
print('cns02 = '+ str(cns2.value[0]))
print('cnf02 = '+ str(cnf2.value[0]))
print('cns03 = '+ str(cns3.value[0]))
print('cnf03 = '+ str(cnf3.value[0]))
print('cns04 = '+ str(cns4.value[0]))
print('cnf04 = '+ str(cnf4.value[0]))
print('cns05 = '+ str(cns5.value[0]))
print('cnf05 = '+ str(cnf5.value[0]))
print('cns06 = '+ str(cns6.value[0]))
print('cnf06 = '+ str(cnf6.value[0]))
plt.figure(1,figsize=(8,5))
plt.plot(m.time,c1.value,'r',label='Predicted c1')
plt.plot(m.time,c2.value,'y',label='Predicted c2')
plt.plot(m.time,c3.value,'c',label='Predicted c3')
plt.plot(m.time,c4.value,'g',label='Predicted c4')
plt.plot(m.time,c5.value,'b',label='Predicted c5')
plt.plot(m.time,c6.value,'m',label='Predicted c6')
plt.plot(tm1,mca1,'rx',label='Meas c1')
plt.plot(tm2,mca2,'yx',label='Meas c2')
plt.plot(tm3,mca3,'cx',label='Meas c3')
plt.plot(tm4,mca4,'go',label='Meas c4')
plt.plot(tm5,mca5,'bo',label='Meas c5')
plt.plot(tm6,mca6,'mo',label='Meas c6')
plt.xlabel('time (h)')
plt.ylabel('Concentration (mgCl2/L)')
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=2)
plt.show()
Related
How to derive scipy's global minimum for a specific equation with trust region reflective algorithm
A new data set was created by adding random variables from a specific data set. We want to solve the problem of matching two inconsistent data sets. So I tried to solve the optimization problem using scipy's minimum_squares function. The code and data used are as follows.
from numba import jit, prange
from scipy.optimize import minimize,least_squares
import scipy.io as sio
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import basinhopping
def Data_loader(name,normalize):
global X
global Y
arr = sio.loadmat(name)
X = arr['X']
Y = X
R = np.array([[1.0171, 0.0249, 0.1269], [0.1865, 0.9383, -0.3441], [0.0172, 0.5278, 1.0347]])
# R = np.random.rand(3,3)
X = np.random.rand(1)*np.dot(X,np.transpose(R))+1
X = X*0.1
X = X[0:len(X) - 20]
Y = Y[19:len(Y)]
if normalize==True :
n = X.shape[0]
m = Y.shape[0]
x_d = np.mean(X, axis=0)
y_d = np.mean(Y, axis=0)
x = X - np.tile(x_d, [n, 1])
y = Y - np.tile(y_d, [m, 1])
x_scale = np.sqrt(np.sum(np.sum(np.power(X,2),axis=0))/n)
y_scale = np.sqrt(np.sum(np.sum(np.power(Y, 2), axis=0))/m)
x = x / x_scale
y = y / y_scale
else :
x=X
y=Y
return x,y,x_scale,y_scale,x_d,y_d
X_data,Y_data,x_scale,y_scale,x_d,y_d = Data_loader(name='cpd_data3D_face.mat',normalize=True)
#jit(nopython=True, parallel=True)
def P_cal(X,Y,sigma2,outlier):
N = X.shape[0]
M = Y.shape[0]
D = X.shape[1]
P = np.zeros(shape = (N,M),dtype=np.float64)
Np = np.zeros(shape = (N,M),dtype=np.float64)
Pmn = np.zeros(shape = (N,M),dtype=np.float64)
ksig = -2.0 * sigma2
outlier_tmp = (outlier*M*((-ksig*3.1415926535)**(0.5*D)))/((1-outlier)*N)
for n in prange(0,N):
sp=np.zeros(shape = (1,),dtype=np.float64)
for m in prange(0,M):
razn=0
for d in prange(0,D):
diff = X[n,d] - Y[m,d]
diff = diff*diff
razn += diff
P[n,m] = np.exp(razn/ksig)
sp = P.sum(axis=1)[n] + outlier_tmp
for m in prange(0,M):
Np[n,m] = P[n,m]/sp
razn = 0
for d in prange(0,D):
diff = X[n,d] - Y[m,d]
diff = diff*diff
razn += diff
Pmn[n,m] = (P[n,m]/sp) * razn
return Pmn, Np
def model(xin):
global T
N = X_data.shape[0]
M = Y_data.shape[0]
D = X_data.shape[1]
B = np.array([[xin[0], xin[1], xin[2]], [xin[3], xin[4], xin[5]], [xin[6], xin[7], xin[8]]])
t = np.array([xin[9], xin[10], xin[11]])
sigma2 = xin[12]
outlier = 0.6
T = np.dot(Y_data,np.transpose(B)) + np.tile(np.transpose(t), [M, 1])
Pmn, Np = P_cal(X_data, T, sigma2, outlier)
Q = ((1 / sigma2) * (np.sum(Pmn))) + (((np.sum(Np) * D) / 2) * np.log(sigma2))
return Q
def opt_function(xin):
Q= model(xin)
print(Q)
Q= 10000+Q
return Q
def plot_function(X, Y, T) :
X_x = X[1:X.shape[0], 0]
X_y = X[1:X.shape[0], 1]
X_z = X[1:X.shape[0], 2]
Y_x = Y[1:Y.shape[0], 0]
Y_y = Y[1:Y.shape[0], 1]
Y_z = Y[1:Y.shape[0], 2]
T_x = T[1:T.shape[0], 0]
T_y = T[1:T.shape[0], 1]
T_z = T[1:T.shape[0], 2]
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111, projection='3d')
ax.scatter(X_x, X_y, X_z)
ax.scatter(Y_x, Y_y, Y_z)
ax.set_title('before')
fig2 = plt.figure(figsize=(10, 10))
ax = fig2.add_subplot(111, projection='3d')
ax.scatter(X_x, X_y, X_z)
ax.scatter(T_x, T_y, T_z)
ax.set_title('after')
plt.show()
def transformer_data(T,Y, xin, x_scale, y_scale, x_d, y_d):
M = Y.shape[0]
Trans_R = np.array([xin[0], xin[1], xin[2], xin[3], xin[4], xin[5], xin[6], xin[7], xin[8]])
Trans_R = Trans_R.reshape((3, 3))
Trans_t = np.array([xin[9], xin[10], xin[11]])
Trans_s = 1*(x_scale/y_scale)
Trans_t = (x_scale*np.transpose(Trans_t)) + np.transpose(x_d) - Trans_s*(Trans_R#np.transpose(y_d))
Trans_T = T * x_scale + np.tile(x_d, [M, 1])
return Trans_T
x0 = np.array([1,0,0,0,1,0,0,0,1,0,0,0,0.0004])
Optimum = least_squares(opt_function,x0,jac='2-point',method='trf',xtol=1e-15,ftol=1e-15,gtol=1e-15,verbose=2)
print(Optimum)
Transformer_T = transformer_data(T,Y,Optimum.x,x_scale,y_scale,x_d,y_d)
plot_function(X,Y,Transformer_T)
[cpd_data3D_face.mat]
-1.26805102825165 0.703154087066650 0.250609993934631
-1.24199450016022 0.766805052757263 0.647247254848480
-1.23889672756195 1.06739985942841 -0.188671588897705
-1.21881270408630 0.807438790798187 1.16736936569214
-1.18697428703308 0.626793265342712 -0.182836949825287
-1.16291773319244 0.822238981723785 1.83739590644836
-1.14834547042847 0.493062376976013 0.635148286819458
-1.13797473907471 0.331227302551270 0.219468995928764
-1.13209247589111 1.01279878616333 -0.797961413860321
-1.11951982975006 0.473790675401688 1.03894352912903
-1.06242597103119 0.390692919492722 1.47615230083466
-1.04522788524628 0.358604818582535 0.543326914310455
-1.04382777214050 0.319657772779465 0.706707119941711
-1.03485214710236 0.254935413599014 0.932691574096680
-1.01772892475128 0.258871674537659 1.21425902843475
-1.01639711856842 0.200742736458778 -0.224685385823250
-1.01449811458588 0.528465926647186 -0.686351001262665
-0.983937680721283 0.189417555928230 1.00790750980377
-0.983821272850037 0.745310366153717 -1.11274755001068
-0.978638589382172 0.299691230058670 0.629687130451202
-0.922493815422058 0.150767520070076 0.828504979610443
-0.909300625324249 0.283628404140472 0.559927582740784
-0.906097352504730 0.933652937412262 -1.29045069217682
-0.891153812408447 0.289580315351486 0.480834931135178
-0.889832675457001 0.129670828580856 0.226587772369385
-0.879191935062408 0.845484137535095 2.29874420166016
-0.854662895202637 0.191300705075264 0.728563904762268
-0.837804734706879 0.327339112758637 1.77278757095337
-0.818458557128906 0.118494018912315 1.31940019130707
-0.813382029533386 0.144385874271393 -0.574904143810272
-0.806623399257660 0.0640521720051765 1.12445175647736
-0.806076943874359 0.179906263947487 0.362056881189346
-0.794836580753326 1.03445231914520 -1.48189103603363
-0.791478812694550 0.239165067672730 0.551705360412598
-0.790624439716339 0.193809807300568 0.438241928815842
-0.787293195724487 0.173615008592606 0.588202476501465
-0.780423700809479 0.399599254131317 -1.30588448047638
-0.775464117527008 -0.00958730466663837 -0.174502417445183
-0.767135858535767 0.172049850225449 0.568943262100220
-0.763602852821350 0.184372812509537 0.565283775329590
-0.762090682983398 0.196064323186874 0.504386007785797
-0.760000824928284 0.188478738069534 0.540778219699860
-0.755698919296265 0.209235891699791 0.508155345916748
-0.746246218681335 0.0437246896326542 0.911044776439667
-0.704111039638519 0.711774110794067 -1.47775316238403
-0.656321167945862 0.119024306535721 -1.07423388957977
-0.650056242942810 0.0959970802068710 0.784161567687988
-0.644072175025940 -0.00454827398061752 -0.505615711212158
-0.602208197116852 0.0996995121240616 0.379885524511337
-0.600404739379883 0.0790802314877510 0.659766137599945
-0.600255548954010 0.106224447488785 0.449432760477066
-0.598033308982849 0.120790801942348 0.452769994735718
-0.596876800060272 0.102229975163937 0.541571617126465
-0.595354616641998 0.0876417979598045 0.651539027690888
-0.594505965709686 0.0156574435532093 -0.975354671478272
-0.593853473663330 0.0964264348149300 0.647029995918274
-0.593455016613007 0.889857470989227 -1.61020958423615
-0.592627882957459 0.101232483983040 0.321590214967728
-0.573011338710785 -0.0138365831226110 -0.755739450454712
-0.556460380554199 -0.00661153765395284 0.212208628654480
-0.543379187583923 0.113693088293076 -1.46306395530701
-0.519296109676361 -0.0184836350381374 -1.20552349090576
-0.489446550607681 0.0686619207262993 0.649044036865234
-0.489271312952042 -0.0751322507858276 -0.480372875928879
-0.488831192255020 0.0591186434030533 0.653709352016449
-0.484343349933624 0.0449181720614433 0.664057910442352
-0.477669984102249 0.0742370337247849 0.549954950809479
-0.464891046285629 -0.0795564651489258 -0.776580691337585
-0.459231346845627 0.543196737766266 -1.60204350948334
-0.457071840763092 0.0801357030868530 0.468159794807434
-0.453608423471451 0.0624924153089523 0.466883003711700
-0.453008860349655 0.0603442415595055 0.380578845739365
-0.450547099113464 -0.137127891182899 -0.103212133049965
-0.449798971414566 0.0621318705379963 0.336125582456589
-0.414782941341400 -0.0911946594715118 -0.962513506412506
-0.404184430837631 -0.105692356824875 -0.845229566097260
-0.401245862245560 0.0562914907932282 0.555313587188721
-0.400264382362366 0.0315503329038620 0.797742426395416
-0.399596065282822 0.0716907083988190 0.495797544717789
-0.397346645593643 0.0558350458741188 0.493942171335220
-0.396078765392303 0.800601005554199 -1.71100533008575
-0.395203620195389 0.0524718016386032 0.630690515041351
-0.395150482654572 0.0250202100723982 0.647774577140808
-0.394768565893173 0.0421436578035355 0.636364042758942
-0.394747585058212 -0.106679096817970 -0.805121719837189
-0.392148017883301 -0.118094518780708 0.929876744747162
-0.385803490877152 -0.149832293391228 -0.273493885993958
-0.364510416984558 0.881731152534485 2.62541174888611
-0.360802918672562 -0.143175244331360 -0.298475503921509
-0.360712468624115 -0.165030494332314 -0.683113873004913
-0.354962378740311 -0.158224657177925 -0.731766164302826
-0.351477324962616 -0.153869569301605 -0.548111081123352
-0.350320786237717 -0.155996680259705 1.20539724826813
-0.347673654556274 0.0197845026850700 0.360736131668091
-0.343419760465622 0.0446846485137939 0.414639592170715
-0.341843873262405 0.0630866438150406 0.510752856731415
-0.341169834136963 0.0492043718695641 0.502054572105408
-0.337658882141113 0.0351469479501247 0.561841547489166
-0.337361693382263 -0.133780807256699 -0.763273239135742
-0.336719214916229 0.0249884724617004 0.577858805656433
-0.336661010980606 0.0514110624790192 0.550272345542908
-0.335812389850616 -0.0870144963264465 0.227063819766045
-0.334714591503143 -0.117261730134487 -0.773931920528412
-0.334714591503143 -0.117261730134487 -0.768631875514984
-0.327723145484924 0.200987175107002 1.97596061229706
-0.321362793445587 -0.147063419222832 -1.27225828170776
-0.319569408893585 -0.0633241608738899 -1.53213012218475
-0.313458234071732 -0.168399006128311 -0.838223099708557
-0.307326167821884 -0.179740279912949 -0.134934157133102
-0.301624149084091 -0.0517277717590332 1.45116829872131
-0.296892076730728 0.0488278791308403 0.533883988857269
-0.296568334102631 -0.140135616064072 -1.04919373989105
-0.295273929834366 0.0496075637638569 0.512347042560577
-0.293163061141968 -0.172096326947212 -0.910953700542450
-0.284124016761780 0.0332795977592468 0.505501687526703
-0.282951623201370 0.0313751697540283 0.540355861186981
-0.269329160451889 0.0239753406494856 0.553192079067230
-0.267992317676544 -0.181957572698593 -0.160023182630539
-0.267015933990479 -0.177904888987541 -0.321233123540878
-0.265541255474091 -0.245476022362709 -0.144584536552429
-0.262671560049057 -0.202741548418999 -0.0718050673604012
-0.261472642421722 0.0231583528220654 0.492826491594315
-0.248518571257591 -0.191129758954048 -0.211526885628700
-0.243474200367928 -0.315041571855545 -0.0893090441823006
-0.230440512299538 -0.228538274765015 -0.0157361794263124
-0.228599637746811 -0.249311119318008 -0.0108560575172305
-0.226721569895744 -0.278264611959457 -0.213387131690979
-0.224844038486481 -0.332080185413361 -0.176096573472023
-0.223889231681824 0.318416297435761 -1.66368579864502
-0.214929193258286 -0.266498714685440 -0.676058530807495
-0.210813075304031 -0.223832860589027 -0.255875170230866
-0.207094073295593 -0.281632810831070 -0.595061719417572
-0.204060018062592 -0.0362701974809170 0.500299334526062
-0.199752956628799 -0.219154447317123 -0.731922388076782
-0.198813900351524 -0.259246885776520 -0.518303036689758
-0.197991833090782 -0.199447497725487 -0.743696689605713
-0.197991833090782 -0.199447497725487 -0.738389372825623
-0.197726234793663 -0.252106904983521 -0.839765965938568
-0.193233251571655 -0.370242148637772 -0.0653356760740280
-0.190501421689987 -0.284990489482880 0.0164594296365976
-0.189626261591911 -0.382326304912567 -0.151752114295959
-0.188193947076797 -0.00516866426914930 0.692332863807678
-0.187763944268227 -0.247889831662178 -0.938535273075104
-0.186157137155533 -0.312341511249542 -0.215669736266136
-0.184836089611053 -0.161518707871437 0.312147319316864
-0.175839409232140 -0.193330869078636 -1.04541945457459
-0.169654235243797 -0.245794147253037 -0.369660794734955
-0.165510952472687 -0.0105156786739826 0.591866850852966
-0.163967385888100 0.744191050529480 -1.78527390956879
-0.163914248347282 -0.269649654626846 -0.244318157434464
-0.159129753708839 -0.366783618927002 -0.183630362153053
-0.153681874275208 -0.152272656559944 0.826754629611969
-0.140536934137344 -0.201197594404221 1.15596377849579
-0.133895203471184 -0.233975455164909 -1.29469370841980
-0.132924452424049 -0.456851601600647 -0.132863938808441
-0.130474045872688 -0.456304907798767 -0.0475655905902386
-0.118909314274788 -0.346768766641617 -0.195299670100212
-0.117933601140976 -0.151625871658325 -1.55685293674469
-0.115535654127598 -0.404653489589691 0.0411310084164143
-0.113641686737537 -0.379890978336334 0.0627729669213295
-0.108262777328491 -0.130481123924255 0.724745750427246
-0.0882218107581139 -0.447865873575211 -0.161400064826012
-0.0852775499224663 -0.299838036298752 -0.391834944486618
-0.0817874670028687 -0.306500911712647 -0.520046114921570
-0.0744881182909012 -0.330223441123962 -0.562329113483429
-0.0678362473845482 -0.302734911441803 -0.270571678876877
-0.0670988634228706 -0.528554797172546 -0.130210265517235
-0.0566540732979775 -0.283123373985291 -0.943625330924988
-0.0483675263822079 -0.303127199411392 -0.828308999538422
-0.0482353903353214 -0.329624116420746 -0.647095203399658
-0.0457053110003471 -0.543731153011322 -0.0194005407392979
-0.0454770512878895 -0.506025552749634 0.0706802755594254
-0.0451158955693245 -0.352757841348648 -0.214075580239296
-0.0422411337494850 -0.172186344861984 0.619287312030792
-0.0417529530823231 -0.456273496150970 0.111095674335957
-0.0337586030364037 -0.300846278667450 0.405985265970230
-0.0330427736043930 -0.261952579021454 -0.736106753349304
-0.0330427736043930 -0.261952579021454 -0.730799376964569
-0.0326184481382370 -0.289149373769760 -0.717697024345398
-0.0299390628933907 -0.481825888156891 -0.177739560604095
-0.0276746246963739 -0.431670397520065 -0.168245419859886
0.0121531123295426 -0.549932301044464 -0.00995034258812666
0.0121531123295426 -0.541036725044251 -0.122998714447021
0.0121531123295426 -0.512953639030457 0.0811826586723328
0.0121531123295426 -0.486913204193115 -0.174502417445183
0.0121531123295426 -0.465853661298752 0.124627687036991
0.0121531123295426 -0.426710635423660 -0.205699533224106
0.0121531123295426 -0.366242557764053 -0.241559505462646
0.0121531123295426 -0.336180478334427 -0.658293366432190
0.0121531123295426 -0.331316232681274 -0.596917092800140
0.0121531123295426 -0.318112850189209 0.404608368873596
0.0121531123295426 -0.316532135009766 -0.281554996967316
0.0121531123295426 -0.302485466003418 -0.736324012279511
0.0121531123295426 -0.296682208776474 -0.830315709114075
0.0121531123295426 -0.292714387178421 -0.534635245800018
0.0121531123295426 -0.277532249689102 -0.937846839427948
0.0121531123295426 -0.276381313800812 -0.391942352056503
0.0121531123295426 -0.261135667562485 -0.756581723690033
0.0121531123295426 -0.261135667562485 -0.751281678676605
0.0121531123295426 -0.259597063064575 -1.31272983551025
0.0121531123295426 -0.216226130723953 -1.03725826740265
0.0121531123295426 -0.211855024099350 0.920858681201935
0.0121531123295426 -0.198895767331123 -1.54220521450043
0.0121531123295426 -0.190407738089561 1.15325403213501
0.0121531123295426 -0.184148013591766 0.634032607078552
0.0121531123295426 -0.182100847363472 0.750254631042481
0.0121531123295426 -0.0808928310871124 1.42999994754791
0.0121531123295426 0.176368087530136 2.03060126304626
0.0121531123295426 0.222836643457413 -1.69991946220398
0.0121531123295426 0.750662505626679 -1.84563207626343
0.0121531123295426 0.938862383365631 2.70322513580322
0.0519808456301689 -0.431670397520065 -0.168245419859886
0.0542452894151211 -0.481825888156891 -0.177739560604095
0.0569246746599674 -0.289149373769760 -0.717697024345398
0.0573489926755428 -0.261952579021454 -0.736106753349304
0.0573489926755428 -0.261952579021454 -0.730799376964569
0.0580648221075535 -0.300846278667450 0.405985265970230
0.0660591870546341 -0.456273496150970 0.111095674335957
0.0665473639965057 -0.172186344861984 0.619287312030792
0.0694221258163452 -0.352757841348648 -0.214075580239296
0.0697832852602005 -0.506025552749634 0.0706802755594254
0.0700115412473679 -0.543731153011322 -0.0194005407392979
0.0725416243076325 -0.329624116420746 -0.647095203399658
0.0726737603545189 -0.303127199411392 -0.828308999538422
0.0809603035449982 -0.283123373985291 -0.943625330924988
0.0914050936698914 -0.528554797172546 -0.130210265517235
0.0921424329280853 -0.302734911441803 -0.270571678876877
0.0987943112850189 -0.330223441123962 -0.562329113483429
0.106093667447567 -0.306500911712647 -0.520046114921570
0.109583787620068 -0.299838036298752 -0.391834944486618
0.112528048455715 -0.447865873575211 -0.161400064826012
0.132569059729576 -0.130481123924255 0.724745750427246
0.137947961688042 -0.379890978336334 0.0627729669213295
0.139841854572296 -0.404653489589691 0.0411310084164143
0.142239809036255 -0.151625871658325 -1.55685293674469
0.143215626478195 -0.346768766641617 -0.195299670100212
0.154780268669128 -0.456304907798767 -0.0475655905902386
0.157230705022812 -0.456851601600647 -0.132863938808441
0.158201426267624 -0.233975455164909 -1.29469370841980
0.164843082427979 -0.201197594404221 1.15596377849579
0.177988156676292 -0.152272656559944 0.826754629611969
0.183435946702957 -0.366783618927002 -0.183630362153053
0.188220471143723 -0.269649654626846 -0.244318157434464
0.188273563981056 0.744191050529480 -1.78527390956879
0.189817219972610 -0.0105156786739826 0.591866850852966
0.193960443139076 -0.245794147253037 -0.369660794734955
0.200145587325096 -0.193330869078636 -1.04541945457459
0.209142416715622 -0.161518707871437 0.312147319316864
0.210463374853134 -0.312341511249542 -0.215669736266136
0.212070122361183 -0.247889831662178 -0.938535273075104
0.212500244379044 -0.00516866426914930 0.692332863807678
0.213932514190674 -0.382326304912567 -0.151752114295959
0.214807629585266 -0.284990489482880 0.0164594296365976
0.217539504170418 -0.370242148637772 -0.0653356760740280
0.222032532095909 -0.252106904983521 -0.839765965938568
0.222298085689545 -0.199447497725487 -0.743696689605713
0.222298085689545 -0.199447497725487 -0.738389372825623
0.223120197653770 -0.259246885776520 -0.518303036689758
0.224059239029884 -0.219154447317123 -0.731922388076782
0.228366270661354 -0.0362701974809170 0.500299334526062
0.231400310993195 -0.281632810831070 -0.595061719417572
0.235119298100472 -0.223832860589027 -0.255875170230866
0.239235371351242 -0.266498714685440 -0.676058530807495
0.248195409774780 0.318416297435761 -1.66368579864502
0.249150261282921 -0.332080185413361 -0.176096573472023
0.251027762889862 -0.278264611959457 -0.213387131690979
0.252905875444412 -0.249311119318008 -0.0108560575172305
0.254746764898300 -0.228538274765015 -0.0157361794263124
0.267780452966690 -0.315041571855545 -0.0893090441823006
0.272824734449387 -0.191129758954048 -0.211526885628700
0.285778939723969 0.0231583528220654 0.492826491594315
0.286977857351303 -0.202741548418999 -0.0718050673604012
0.289847403764725 -0.245476022362709 -0.144584536552429
0.291322171688080 -0.177904888987541 -0.321233123540878
0.292298495769501 -0.181957572698593 -0.160023182630539
0.293635338544846 0.0239753406494856 0.553192079067230
0.307257831096649 0.0313751697540283 0.540355861186981
0.308430194854736 0.0332795977592468 0.505501687526703
0.317469328641892 -0.172096326947212 -0.910953700542450
0.319580137729645 0.0496075637638569 0.512347042560577
0.320874541997910 -0.140135616064072 -1.04919373989105
0.321198403835297 0.0488278791308403 0.533883988857269
0.325930356979370 -0.0517277717590332 1.45116829872131
0.331632345914841 -0.179740279912949 -0.134934157133102
0.337764531373978 -0.168399006128311 -0.838223099708557
0.343875676393509 -0.0633241608738899 -1.53213012218475
0.345669031143189 -0.147063419222832 -1.27225828170776
0.352029412984848 0.200987175107002 1.97596061229706
0.359020859003067 -0.117261730134487 -0.773931920528412
0.359020859003067 -0.117261730134487 -0.768631875514984
0.360118597745895 -0.0870144963264465 0.227063819766045
0.360967248678207 0.0514110624790192 0.550272345542908
0.361025452613831 0.0249884724617004 0.577858805656433
0.361667960882187 -0.133780807256699 -0.763273239135742
0.361965119838715 0.0351469479501247 0.561841547489166
0.365476071834564 0.0492043718695641 0.502054572105408
0.366150081157684 0.0630866438150406 0.510752856731415
0.367725998163223 0.0446846485137939 0.414639592170715
0.371979862451553 0.0197845026850700 0.360736131668091
0.374627053737640 -0.155996680259705 1.20539724826813
0.375783562660217 -0.153869569301605 -0.548111081123352
0.379268586635590 -0.158224657177925 -0.731766164302826
0.385018706321716 -0.165030494332314 -0.683113873004913
0.385109037160873 -0.143175244331360 -0.298475503921509
0.388816595077515 0.881731152534485 2.62541174888611
0.410109728574753 -0.149832293391228 -0.273493885993958
0.416454315185547 -0.118094518780708 0.929876744747162
0.419053852558136 -0.106679096817970 -0.805121719837189
0.419074833393097 0.0421436578035355 0.636364042758942
0.419456660747528 0.0250202100723982 0.647774577140808
0.419509917497635 0.0524718016386032 0.630690515041351
0.420385032892227 0.800601005554199 -1.71100533008575
0.421652972698212 0.0558350458741188 0.493942171335220
0.423902332782745 0.0716907083988190 0.495797544717789
0.424570620059967 0.0315503329038620 0.797742426395416
0.425552159547806 0.0562914907932282 0.555313587188721
0.428490787744522 -0.105692356824875 -0.845229566097260
0.439089149236679 -0.0911946594715118 -0.962513506412506
0.474105179309845 0.0621318705379963 0.336125582456589
0.474853336811066 -0.137127891182899 -0.103212133049965
0.477315068244934 0.0603442415595055 0.380578845739365
0.477914631366730 0.0624924153089523 0.466883003711700
0.481377989053726 0.0801357030868530 0.468159794807434
0.483537673950195 0.543196737766266 -1.60204350948334
0.489197254180908 -0.0795564651489258 -0.776580691337585
0.501976311206818 0.0742370337247849 0.549954950809479
0.508649528026581 0.0449181720614433 0.664057910442352
0.513137459754944 0.0591186434030533 0.653709352016449
0.513577580451965 -0.0751322507858276 -0.480372875928879
0.513752877712250 0.0686619207262993 0.649044036865234
0.543602466583252 -0.0184836350381374 -1.20552349090576
0.567685365676880 0.113693088293076 -1.46306395530701
0.580766677856445 -0.00661153765395284 0.212208628654480
0.597317516803742 -0.0138365831226110 -0.755739450454712
0.616934120655060 0.101232483983040 0.321590214967728
0.617761254310608 0.889857470989227 -1.61020958423615
0.618159651756287 0.0964264348149300 0.647029995918274
0.618812203407288 0.0156574435532093 -0.975354671478272
0.619660854339600 0.0876417979598045 0.651539027690888
0.621182978153229 0.102229975163937 0.541571617126465
0.622339546680450 0.120790801942348 0.452769994735718
0.624561727046967 0.106224447488785 0.449432760477066
0.624710798263550 0.0790802314877510 0.659766137599945
0.626514434814453 0.0996995121240616 0.379885524511337
0.668378472328186 -0.00454827398061752 -0.505615711212158
0.674362480640411 0.0959970802068710 0.784161567687988
0.680627405643463 0.119024306535721 -1.07423388957977
0.728417158126831 0.711774110794067 -1.47775316238403
0.770552396774292 0.0437246896326542 0.911044776439667
0.780005037784576 0.209235891699791 0.508155345916748
0.784307062625885 0.188478738069534 0.540778219699860
0.786396920681000 0.196064323186874 0.504386007785797
0.787909030914307 0.184372812509537 0.565283775329590
0.791442155838013 0.172049850225449 0.568943262100220
0.799770295619965 -0.00958730466663837 -0.174502417445183
0.804729878902435 0.399599254131317 -1.30588448047638
0.811599373817444 0.173615008592606 0.588202476501465
0.814930796623230 0.193809807300568 0.438241928815842
0.815785050392151 0.239165067672730 0.551705360412598
0.819142878055573 1.03445231914520 -1.48189103603363
0.830383181571960 0.179906263947487 0.362056881189346
0.830929577350617 0.0640521720051765 1.12445175647736
0.837688326835632 0.144385874271393 -0.574904143810272
0.842764794826508 0.118494018912315 1.31940019130707
0.862110912799835 0.327339112758637 1.77278757095337
0.878969132900238 0.191300705075264 0.728563904762268
0.903498232364655 0.845484137535095 2.29874420166016
0.914138853549957 0.129670828580856 0.226587772369385
0.915460050106049 0.289580315351486 0.480834931135178
0.930403351783752 0.933652937412262 -1.29045069217682
0.933606922626495 0.283628404140472 0.559927582740784
0.946800053119659 0.150767520070076 0.828504979610443
1.00294494628906 0.299691230058670 0.629687130451202
1.00812780857086 0.745310366153717 -1.11274755001068
1.00824403762817 0.189417555928230 1.00790750980377
1.03880441188812 0.528465926647186 -0.686351001262665
1.04070341587067 0.200742736458778 -0.224685385823250
1.04203522205353 0.258871674537659 1.21425902843475
1.05915844440460 0.254935413599014 0.932691574096680
1.06813418865204 0.319657772779465 0.706707119941711
1.06953442096710 0.358604818582535 0.543326914310455
1.08673226833344 0.390692919492722 1.47615230083466
1.14382600784302 0.473790675401688 1.03894352912903
1.15639877319336 1.01279878616333 -0.797961413860321
1.16228091716766 0.331227302551270 0.219468995928764
1.17265176773071 0.493062376976013 0.635148286819458
1.18722403049469 0.822238981723785 1.83739590644836
1.21128082275391 0.626793265342712 -0.182836949825287
1.24311912059784 0.807438790798187 1.16736936569214
1.26320302486420 1.06739985942841 -0.188671588897705
1.26630091667175 0.766805052757263 0.647247254848480
1.29235732555389 0.703154087066650 0.250609993934631
[problem in code]
That global optimization solution has a value of around -8000. However, in that code it finds a solution up to about -3000 and then exits. I tried to use the trust region reflective algorithm because the formula to be optimized is non-linear, and as a result, it failed to find the global minimum.
[Question content]
I want to find the global minimum. I'd appreciate it if you could give me some feedback to help solve the problem.
how can i smooth the graph values or extract main signals only
when i try to run the code below i get this graph
my code:
from numpy import nan
import json
import os
import numpy as np
import subprocess
import math
import matplotlib.pyplot as plt
from statistics import mean, stdev
def smooth(t):
new_t = []
for i, x in enumerate(t):
neighbourhood = t[max(i-2,0): i+3]
m = mean(neighbourhood)
s = stdev(neighbourhood, xbar=m)
if abs(x - m) > s:
x = ( t[i - 1 + (i==0)*2] + t[i + 1 - (i+1==len(t))*2] ) / 2
new_t.append(x)
return new_t
def outLiersFN(*U):
outliers=[] # after preprocessing list
#preprocessing Fc =| 2*LF1 prev by 1 - LF2 prev by 2 |
c0 = -2 #(previous) by 2 #from original
c1 =-1 #(previous) #from original
c2 =0 #(current) #from original
c3 = 1 #(next) #from original
preP = U[0] # original list
if c2 == 0:
outliers.append(preP[0])
c1+=1
c2+=1
c0+=1
c3+=1
oldlen = len(preP)
M_RangeOfMotion = 90
while oldlen > c2 :
if c3 == oldlen:
outliers.insert(c2, preP[c2]) #preP[c2] >> last element in old list
break
if (preP[c2] > M_RangeOfMotion and preP[c2] < (preP[c1] + preP[c3])/2) or (preP[c2] < M_RangeOfMotion and preP[c2] > (preP[c1] + preP[c3])/2): #Check Paper 3.3.1
Equ = (preP[c1] + preP[c3])/2 #fn of preprocessing # From third index # ==== inserting current frame
formatted_float = "{:.2f}".format(Equ) #with .2 number only
equu = float(formatted_float) #from string float to float
outliers.insert(c2,equu) # insert the preprocessed value to the List
c1+=1
c2+=1
c0+=1
c3+=1
else :
Equ = preP[c2] # fn of preprocessing #put same element (do nothing)
formatted_float = "{:.2f}".format(Equ) # with .2 number only
equu = float(formatted_float) # from string float to float
outliers.insert(c2, equu) # insert the preprocessed value to the List
c1 += 1
c2 += 1
c0 += 1
c3 += 1
return outliers
def remove_nan(list):
newlist = [x for x in list if math.isnan(x) == False]
return newlist
the_angel = [176.04, 173.82, 170.09, 165.3, 171.8, 178.3, 178.77, 179.24, 179.93, 180.0, 173.39, 166.78, 166.03, 165.28, 165.72, 166.17, 166.71, 167.26, 168.04, 167.22, 166.68, 166.13, 161.53, 165.81, 170.1, 170.05, 170.5, 173.01, 176.02, 174.53, 160.09, 146.33, 146.38, 146.71, 150.33, 153.95, 154.32, 154.69, 134.52, 114.34, 115.6, 116.86, 134.99, 153.12, 152.28, 151.43, 151.36, 152.32, 158.9, 166.52, 177.74, 178.61, 179.47, 167.44, 155.4, 161.54, 167.68, 163.96, 160.24, 137.45, 114.66, 117.78, 120.89, 139.95, 139.62, 125.51, 111.79, 112.07, 112.74, 110.22, 107.7, 107.3, 106.52, 105.73, 103.07, 101.35, 102.5, 104.59, 104.6, 104.49, 104.38, 102.81, 101.25, 100.62, 100.25, 100.15, 100.32, 99.84, 99.36, 100.04, 100.31, 99.14, 98.3, 97.92, 97.41, 96.9, 96.39, 95.88, 95.9, 95.9, 96.02, 96.14, 96.39, 95.2, 94.56, 94.02, 93.88, 93.8, 93.77, 93.88, 94.04, 93.77, 93.65, 93.53, 94.2, 94.88, 92.59, 90.29, 27.01, 32.9, 38.78, 50.19, 61.59, 61.95, 62.31, 97.46, 97.38, 97.04, 96.46, 96.02, 96.1, 96.33, 95.61, 89.47, 89.34, 89.22, 89.48, 89.75, 90.02, 90.28, 88.16, 88.22, 88.29, 88.17, 88.17, 94.98, 94.84, 94.69, 94.94, 94.74, 94.54, 94.69, 94.71, 94.64, 94.58, 94.19, 94.52, 94.85, 87.7, 87.54, 87.38, 95.71, 96.57, 97.11, 97.05, 96.56, 96.07, 95.76, 95.56, 95.35, 95.28, 95.74, 96.2, 96.32, 96.33, 96.2, 96.14, 96.07, 96.07, 96.12, 96.17, 96.28, 96.31, 96.33, 96.16, 96.05, 95.94, 95.33, 88.96, 95.0, 95.78, 88.19, 88.19, 88.19, 87.92, 87.93, 88.03, 87.94, 87.86, 87.85, 87.89, 88.08, 88.01, 87.88, 88.02, 88.15, 88.15, 88.66, 88.73, 88.81, 88.41, 88.55, 88.68, 88.69, 88.02, 87.35, 95.19, 95.39, 95.38, 95.37, 95.27, 95.17, 95.33, 95.32, 95.31, 95.37, 95.42, 95.34, 95.44, 95.53, 95.47, 95.41, 95.13, 94.15, 94.78, 97.64, 97.1, 96.87, 97.03, 96.76, 35.44, 23.63, 23.27, 24.71, 26.16, 96.36, 113.13, 129.9, 96.82, 63.74, 34.25, 33.42, 32.6, 30.69, 31.06, 31.43, 97.14, 97.51, 97.23, 98.54, 100.13, 100.95, 28.82, 33.81, 66.81, 99.82, 102.63, 101.9, 101.44, 102.19, 103.22, 103.67, 104.13, 104.07, 104.73, 105.46, 103.74, 102.02, 103.32, 102.59, 29.54, 28.08, 28.76, 29.79, 30.82, 113.51, 129.34, 145.16, 143.18, 148.29, 153.67, 166.14, 161.16, 151.64, 149.27, 146.9, 151.67, 153.02, 149.28, 145.53, 149.1, 152.67, 158.78, 164.89, 164.84, 164.8, 162.11, 159.42, 156.73, 156.28, 155.83, 156.4, 161.0, 165.59, 164.44, 159.73, 155.76, 156.97, 158.92, 159.15, 159.39, 159.99, 160.44, 160.88, 163.89, 166.9, 167.71, 167.11, 167.0, 167.44, 168.38, 153.16, 137.94, 137.65, 152.09, 169.49, 171.36, 173.22, 174.01, 174.0, 174.2, 174.41, 157.74, 141.09, 149.32, 157.57, 156.4, 148.4, 140.78, 141.06, 141.73, 143.05, 143.91, 156.59, 169.29, 172.17, 175.05, 175.29, 175.27, 175.15, 175.02, 174.81, 174.59, 174.76, 174.94, 175.18, 175.41, 175.23, 174.51, 174.64, 174.77, 174.56, 173.25, 172.38, 174.17, 176.4, 177.27, 177.29, 177.33, 178.64, 179.98, 179.99, 176.0, 172.88, 173.77, 173.8, 173.97, 174.72, 175.24, 176.89, 179.07, 179.27, 178.78, 178.29, 175.61, 174.21, 172.8, 173.05, 173.41, 173.77, 174.65, 175.52, 175.58, 176.15, 176.71, 159.12, 141.54, 141.12, 155.62, 170.53, 165.54, 160.71, 158.22, 156.35, 156.82, 158.55, 160.27, 161.33, 162.39, 162.37, 159.48, 156.59, 156.77, 158.05, 159.32, 158.49, 157.66, 157.7, 157.74, 158.44, 159.14, 150.13, 143.06, 136.0, 125.7, 115.41, 111.19, 106.97, 107.1, 107.24, 107.45, 107.67, 113.34, 119.01, 144.87, 170.73, 174.31, 177.89, 174.78, 171.67, 163.26, 134.58, 105.9, 102.98, 100.77, 101.05, 101.39, 101.73, 99.79, 98.71, 97.64, 97.8, 97.89, 96.67, 95.45, 94.33, 93.38, 92.44, 48.53, 91.4, 91.35, 91.34, 91.33, 90.92, 90.51, 88.63, 87.0, 86.74, 86.48, 96.79, 96.09, 95.46, 95.39, 94.32, 93.25, 93.31, 93.37, 93.11, 92.57, 93.41, 94.25, 96.48, 92.71, 88.94, 90.07, 90.43, 78.06, 77.69, 77.32, 90.1, 89.15, 89.14, 88.85, 88.38, 87.63, 121.2, 120.66, 86.89, 86.42, 85.69, 84.86, 84.86, 85.34, 85.82, 86.07, 86.32, 85.82, 85.32, 86.23, 86.69, 87.15, 87.04, 86.87, 86.58, 86.0, 85.41, 85.41, 85.53, 85.66, 85.7, 85.72, 85.75, 85.92, 86.09, 85.77, 85.45, 84.94, 85.55, 86.16, 86.21, 86.1, 85.77, 85.27, 84.56, 84.99, 85.38, 85.42, 85.98, 86.54, 86.5, 86.45, 86.56, 86.63, 86.35, 86.08, 85.82, 85.51, 85.21, 84.6, 84.84, 84.97, 85.1, 86.12, 86.88, 86.8, 86.46, 86.47, 87.23, 87.8, 88.0, 88.08, 88.16, 87.72, 87.63, 87.37, 86.42, 86.48, 87.24, 87.97, 88.09, 88.19, 88.32, 88.44, 87.82, 87.2, 86.03, 85.78, 91.5, 93.0, 88.2, 88.52, 88.42, 87.28, 85.73, 85.62, 85.5, 85.5, 87.06, 87.6, 88.1, 88.31, 88.53, 88.77, 89.14, 89.52, 89.46, 89.4, 90.28, 89.74, 91.28, 92.17, 92.16, 92.15, 93.08, 94.0, 94.66, 95.32, 94.13, 93.7, 93.32, 93.69, 94.58, 95.47, 97.25, 99.03, 99.63, 99.67, 99.71, 100.33, 101.58, 103.36, 103.49, 103.41, 106.31, 109.34, 109.28, 109.21, 107.76, 106.31, 105.43, 104.94, 104.44, 111.19, 117.93, 115.59, 113.24, 116.15, 119.06, 125.43, 140.72, 156.0, 161.7, 143.52, 135.33, 127.13, 127.68, 148.68, 169.68, 172.2, 174.72, 174.75, 174.66, 158.57, 142.63, 145.13, 153.29, 161.45, 163.34, 165.24, 162.25, 159.89, 159.07, 156.39, 155.21, 156.04, 159.29, 160.07, 160.85, 163.45, 162.93, 161.71, 160.06, 158.4, 144.74, 132.64, 134.57, 150.22, 165.86, 172.95, 174.12, 175.3, 175.5, 176.31, 177.71, 179.72, 168.13, 156.55, 146.24, 155.75, 176.0, 175.99, 175.98, 176.0, 176.02, 176.25, 175.13, 174.26, 173.38, 173.37, 173.46, 176.34, 174.55, 172.77, 168.45, 166.35, 166.47, 168.81, 167.43, 166.79, 167.35, 168.65, 168.51, 168.37, 168.88, 169.74, 171.19, 171.33, 169.91, 168.49, 167.11, 166.83, 167.01, 168.68, 170.34, 170.43, 172.15, 173.86, 177.62, 177.61, 175.34, 173.06, 176.47, 179.87, 179.9, 177.67, 175.67, 175.39, 175.36, 177.03, 176.0, 174.98, 174.96, 174.94, 175.76, 176.57, 169.05, 162.99, 164.97, 168.74, 172.51, 167.38, 165.08, 163.03, 163.81, 164.83, 164.81, 164.8, 165.88, 165.36, 159.61, 153.86, 153.57, 153.61, 153.65, 154.62, 155.58, 157.97, 156.35, 155.66, 154.98, 156.11, 157.24, 159.25, 159.6, 160.43, 161.26, 164.71, 168.17, 147.46, 126.92, 106.38, 105.23, 104.4, 105.37, 106.65, 109.21, 107.44, 104.65, 101.86, 102.35, 102.84, 102.79, 102.19, 101.59, 100.98, 100.38, 98.72, 97.73, 97.32, 96.9, 95.11, 93.97, 94.12, 94.12, 93.1, 92.08, 89.29, 90.35, 90.35, 90.35, 90.35, 86.95, 86.37, 86.06, 85.74, 94.56, 93.16, 92.46, 91.76, 88.55, 85.33, 87.52, 92.18, 93.68, 95.18, 94.4, 92.17, 89.94, 89.4, 89.37, 99.44, 100.98, 102.52, 103.18, 88.96, 88.23, 87.5, 85.2, 85.19, 86.87, 121.42, 155.96, 155.97, 155.97, 86.2, 86.5, 86.8, 87.22, 87.36, 87.34, 87.03, 87.04, 87.05, 86.36, 85.68, 85.71, 85.84, 85.93, 86.01, 86.04, 86.08, 85.92, 86.05, 86.18, 86.17, 86.19, 86.23, 86.22, 86.09, 85.92, 85.66, 85.69, 85.69, 85.31, 84.91, 84.93, 84.95, 84.93, 84.91, 84.9, 84.9, 84.9, 84.9, 85.38, 85.52, 85.66, 85.66, 85.4, 85.14, 85.47, 85.8, 85.72, 85.64, 86.09, 85.84, 85.27, 85.47, 85.66, 85.59, 85.52, 85.38, 85.39, 85.28, 85.17, 85.39, 85.7, 85.98, 86.26, 86.61, 92.97, 93.15, 86.58, 86.58, 86.53, 86.47, 98.55, 99.41, 100.16, 100.9, 89.19, 90.28, 91.38, 91.39, 91.4, 91.44, 92.05, 131.05, 170.63, 170.13, 162.43, 125.64, 88.85, 88.85, 99.08, 100.38, 101.69, 100.74, 99.79, 96.33, 93.31, 93.73, 94.87, 96.01, 96.93, 97.85, 98.97, 97.85, 98.14, 99.37, 102.01, 103.8, 105.58, 108.52, 108.12, 107.72, 106.75, 106.82, 109.08, 112.37, 112.52, 112.66, 112.97, 114.12, 115.64, 117.1, 118.57, 126.13, 133.69, 149.27, 163.96, 166.62, 169.27, 164.94, 160.61, 149.35, 141.18, 143.41, 143.57, 149.26, 157.49, 159.94, 151.93, 147.47, 145.97, 145.56, 145.15, 143.85, 142.54, 142.18, 142.43, 143.12, 144.41, 144.38, 151.99, 159.59, 174.81, 174.94, 175.84, 176.87, 162.41, 152.94, 151.59, 155.24, 155.22, 155.19, 155.04]
p0 = outLiersFN(smooth(remove_nan(the_angel)))
the_angel = p0
plt.plot(the_angel) #list(filter(fun, L1))
plt.show()
print((the_angel))
how can i smooth the values in (the_angel) to get graph like this (red line)
i mean ignoring all unnecessary and noisy values and get only main line instead
you can edit my code or suggest me new filter or algorithm
pandas has a rolling() method for dataframes that you can use to calculate the mean over a window of values, e.g. the 70 closest ones:
import pandas as pd
import matplotlib.pyplot as plt
WINDOW_SIZE = 70
the_angel = [176.04, 173.82, 170.09, 165.3, 171.8, # ...
]
df = pd.DataFrame({'the angel': the_angel})
df[f'mean of {WINDOW_SIZE}'] = df['the angel'].rolling(
window=WINDOW_SIZE, center=True).mean()
df.plot(color=['blue', 'red']);
np.vectorize giving out nan values in ETS time series forecasting
re_1=np.vectorize(ETStest)(Trend_seasonal['trend'],Trend_seasonal['seasonal'],seasonal_period=12,i = 0)
This is the line which is showing me the error
def ets_pred (train):
trend=['add', 'add', 'mul','mul']
sea= ['add', 'mul', 'mul','add']
Trend_seasonal=pd.DataFrame()
Trend_seasonal['trend']=trend
Trend_seasonal['seasonal']=sea
re_1=np.vectorize(ETStest) (Trend_seasonal['trend'],Trend_seasonal['seasonal'],seasonal_period=12,i = 0)
result_table_ETS=pd.DataFrame(re_1).transpose()
result_table_ETS.columns=['Trend','Seasonal','seasonal_period','error_mae_f','error_mae_p','Accuracy_mae_f','Accuracy_mae_p','Diff','Product','mae_pm','mae_smly','MAE_CV'] result_table_ETS_1=result_table_ETS[~result_table_ETS.Trend.str.contains("Fals")].dropna()
result_table_ETS_1 = result_table_ETS_1.sort_values(by='MAE_CV', ascending=True).reset_index(drop=True)
return(result_table_ETS_1)
Following is the place where I am calling the function
run_results = pd.DataFrame()
forecast_results = pd.DataFrame()
model_result=[]
for i in df.columns:
ts = df[i]
n=uservalue
train, test = ts[0:len(ts)-n], ts[len(ts)-n:]
train1, test1 = ts[0:len(ts)], ts[0:len(ts)]
re = ets_pred(train)
re['Target'] = pd.DataFrame({'Target' : np.tile([i], len(re))})['Target']
re['Refresh_month'] = pd.DataFrame({'Target' : np.tile([ref_month], len(re))})['Target']
run_results=run_results.append(re)
run_results=run_results.fillna(0)
display(re)
optimal_row = 0
model_ETS= ExponentialSmoothing( endog=ts, trend=re.iloc[optimal_row,0], seasonal=re.iloc[optimal_row,1],
seasonal_periods=re.iloc[optimal_row,2],damped=True, dates=None, freq=None, missing='none').fit()
res = model_ETS.fittedvalues[-10:]
print(res)
predictions_ETS = model_ETS.predict(start=len(train),end=len(train)+n-1)
predictions = predictions_ETS
error_ETS_mae_f = re.iloc[optimal_row,3]
error_ETS_mae_p = re.iloc[optimal_row,4]
print('Summary of Best Model')
display(model_ETS.summary())
model_result.append([i,
ref_month,
re.iloc[optimal_row,0],
re.iloc[optimal_row,1],
re.iloc[optimal_row,2],
model_ETS.params['smoothing_level'],
#model_ETS.params['smoothing_trend'],
model_ETS.params['smoothing_seasonal'],
#model_ETS.params['damping_trend'],
model_ETS.params['initial_level'],
#model_ETS.params['initial_trend'],
re.iloc[optimal_row,5],
re.iloc[optimal_row,6],
re.iloc[optimal_row,7],
re.iloc[optimal_row,8]])
forecast=pd.DataFrame()
forecast['Date'] = model_ETS.predict(start=len(train), end=len(train)+20).index
forecast['Forecast'] = model_ETS.predict(start=len(train), end=len(train)+20).values
forecast['Target'] = pd.DataFrame({'Target' : np.tile([i], len(forecast))})['Target']
forecast['Refresh_month'] = pd.DataFrame({'Target' : np.tile([ref_month], len(forecast))})['Target']
forecast_results=forecast_results.append(forecast)
display(forecast)
I think that np.vectorize is giving out some nan values and that is why I am getting the error. The above is the function for the same
max curve time error in Heston calibration quantlib python
I am running a compiled from source SWIG python 1.16 version of QuantLib.
I have been trying to calibrate a heston model following this example.
I am only using the QL calibration at the moment to test it out before trying others.
I need time dependent parameters so I am using PiecewiseTimeDependentHestonModel.
Here is the relevant portion of my code.
Helper functions :
def tenor2date(s, base_date=None,ql=False):
# returns a date from a tenor and a base date
if base_date is None:
base_date = datetime.today()
num = float(s[:-1])
period = s[-1].upper()
if period == "Y":
return_date = base_date + relativedelta(years=num)
elif period == "M":
return_date = base_date + relativedelta(months=num)
elif period == "W":
return_date = base_date + relativedelta(weeks=num)
elif period == "D":
return_date = base_date + relativedelta(days=num)
else:
return_date = base_date
if ql:
return Date(return_date.strftime("%F"),"yyyy-mm-dd")
else:
return return_date
def setup_model(yield_ts, dividend_ts, spot, times,init_condition=(0.02, 0.2, 0.5, 0.1, 0.01)):
theta, kappa, sigma, rho, v0 = init_condition
model = ql.PiecewiseTimeDependentHestonModel(yield_ts, dividend_ts, ql.QuoteHandle(ql.SimpleQuote(spot)), v0, ql.Parameter(), ql.Parameter(),
ql.Parameter(), ql.Parameter(), ql.TimeGrid(times))
engine = ql.AnalyticPTDHestonEngine(model)
return model, engine
def setup_helpers(engine, vol_surface, ref_date, spot, yield_ts, dividend_ts):
heston_helpers = []
grid_data = []
for tenor in vol_surface:
expiry_date = tenor2date(tenor, datetime(ref_date.year(), ref_date.month(), ref_date.dayOfMonth()), True)
t = (expiry_date - ref_date)
print(f"{tenor} : {t / 365}")
p = ql.Period(t, ql.Days)
for strike, vol in zip(vol_surface[tenor]["strikes"], vol_surface[tenor]["volatilities"]):
print((strike, vol))
helper = ql.HestonModelHelper(p, calendar, spot, strike, ql.QuoteHandle(ql.SimpleQuote(vol / 100)), yield_ts, dividend_ts)
helper.setPricingEngine(engine)
heston_helpers.append(helper)
grid_data.append((expiry_date, strike))
return heston_helpers, grid_data
Market data :
vol_surface = {'12M': {'strikes': [1.0030154025220293, 0.9840808634190958, 0.9589657270688433, 0.9408279805370683, 0.9174122318462831, 0.8963792435025802, 0.8787138822765832, 0.8538712672800733, 0.8355036501980958], 'volatilities': [6.7175, 6.5, 6.24375, 6.145, 6.195, 6.425, 6.72125, 7.21, 7.5625], 'forward': 0.919323}, '1M': {'strikes': [0.9369864196692815, 0.9324482223892986, 0.9261255003380027, 0.9213195223581382, 0.9150244003650484, 0.9088253068972495, 0.9038936313900919, 0.897245676067657, 0.8924388848562849], 'volatilities': [6.3475, 6.23375, 6.1075, 6.06, 6.09, 6.215, 6.3725, 6.63125, 6.8225], 'forward': 0.915169}, '1W': {'strikes': [0.9258809998009043, 0.9236526412979602, 0.920487656155217, 0.9180490618315417, 0.9148370595017086, 0.9116231311263782, 0.9090950947170667, 0.9057357691404444, 0.9033397443834199], 'volatilities': [6.7175, 6.63375, 6.53625, 6.5025, 6.53, 6.6425, 6.77875, 6.99625, 7.1525], 'forward': 0.914875}, '2M': {'strikes': [0.9456173410343232, 0.9392447942175677, 0.9304717860942596, 0.9238709412876663, 0.9152350197527926, 0.9068086964842931, 0.9000335970840222, 0.8908167643473346, 0.884110721680849], 'volatilities': [6.1575, 6.02625, 5.8825, 5.8325, 5.87, 6.0175, 6.1975, 6.48875, 6.7025], 'forward': 0.915506}, '3M': {'strikes': [0.9533543407827232, 0.945357456067501, 0.9343646071178692, 0.9261489737826977, 0.9154251386183144, 0.9050707394248945, 0.8966770979707913, 0.8851907303568785, 0.876803402158318], 'volatilities': [6.23, 6.09125, 5.93, 5.8725, 5.915, 6.0775, 6.28, 6.60375, 6.84], 'forward': 0.915841}, '4M': {'strikes': [0.9603950279333742, 0.9509237742916833, 0.9379657828957041, 0.928295643018581, 0.9156834006905108, 0.9036539552069216, 0.8938804229269658, 0.8804999196762403, 0.870730837142799], 'volatilities': [6.3175, 6.17125, 6.005, 5.94375, 5.985, 6.15125, 6.36, 6.69375, 6.9375], 'forward': 0.916255}, '6M': {'strikes': [0.9719887962018352, 0.9599837798239937, 0.943700651576822, 0.9316544554849711, 0.9159768970939797, 0.9013018796367052, 0.8892904835162911, 0.8727031923006017, 0.8605425787295339], 'volatilities': [6.3925, 6.22875, 6.04125, 5.9725, 6.01, 6.1875, 6.41375, 6.78625, 7.0575], 'forward': 0.916851}, '9M': {'strikes': [0.9879332225745909, 0.9724112749400833, 0.951642771321364, 0.936450663789222, 0.9167103888580063, 0.8985852649047051, 0.8835274087791912, 0.8625837214139542, 0.8472311260811375], 'volatilities': [6.54, 6.34875, 6.1325, 6.055, 6.11, 6.32, 6.5875, 7.01625, 7.32], 'forward': 0.918086}}
spotDates = [ql.Date(1,7,2019), ql.Date(8,7,2019), ql.Date(1,8,2019), ql.Date(1,9,2019), ql.Date(1,10,2019), ql.Date(1,11,2019), ql.Date(1,1,2020), ql.Date(1,4,2020), ql.Date(1,7,2020)]
spotRates = [0.9148, 0.914875, 0.915169, 0.915506, 0.915841, 0.916255, 0.916851, 0.918086, 0.919323]
udl_value = 0.9148
todaysDate = ql.Date("2019-07-01","yyyy-mm-dd")
settlementDate = ql.Date("2019-07-03","yyyy-mm-dd")
and the script itself:
ql.Settings.instance().evaluationDate = todaysDate
dayCounter = ql.Actual365Fixed()
interpolation = ql.Linear()
compounding = ql.Compounded
compoundingFrequency = ql.Annual
times = [(x - spotDates[0]) / 365 for x in spotDates][1:]
discountFactors = [-log(x / spotRates[0]) / (times[i]) for i, x in enumerate(spotRates[1:])]
fwdCurve = ql.ZeroCurve(spotDates, [0] + discountFactors, dayCounter, calendar, interpolation, compounding, compoundingFrequency)
fwdCurveHandle = ql.YieldTermStructureHandle(fwdCurve)
dividendCurveHandle = ql.YieldTermStructureHandle(ql.FlatForward(settlementDate, 0, dayCounter))
hestonModel, hestonEngine = setup_model(fwdCurveHandle, dividendCurveHandle, udl_value, times)
heston_helpers, grid_data = setup_helpers(hestonEngine, vol_surface, todaysDate, udl_value, fwdCurveHandle, dividendCurveHandle)
lm = ql.LevenbergMarquardt(1e-8, 1e-8, 1e-8)
hestonModel.calibrate(heston_helpers, lm, ql.EndCriteria(500, 300, 1.0e-8, 1.0e-8, 1.0e-8))
When I run the last line I get the following error message :
RuntimeError: time (1.42466) is past max curve time (1.00274)
I do not understand how it can try to price things beyond 1Y as both the helpers and the forwards curve are defined on the same set of dates.
In case it helps someone, posting here the answer I got from the quantlb mailing :
specifying the maturity in days
t = (expiry_date - ref_date)
print(f"{tenor} : {t / 365}")
p = ql.Period(t, ql.Days)
might have an counterintuitive effect here as the specified calendar is used
to calculate the real expiry date. If the calendar is e.g. ql.UnitedStates
then this takes weekends and holidays into consideration,
ql.UnitedStates().advance(ql.Date(1,1,2019),ql.Period(365, ql.Days)) => Date(12,6,2020)
whereas
ql.NullCalendar().advance(ql.Date(1,1,2019),ql.Period(365, ql.Days)) => Date(1,1,2020)
hence I guess the interest rate curve is not long enough and throws the
error message.
So the fix is to make sure to use ql.NullCalendar() accross.
Duplicate Labels & No Line Appearing in Plot in Python
I'm plotting two lines: a set of experimental data points and a mathematical model. I'm getting the experimental data to plot as expected, but the mathematical model will not plot a line (only the symbol). Additionally, I'm getting duplicate legends and despite trying some of the suggestion I've seen on the site I'm not getting to work (probably I'm not executing it correctly).
import numpy as np
from sympy import *
from sympy import Matrix
import matplotlib.pyplot as plt
Stretch = [0.998122066, 1.0157277, 1.034507042, 1.052112676, 1.06971831, 1.088497653, 1.106103286, 1.12370892, 1.143661972, 1.160093897, 1.178873239, 1.196478873, 1.214084507, 1.23286385, 1.249295775, 1.266901408, 1.28685446, 1.303286385, 1.322065728, 1.339671362, 1.357276995, 1.374882629, 1.393661972, 1.411267606, 1.430046948, 1.447652582, 1.464084507, 1.48286385, 1.500469484, 1.518075117, 1.535680751, 1.554460094, 1.572065728, 1.59084507, 1.608450704, 1.626056338, 1.643661972, 1.661267606, 1.680046948, 1.697652582, 1.715258216, 1.734037559, 1.751643192, 1.770422535, 1.78685446, 1.805633803, 1.824413146, 1.844366197, 1.860798122, 1.878403756, 1.894835681, 1.912441315, 1.930046948, 1.948826291, 1.967605634, 1.985211268, 2.00399061, 2.021596244, 2.038028169, 2.057981221, 2.075586854, 2.092018779, 2.110798122, 2.128403756, 2.147183099, 2.165962441, 2.183568075, 2.201173709, 2.218779343, 2.237558685, 2.255164319, 2.272769953, 2.291549296, 2.307981221, 2.326760563, 2.344366197, 2.361971831, 2.380751174, 2.398356808, 2.415962441, 2.434741784, 2.452347418, 2.469953052, 2.488732394, 2.505164319]
Stress = [0.010526316, 0.010549481, 0.01188998, 0.011913146, 0.012594206, 0.012618915, 0.013299975, 0.013323141, 0.014665184, 0.0153447, 0.016027304, 0.016708364, 0.017389424, 0.018729923, 0.018751544, 0.019432604, 0.019458858, 0.019480479, 0.020163084, 0.020844144, 0.020867309, 0.021548369, 0.022230974, 0.022254139, 0.022278849, 0.023617803, 0.024297319, 0.024979923, 0.025660983, 0.026999938, 0.027023104, 0.027705708, 0.029044663, 0.029069372, 0.030408327, 0.031747282, 0.033086237, 0.034425191, 0.035107796, 0.036446751, 0.037785705, 0.039784099, 0.041123054, 0.042463553, 0.044458858, 0.046457252, 0.048455646, 0.051113479, 0.053108784, 0.055763529, 0.059074623, 0.061729367, 0.065042006, 0.069014085, 0.072986163, 0.077614591, 0.081586669, 0.086872992, 0.092815666, 0.099420867, 0.106680875, 0.114597233, 0.123174574, 0.132408265, 0.142301396, 0.152852422, 0.164059797, 0.177240857, 0.191079812, 0.206236101, 0.22073295, 0.238519274, 0.256307141, 0.273434025, 0.293195577, 0.314929269, 0.335347171, 0.357740301, 0.382105572, 0.406470843, 0.434785026, 0.461123981, 0.488778725, 0.516435014, 0.544088213]
c= [6.11739377e+00, 4.78409591e-04]
plt.show()
for o, d in zip (Stress, Stretch):
d1 = d2 = d
d3 = 1/(d1*d2)
d3 = 1/(d1*d2)
C11 = d1**2
C22 = d2**2
C33 = d3**2
p = 4*(c[1]/c[0])*(d3**(c[0]-2))
S11 = -p/C11 + 4*C11*(c[1]/c[0])*(d1**(c[0]-2))
S22 = -p/C22 + 4*C22*(c[1]/c[0])*(d2**(c[0]-2))
T112 = (d1*S11)/(d2*d3)
T222 = (d2*S22)/(d1*d3)
plt.plot(d, T112, 'g--^', label = 'Model')
plt.plot(Stretch, Stress, 'b-o', label = 'Experimental')
plt.subplots_adjust(left=0.15)
plt.grid(True)
plt.ylabel('Stress')
plt.xlabel('Applied Stretch')
plt.title('Stress as a Function of Applied Stretch')
plt.legend()
plt.show()
plt.plot should not be included in a loop over the individual data because it work with 2 lists or 2 arrays.As a first step, i've created 2 lists for your model data, then i was able to plot it :
import matplotlib.pyplot as plt
Stretch = [0.998122066, 1.0157277, 1.034507042, 1.052112676, 1.06971831, 1.088497653, 1.106103286, 1.12370892, 1.143661972, 1.160093897, 1.178873239, 1.196478873, 1.214084507, 1.23286385, 1.249295775, 1.266901408, 1.28685446, 1.303286385, 1.322065728, 1.339671362, 1.357276995, 1.374882629, 1.393661972, 1.411267606, 1.430046948, 1.447652582, 1.464084507, 1.48286385, 1.500469484, 1.518075117, 1.535680751, 1.554460094, 1.572065728, 1.59084507, 1.608450704, 1.626056338, 1.643661972, 1.661267606, 1.680046948, 1.697652582, 1.715258216, 1.734037559, 1.751643192, 1.770422535, 1.78685446, 1.805633803, 1.824413146, 1.844366197, 1.860798122, 1.878403756, 1.894835681, 1.912441315, 1.930046948, 1.948826291, 1.967605634, 1.985211268, 2.00399061, 2.021596244, 2.038028169, 2.057981221, 2.075586854, 2.092018779, 2.110798122, 2.128403756, 2.147183099, 2.165962441, 2.183568075, 2.201173709, 2.218779343, 2.237558685, 2.255164319, 2.272769953, 2.291549296, 2.307981221, 2.326760563, 2.344366197, 2.361971831, 2.380751174, 2.398356808, 2.415962441, 2.434741784, 2.452347418, 2.469953052, 2.488732394, 2.505164319]
Stress = [0.010526316, 0.010549481, 0.01188998, 0.011913146, 0.012594206, 0.012618915, 0.013299975, 0.013323141, 0.014665184, 0.0153447, 0.016027304, 0.016708364, 0.017389424, 0.018729923, 0.018751544, 0.019432604, 0.019458858, 0.019480479, 0.020163084, 0.020844144, 0.020867309, 0.021548369, 0.022230974, 0.022254139, 0.022278849, 0.023617803, 0.024297319, 0.024979923, 0.025660983, 0.026999938, 0.027023104, 0.027705708, 0.029044663, 0.029069372, 0.030408327, 0.031747282, 0.033086237, 0.034425191, 0.035107796, 0.036446751, 0.037785705, 0.039784099, 0.041123054, 0.042463553, 0.044458858, 0.046457252, 0.048455646, 0.051113479, 0.053108784, 0.055763529, 0.059074623, 0.061729367, 0.065042006, 0.069014085, 0.072986163, 0.077614591, 0.081586669, 0.086872992, 0.092815666, 0.099420867, 0.106680875, 0.114597233, 0.123174574, 0.132408265, 0.142301396, 0.152852422, 0.164059797, 0.177240857, 0.191079812, 0.206236101, 0.22073295, 0.238519274, 0.256307141, 0.273434025, 0.293195577, 0.314929269, 0.335347171, 0.357740301, 0.382105572, 0.406470843, 0.434785026, 0.461123981, 0.488778725, 0.516435014, 0.544088213]
c= [6.11739377e+00, 4.78409591e-04]
Stretch_mod=[]
Stress_mod=[]
for o, d in zip (Stress, Stretch):
d1 = d2 = d
d3 = 1/(d1*d2)
d3 = 1/(d1*d2)
C11 = d1**2
C22 = d2**2
C33 = d3**2
p = 4*(c[1]/c[0])*(d3**(c[0]-2))
S11 = -p/C11 + 4*C11*(c[1]/c[0])*(d1**(c[0]-2))
S22 = -p/C22 + 4*C22*(c[1]/c[0])*(d2**(c[0]-2))
T112 = (d1*S11)/(d2*d3)
T222 = (d2*S22)/(d1*d3)
Stretch_mod.append(d)
Stress_mod.append(T112)
plt.plot(Stretch_mod, Stress_mod, 'g--^', label = 'Model')
plt.plot(Stretch, Stress, 'b-o', label = 'Experimental')
plt.subplots_adjust(left=0.15)
plt.grid(True)
plt.ylabel('Stress')
plt.xlabel('Applied Stretch')
plt.title('Stress as a Function of Applied Stretch')
plt.legend()
plt.show()