Develop Reference

Conjugate gradient with tensorflow and sparse tensor runs slower than scipy with sparse matrices

I implemented the conjugate gradient method using TensorFlow to invert a sparse matrix.
The matrix I used to test the method is well-conditioned, as it is the sum of a mass matrix and a stiffness matrix obtained with finite elements.
I compared with the same method implemented using scipy and on the same data.
The solutions obtained with either methods are the same, however, TensorFlow is 5 times slower (I tested under colab environment).
Under colab environment, scipy ran in 0.27 s, while TensorFlow required 1.37 s
Why the algorithm is so slow under TensorFlow?
I can not cast to dense matrices, as I want to use the formula with matrices of large size (100k X100k or more).
Thanks,
Cesare
Here is the code I used to test this:
import tensorflow as tf
import numpy as np
from scipy.sparse import coo_matrix,linalg
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
from time import time
from scipy.spatial import Delaunay
def create_mesh(Lx=1,Ly=1,Nx=100,Ny=100):
mesh0=dict()
dx = Lx/Nx
dy = Ly/Ny
XX,YY=np.meshgrid(np.arange(0,Lx+dx,dx),np.arange(0,Ly+dy,dy))
points=np.vstack((XX.ravel(),YY.ravel())).T
#np.random.shuffle(points)
tri = Delaunay(points)
mesh0['Pts']=np.copy(points).astype(np.float32)
mesh0['Tria']=np.copy(tri.simplices).astype(int)
return(mesh0)
def eval_connectivity(mesh0):
print('computing mesh connectivity')
npt=mesh0['Pts'].shape[0]
connectivity = {}
for jpt in range(npt):
connectivity[jpt] = []
for Tria in mesh0['Tria']:
for ilpt in range(3):
iglobalPt=Tria[ilpt]
for jlpt in range(1+ilpt,3):
jglobalPt=Tria[jlpt]
connectivity[iglobalPt].append(jglobalPt)
connectivity[jglobalPt].append(iglobalPt)
for key,value in connectivity.items():
connectivity[key]=np.unique(np.array(value,dtype=int))
return(connectivity)
def eval_local_mass(mesh0,iTri):
lmass = np.zeros(shape=(3,3),dtype=np.float32)
Tria=mesh0['Tria'][iTri]
v10 = mesh0['Pts'][Tria[1],:]-mesh0['Pts'][Tria[0],:]
v20 = mesh0['Pts'][Tria[2],:]-mesh0['Pts'][Tria[0],:]
N12 = np.cross(v10,v20)
Tsurf = 0.5*np.linalg.norm(N12)
for ipt in range(3):
lmass[ipt,ipt]=1.0/12.0
for jpt in range(1+ipt,3):
lmass[ipt,jpt] = 1.0/24.0
lmass[jpt,ipt] = lmass[ipt,jpt]
lmass = 2.0*Tsurf*lmass
return(lmass)
def eval_local_stiffness(mesh0,iTri):
Tria = mesh0['Tria'][iTri]
v10 = mesh0['Pts'][Tria[1],:]-mesh0['Pts'][Tria[0],:]
v20 = mesh0['Pts'][Tria[2],:]-mesh0['Pts'][Tria[0],:]
N12 = np.cross(v10,v20)
Tsurf = 0.5*np.linalg.norm(N12)
covbT = np.zeros(shape=(3,3),dtype=np.float32)
covbT[0,:2] = v10
covbT[1,:2] = v20
covbT[2,2] = N12/(2*Tsurf)
contrb = np.linalg.inv(covbT)
v1 = contrb[:,0]
v2 = contrb[:,1]
a = np.dot(v1,v1)
b = np.dot(v1,v2)
c = np.dot(v2,v2)
gij_c = np.array([[a,b],[b,c]],dtype=np.float32)
lgrad = np.array([[-1.0,1.0,0.0], [-1.0,0.0,1.0] ],dtype=np.float32)
lstif = Tsurf*np.matmul( np.matmul(lgrad.T,gij_c), lgrad )
return(lstif)
def compute_vectors_sparse_matrices(mesh0):
npt = mesh0['Pts'].shape[0]
connect = eval_connectivity(mesh0)
nzero = 0
for key,value in connect.items():
nzero += (1+value.shape[0])
I = np.zeros(shape=(nzero),dtype=int)
J = np.zeros(shape=(nzero),dtype=int)
VM = np.zeros(shape=(nzero),dtype=np.float32)
VS = np.zeros(shape=(nzero),dtype=np.float32)
k0 = np.zeros(shape=(npt+1),dtype=int)
k0[0] = 0
k = -1
for jpt in range(npt):
loc_con = connect[jpt].tolist()[:]
loc_con.append(jpt)
loc_con = np.sort(loc_con)
k0[jpt+1]=k0[jpt]+loc_con.shape[0]
for jloc in range(loc_con.shape[0]):
k=k+1
I[k]= jpt
J[k]= loc_con[jloc]
for iTr, Tria in enumerate(mesh0['Tria']):
lstiff = eval_local_stiffness(mesh0,iTr)
lmass = eval_local_mass(mesh0,iTr)
for iEntry,irow in enumerate(Tria):
loc_con = connect[irow].tolist()[:]
loc_con.append(irow)
loc_con = np.sort(loc_con)
for jEntry,jcol in enumerate(Tria):
indexEntry = k0[irow]+np.where(loc_con==jcol)[0]
VM[indexEntry] = VM[indexEntry]+lmass[iEntry,jEntry]
VS[indexEntry] = VS[indexEntry]+lstiff[iEntry,jEntry]
return(I,J,VM,VS)
def compute_global_sparse_matrices(mesh0):
I,J,VM,VS = compute_vectors_sparse_matrices(mesh0)
npt = mesh0['Pts'].shape[0]
MASS = coo_matrix((VM,(I,J)),shape=(npt,npt))
STIFF = coo_matrix((VS,(I,J)),shape=(npt,npt))
return(MASS,STIFF)
def compute_global_sparse_tensors(mesh0):
I,J,VM,VS = compute_vectors_sparse_matrices(mesh0)
npt = mesh0['Pts'].shape[0]
indices = np.hstack([I[:,np.newaxis], J[:,np.newaxis]])
MASS = tf.sparse.SparseTensor(indices=indices, values=VM.astype(np.float32), dense_shape=[npt, npt])
STIFF = tf.sparse.SparseTensor(indices=indices, values=VS.astype(np.float32), dense_shape=[npt, npt])
return(MASS,STIFF)
def compute_matrices_scipy(mesh0):
MASS,STIFF = compute_global_sparse_matrices(mesh0)
return(MASS,STIFF)
def compute_matrices_tensorflow(mesh0):
MASS,STIFF = compute_global_sparse_tensors(mesh0)
return(MASS,STIFF)
def conjgrad_scipy(A,b,x0,niter=100,toll=1.e-5):
x = np.copy(x0)
r = b - A * x
p = np.copy(r)
rsold = np.dot(r,r)
for it in range(niter):
Ap = A * p
alpha = rsold /np.dot(p,Ap)
x += alpha * p
r -= alpha * Ap
rsnew = np.dot(r,r)
if (np.sqrt(rsnew) < toll):
break
p = r + (rsnew / rsold) * p
rsold = rsnew
return([x,it,np.sqrt(rsnew)])
def conjgrad_tensorflow(A,b,x0,niter=100,toll=1.e-5):
x = x0
r = b - tf.sparse.sparse_dense_matmul(A,x)
p = r
rsold = tf.reduce_sum(tf.multiply(r, r))
for it in range(niter):
Ap = tf.sparse.sparse_dense_matmul(A,p)
alpha = rsold /tf.reduce_sum(tf.multiply(p, Ap))
x += alpha * p
r -= alpha * Ap
rsnew = tf.reduce_sum(tf.multiply(r, r))
if (tf.sqrt(rsnew) < toll):
break
p = r + (rsnew / rsold) * p
rsold = rsnew
return([x,it,tf.sqrt(rsnew)])
mesh = create_mesh(Lx=10,Ly=10,Nx=100,Ny=100)
x0 = tf.constant( (mesh['Pts'][:,0]<5 ).astype(np.float32) )
nit_time = 10
dcoef = 1.0
maxit = x0.shape[0]//2
stoll = 1.e-6
print('nb of nodes:\t{}'.format(mesh['Pts'].shape[0]))
print('nb of trias:\t{}'.format(mesh['Tria'].shape[0]))
t0 = time()
MASS0,STIFF0 = compute_matrices_scipy(mesh)
elapsed_scipy=time()-t0
print('Matrices; elapsed: {:3.5f} s'.format(elapsed_scipy))
A = MASS0+dcoef*STIFF0
x = np.copy(np.squeeze(x0.numpy()) )
t0 = time()
for jt in range(nit_time):
b = MASS0*x
x1,it,tol=conjgrad_scipy(A,b,x,niter=maxit,toll=stoll)
x=np.copy(x1)
print('time {}; iters {}; resid: {:3.2f}'.format(1+jt,it,tol) )
elapsed_scipy=time()-t0
print('elapsed, scipy: {:3.5f} s'.format(elapsed_scipy))
t0 = time()
MASS,STIFF =compute_matrices_tensorflow(mesh)
elapsed=time()-t0
print('Matrices; elapsed: {:3.5f} s'.format(elapsed))
x = None
x1 = None
A = tf.sparse.add(MASS,tf.sparse.map_values(tf.multiply, STIFF, dcoef))
x = tf.expand_dims(tf.identity(x0),axis=1)
t0 = time()
for jt in range(nit_time):
b = tf.sparse.sparse_dense_matmul(MASS,x)
x1,it,tol=conjgrad_tensorflow(A,b,x,niter=maxit,toll=stoll)
x = x1
print('time {}; iters {}; resid: {:3.2f}'.format(1+jt,it,tol) )
elapsed_tf=time()-t0
print('elapsed, tf: {:3.2f} s'.format(elapsed_tf))
print('elapsed times:')
print('scipy: {:3.2f} s\ttf: {:3.2f} s'.format(elapsed_scipy,elapsed_tf))

Run out of memory issue in Gekko python

Now, I'm trying to solve the optimization problem as above and I made it in a code as belows.
However for large N (such as 300), the code failed to work because of run out of the memory.
So, I tried m = GEKKO(remote=True) instead of m = GEKKO(remote=False), but it was not done even after 10-12 hours.
I'm now finding a way to solve this problem while using m = GEKKO(remote=False).
Code
# Import package
from gekko import GEKKO
import numpy as np
# Define parameters
P_CO = 600 # $/tonCO
beta_CO2 = 1 # no unit
P_CO2 = 60 # $/tonCO2eq
E_ref = 3.1022616 # tonCO2eq/tonCO
E_dir = -1.600570692 # tonCO2eq/tonCO
E_indir_others = 0.3339226804 # tonCO2eq/tonCO
E_indir_elec_cons = 18.46607256 # GJ/tonCO
C1_CAPEX = 285695 # no unit
C2_CAPEX = 188.42 # no unit
C1_FOX = 82282 # no unit
C2_FOX = 24.094 # no unit
C1_ROX = 4471.5 # no unit
C2_ROX = 96.034 # no unit
C1_UOX = 1983.7 # no unit
C2_UOX = 249.79 # no unit
r = 0.08 # discount rate
N = 300 # number of scenarios
T = 30 # total time period
GWP_init = 0.338723235 # 2020 Electricity GWP in EU 27 countries
theta_max = 1600000 # Max capacity
# Function to make GWP_EU matrix (TxN matrix)
def Electricity_GWP(GWP_init, n_years, num_episodes):
GWP_mean = 0.36258224*np.exp(-0.16395611*np.arange(1, n_years+2)) + 0.03091272
GWP_mean = GWP_mean.reshape(-1,1)
GWP_Yearly = np.tile(GWP_mean, num_episodes)
noise = np.zeros((n_years+1, num_episodes))
stdev2050 = GWP_mean[-1] * 0.25
stdev = np.arange(0, stdev2050 * (1 + 1/n_years), stdev2050/n_years)
for i in range(n_years+1):
noise[i,:] = np.random.normal(0, stdev[i], num_episodes)
GWP_forecast = GWP_Yearly + noise
return GWP_forecast
GWP_EU = Electricity_GWP(GWP_init, T, N) # (T+1)*N matrix
GWP_EU = GWP_EU[1:,:] # T*N matrix
print(np.shape(GWP_EU))
# Build Gekko model
m = GEKKO(remote=False)
theta = m.Array(m.Var, N, lb=0, ub=theta_max)
demand = np.ones((T,1))
demand[0] = 8031887.589
for k in range(1,11):
demand[k] = demand[k-1] * 1.026
for k in range(11,21):
demand[k] = demand[k-1] * 1.016
for k in range(21,T):
demand[k] = demand[k-1] * 1.011
demand = 0.12 * demand
demand = np.tile(demand, N) # T*N matrix
print(np.shape(demand))
obj = m.sum([m.sum([((1/(1+r))**(t+1))*((P_CO*m.min3(demand[t,s], theta[s])) \
+ (beta_CO2*P_CO2*m.min3(demand[t,s], theta[s])*(E_ref-E_dir-E_indir_others-E_indir_elec_cons*GWP_EU[t,s])) \
- (C1_CAPEX+C2_CAPEX*theta[s]+C1_FOX+C2_FOX*theta[s])-(C1_ROX+C2_ROX*m.min3(demand[t,s], theta[s])+C1_UOX+C2_UOX*m.min3(demand[t,s], theta[s]))) for t in range(T)]) for s in range(N)])
m.Maximize(obj/N)
m.solve()

The multiple m.min3() expressions lead to many additional variables. Try defining it once and substituting the value m3 into the objective expression.
m3 = [[m.min3(demand[t,s],theta[s]) for t in range(T)] for s in range(N)]
obj = m.sum([sum([((1/(1+r))**(t+1))*((P_CO*m3[s][t]) \
+ (beta_CO2*P_CO2*m3[s][t]\
*(E_ref-E_dir-E_indir_others-E_indir_elec_cons*GWP_EU[t,s])) \
- (C1_CAPEX+C2_CAPEX*theta[s]+C1_FOX+C2_FOX*theta[s])\
- (C1_ROX+C2_ROX*m3[s][t]\
+C1_UOX+C2_UOX*m3[s][t])) \
for t in range(T)]) for s in range(N)])
Below is a test script that increases the size of N from 10 to 100.
It shows the compile time and solution time for each of the cases. The number of variables for N=10 to N=100 is shown as the x-axis.
# Import package
from gekko import GEKKO
import numpy as np
import time
import matplotlib.pyplot as plt
# Define parameters
P_CO = 600 # $/tonCO
beta_CO2 = 1 # no unit
P_CO2 = 60 # $/tonCO2eq
E_ref = 3.1022616 # tonCO2eq/tonCO
E_dir = -1.600570692 # tonCO2eq/tonCO
E_indir_others = 0.3339226804 # tonCO2eq/tonCO
E_indir_elec_cons = 18.46607256 # GJ/tonCO
C1_CAPEX = 285695 # no unit
C2_CAPEX = 188.42 # no unit
C1_FOX = 82282 # no unit
C2_FOX = 24.094 # no unit
C1_ROX = 4471.5 # no unit
C2_ROX = 96.034 # no unit
C1_UOX = 1983.7 # no unit
C2_UOX = 249.79 # no unit
r = 0.08 # discount rate
T = 30 # total time period
GWP_init = 0.338723235 # 2020 Electricity GWP in EU 27 countries
theta_max = 1600000 # Max capacity
# Function to make GWP_EU matrix (TxN matrix)
def Electricity_GWP(GWP_init, n_years, num_episodes):
GWP_mean = 0.36258224*np.exp(-0.16395611*np.arange(1, n_years+2)) + 0.03091272
GWP_mean = GWP_mean.reshape(-1,1)
GWP_Yearly = np.tile(GWP_mean, num_episodes)
noise = np.zeros((n_years+1, num_episodes))
stdev2050 = GWP_mean[-1] * 0.25
stdev = np.arange(0, stdev2050 * (1 + 1/n_years), stdev2050/n_years)
for i in range(n_years+1):
noise[i,:] = np.random.normal(0, stdev[i], num_episodes)
GWP_forecast = GWP_Yearly + noise
return GWP_forecast
Nx = np.array([10,20,30,40,60,80,100])
tv = 122*Nx+1
tt = np.zeros_like(Nx)
tsolve = np.zeros_like(Nx)
for i,N in enumerate(Nx):
# N=number of scenarios
print(N)
ts = time.time()
GWP_EU = Electricity_GWP(GWP_init, T, N) # (T+1)*N matrix
GWP_EU = GWP_EU[1:,:] # T*N matrix
# Build Gekko model
m = GEKKO(remote=False)
theta = m.Array(m.Var, N, lb=0, ub=theta_max)
demand = np.ones((T,1))
demand[0] = 8031887.589
for k in range(1,11):
demand[k] = demand[k-1] * 1.026
for k in range(11,21):
demand[k] = demand[k-1] * 1.016
for k in range(21,T):
demand[k] = demand[k-1] * 1.011
demand = 0.12 * demand
demand = np.tile(demand, N) # T*N matrix
m3 = [[m.min3(demand[t,s],theta[s]) for t in range(T)] for s in range(N)]
obj = m.sum([sum([((1/(1+r))**(t+1))*((P_CO*m3[s][t]) \
+ (beta_CO2*P_CO2*m3[s][t]\
*(E_ref-E_dir-E_indir_others-E_indir_elec_cons*GWP_EU[t,s])) \
- (C1_CAPEX+C2_CAPEX*theta[s]+C1_FOX+C2_FOX*theta[s])\
- (C1_ROX+C2_ROX*m3[s][t]\
+C1_UOX+C2_UOX*m3[s][t])) \
for t in range(T)]) for s in range(N)])
m.Maximize(obj/N)
m.solve(disp=True)
tsolve[i] = m.options.SOLVETIME
tt[i] = time.time()-ts-tsolve[i]
plt.figure(figsize=(8,5))
plt.plot(tv,tt,'ro-',label='Compile time')
plt.plot(tv,tsolve,'b--',label='Solve time')
plt.legend(); plt.grid()
plt.ylabel('Time (sec)'); plt.xlabel('Problem Size')
plt.savefig('Solution_time.png',dpi=300)
plt.show()
One other thing that improved the solution time is to use sum() instead of m.sum() for the inner summation. It is faster to solve this way, but other problems are faster with m.sum().

solve_ivp error : Required step size is less than spacing between numbers

I have been trying to implement a model of unstable glacier flow in Python, solving the ODEs in scipy, with the RK45 method.
The original model publication can be found here.
Now, I think I understand what is going on with the error but I cannot find a way to fix it.
I don't know if it comes from my implementation or from the ODEs themselves.
I've been through the units several times, checking that all times were in seconds, all distances in meters and so on.
I've tried with different t_eval and even different values of certain constants, but not been able to solve my problem.
I started by creating a class with all constants.
import numpy as np
import scipy.integrate
import matplotlib.pyplot as plt
import astropy.units as u
SECONDS_PER_YEAR = 3600*24*365.15
class Cst:
#Glenn's flow Law
A = 2.4e-25
n = 3.
#Standard physical constants
g = 10.#*(u.m)*(u.second**-2)
rho = 916#*(u.kilogram*(u.m**-3))
#Thermodynamics
cp = 2000#**(u.Joule)*(u.kilogram**-1)*(u.Kelvin**-1)
L = 3.3e5#*(u.Joule)*(u.kilogram**-1)
k = 2.1 #*(u.Watt)*(u.m**-1)*'(u.Kelvin**-1)'
DDF = 0.1/SECONDS_PER_YEAR #*(u.m)*(u.yr**-1)*'(u.Kelvin**-1)
K = 2.3e-47#*((3600*24*365.15)**9)#*((u.kilogram**-5)*(u.m**2)*(u.second**9))
C = 9.2e13#*((u.Pascal)*(u.Joule)*(u.m**-2))
#Weertman friction law
q = 1
p = 1/3
R = 15.7#*((u.m**(-1/3))*(u.second**(1/3)))
d = 10#*u.m
sin_theta = 0.05
Tm = 0+273.15 #*u.Kelvin
T_offset = -10+273.15#*u.Kelvin
w = 0.6 #u.m
Wc = 1000.#*u.m
#Velocities
u1 = 0/SECONDS_PER_YEAR #m/s
u2 = 100/SECONDS_PER_YEAR # m/s
#Dimensionless parameters
alpha = 5.
Then I declared the problem-specific parameters specified in the paper:
#All values are from Table 1
a0 = 1./SECONDS_PER_YEAR#* m/s (u.meter*((u.second)**-1))
l0 = 10000#*(u.meter)
E0 = 1.8e8#(Cst.g*Cst.sin_theta*a0*(l0**2))/(Cst.L*Cst.K))**(1/Cst.alpha)#*(u.Joule/u.m**2)
T0 = 10#E0/(Cst.rho*Cst.cp*Cst.d)#*u.Kelvin
w0 = 0.6#E0/(Cst.rho*Cst.L)#*u.m
N0 = 0.5#Cst.C/E0#*u.Pascal
H0 = 200 #((Cst.R*(Cst.C**Cst.q)*(a0**Cst.p)*(l0**Cst.p))/(Cst.rho*Cst.g*Cst.sin_theta*(E0**Cst.q)))**(1/(Cst.p+1))
t0 = 200 #H0/a0
u0 = 50/SECONDS_PER_YEAR#((Cst.rho*Cst.g*Cst.sin_theta*(E0**Cst.q)*a0*l0)/(Cst.R*(Cst.C**Cst.q)))**(1/(Cst.p+1))
Q0 = (Cst.g*Cst.sin_theta*a0*(l0**2))/Cst.L
S0 = ((Cst.g*Cst.sin_theta*a0*(l0**2)*Cst.Wc)/(Cst.L*Cst.K*((Cst.rho*Cst.g*Cst.sin_theta)**(1/2))))**(3/4)
lamb = ((2.*Cst.A*(Cst.rho*Cst.g*Cst.sin_theta)**Cst.n)*(H0**(Cst.n+1)))/((Cst.n+2)*u0)
chi = N0/(Cst.rho*Cst.g*H0)
gamma = 0.41
kappa = 0.7
phi = 0.2
delta = 66
mu = 0.2
Define the model :
def model(t, x):
#Initial values
H_hat = x[0]
E_hat = x[1]
#Thickness
H = H_hat*H0
#Enthalpy
E_hat_plus = max(E_hat, 0)
E_hat_minus = min(E_hat, 0)
E_plus = E_hat_plus*E0
E_minus = E_hat_minus*E0
a_hat = 1.
theta_hat = Cst.sin_theta/Cst.sin_theta
l_hat =l0/l0
T_a = 0+273.15
T = -10+273.15
# Equation 3
m_hat = (Cst.DDF*(T_a-Cst.T_offset))/a0
S_hat = 0.
T_a_hat = T_a/T0
#Equation A7
if E_plus > 0:
N = min(H/chi, 1./E_plus)
else:
N = H/chi
phi = min(1., E_plus/(H/chi))
#Equation 8
inv_p = 1./Cst.p
u = (Cst.rho*Cst.g*Cst.sin_theta/Cst.R * H * (N**(-Cst.q)))**inv_p
#Equation A7
beta = min(max(0, (u-Cst.u1)/(Cst.u2-Cst.u1)), 1)
#Equation A4
dHdt_hat = (
a_hat - m_hat
+ 1./l_hat*(
theta_hat**inv_p
* H_hat**(1.+inv_p)
* N**(-Cst.q*inv_p)
+ lamb*(theta_hat**Cst.n)
)
)
#Equation A5
dEdt_hat = 1./mu*(
theta_hat**(1+inv_p) * H_hat**(1.+inv_p) * N**(-Cst.q*inv_p)
+ gamma
+ kappa*(E_hat_minus - T_a_hat)/H_hat
- 1./l_hat * (
theta_hat * E_hat_plus**Cst.alpha
+ phi * theta_hat**(1./2) * S_hat**(4/3.)
)
+ delta * beta * m_hat
)
return [dHdt_hat, dEdt_hat]
And finally call it :
tmax = 200*SECONDS_PER_YEAR# *u.years
t = np.linspace(0, tmax, 10000)
sol = scipy.integrate.solve_ivp(model, t_span=[t[0], t[-1]], y0=[1, 1], t_eval=t, method='RK23')
print(sol)
Which yields
message: 'Required step size is less than spacing between numbers.'
nfev: 539
njev: 0
nlu: 0
sol: None
status: -1
success: False
t: array([0.])
t_events: None
y: array([[1.],
[1.]])
y_events: None

Matlab to python - different results, cannot solve

I have been trying to convert the following matlab code into python and I am having difficulties with construction of the matrix algebra.
The product of the python code is vastly different from matlab and I can't figure out the problem (I assume its in the matrix multiplication).
Just for some background info: Using the data set, I was attempting to create a forecast with one dummy varialbe 'D1'
Here is the matlab code:
load 'AUSRetail.csv'
y = AUSRetail(:,1); Q = AUSRetail(:,2);
T = length(y); t = (1:T)';
D4 = (Q == 4);
T0 = 15;
h = 1; % h−step−ahead forecast
syhat = zeros(T-h-T0+1,1);
ytph = y(T0+h:end); % observed y {t+h}
for t = T0:T-h
yt = y(1:t);
D4t = D4(1:t);
Xt = [ones(t,1) (1:t)' D4t];
beta2 = (Xt'*Xt)\(Xt'*yt);
yhat2 = [1 t+h D4(t+h)]*beta2;
syhat(t-T0+1) = yhat2;
end
MSFE2 = mean((ytph-syhat).^2);
plot([1:T], y)
hold on
plot([T0 + h:T], syhat)
and here is my python code attempt:
data = pd.read_csv("dataretail.csv").dropna()
D4 = np.asarray(data["Dummy4"])
dataValues = np.asarray(data["Value"])
T = len(D4)
T0 = 15
h = 1
syhat = []
ytph = np.asarray(dataValues) # Real values to test against
for t in range(T0,T-1):
yt = dataValues[:t].reshape(t,1)
D4t = D4[:t]
#Construction of Xt
xt1 = np.ones((t,1))
xt2 = np.arange(1,t+1).reshape(t,1)
xt3 = D4t.reshape(t,1)
Xt = np.column_stack((xt1,xt2,xt3))
Xt2 = np.transpose(Xt)
A = Xt2 # Xt
b = Xt2 # yt
beta2 = np.transpose(A) # b
yhat2 = np.array([1, t+h, D4[t+h]]) # beta2
syhat.append(int(yhat2))
fig = plt.figure()
axes = fig.add_axes([0.2, 0.2, 0.8, 1])
axes.set_xlabel("T (time)")
axes.set_ylabel("Yhat2")
axes.plot(range(25), abc)
axes.set_title("Model")

Having successfully converted complex Matlab code to Python, how to run the code?

This question is a follow-up to my previous question here: Assistance, tips and guidelines for converting Matlab code to Python
I have converted the Matlab code manually. I am using a MAC OS and running Python from the terminal. But how do I run the code below, for some value of N, where N is an even number? I should get a graph (specified by the plot code).
When I run it as is, I get nothing.
My code is below:
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
def Array(N):
K00 = np.logspace(0,3,101,10)
len1 = len(K00)
y0 = [0]*(3*N/2+3)
S = [np.zeros((len1,1)) for kkkk in range(N/2+1)]
KS = [np.zeros((len1,1)) for kkkk in range(N/2)]
PS = [np.zeros((len1,1)) for kkkk in range(N/2)]
Splot = [np.zeros((len1,1)) for kkkk in range(N/2+1)]
KSplot = [np.zeros((len1,1)) for kkkk in range(N/2)]
PSplot = [np.zeros((len1,1)) for kkkk in range(N/2)]
Kplot = np.zeros((len1,1))
Pplot = np.zeros((len1,1))
for series in range(0,len1):
K0 = K00[series]
Q = 10
r1 = 0.0001
r2 = 0.001
d = 0.001
a = 0.001
k = 0.999
P0 = 1
S10 = 1e5
tf = 1e10
time = np.linspace(0,tf,len1)
y0[0] = S10
y0[3*N/2+1] = K0
y0[3*N/2+2] = P0
for i in range(1,3*N/2+1):
y0[i] = 0
[t,y] = odeint(EqnsArray,y0,time, mxstep = 5000)
for alpha in range(0,(N/2+1)):
S[alpha] = y[:,alpha]
for beta in range((N/2)+1,N+1):
KS[beta-N/2-1] = y[:,beta]
for gamma in range(N+1,3*N/2+1):
PS[gamma-N-1] = y[:,gamma]
for alpha in range(0,(N/2+1)):
Splot[alpha][series] = y[len1-1,alpha]
for beta in range((N/2)+1,N+1):
KSplot[beta-N/2-1][series] = y[len1-1,beta]
for gamma in range(N+1,3*N/2+1):
PSplot[gamma-N-1][series] = y[len1-1,gamma]
for alpha in range(0,(N/2+1)):
u1 = u1 + Splot[alpha]
for beta in range((N/2)+1,N+1):
u2 = u2 + KSplot[beta-N/2-1]
for gamma in range(N+1,3*N/2+1):
u3 = u3 + PSplot[gamma-N-1]
K = soln[:,3*N/2+1]
P = soln[:,3*N/2+2]
Kplot[series] = soln[len1-1,3*N/2+1]
Pplot[series] = soln[len1-1,3*N/2+2]
utot = u1+u2+u3
#Plot
plt.plot(np.log10(K00),utot)
plt.show()
def EqnsArray(y,t):
for alpha in range(0,(N/2+1)):
S[alpha] = y[alpha]
for beta in range((N/2)+1,N+1):
KS[beta-N/2-1] = y[beta]
for gamma in range(N+1,3*N/2+1):
PS[gamma-N-1] = y[gamma]
K = y[3*N/2+1]
P = y[3*N/2+2]
# The model equations
ydot = np.zeros((3*N/2+3,1))
B = range((N/2)+1,N+1)
G = range(N+1,3*N/2+1)
runsumPS = 0
runsum1 = 0
runsumKS = 0
runsum2 = 0
for m in range(0,N/2):
runsumPS = runsumPS + PS[m]
runsum1 = runsum1 + S[m+1]
runsumKS = runsumKS + KS[m]
runsum2 = runsum2 + S[m]
ydot[B[m]] = a*K*S[m]-(d+k+r1)*KS[m]
for i in range(0,N/2-1):
ydot[G[i]] = a*P*S[i+1]-(d+k+r1)*PS[i]
for p in range(1,N/2):
ydot[p] = -S[p]*(r1+a*K+a*P)+k*KS[p-1]+d*(PS[p-1]+KS[p])
ydot[0] = Q-(r1+a*K)*S[0]+d*KS[0]+k*runsumPS
ydot[N/2] = k*KS[N/2-1]-(r2+a*P)*S[N/2]+d*PS[N/2-1]
ydot[G[N/2-1]] = a*P*S[N/2]-(d+k+r2)*PS[N/2-1]
ydot[3*N/2+1] = (d+k+r1)*runsumKS-a*K*runsum2
ydot[3*N/2+2] = (d+k+r1)*(runsumPS-PS[N/2-1])- \
a*P*runsum1+(d+k+r2)*PS[N/2-1]
ydot_new = []
for j in range(0,3*N/2+3):
ydot_new.extend(ydot[j])
return ydot_new

You have to call your function, like:
Array(12)
You have to add this at the end of your code.