I want to modify the following NumPyro model:
import jax.numpy as jnp
from jax import random, vmap
import numpy as np
import numpyro
import numpyro.distributions as dist
from numpyro.infer import MCMC, NUTS
numpyro.set_host_device_count(6)
def model(y=None, X=None):
n_predictors = X.shape[1]
with numpyro.plate('state', n_predictors):
theta = numpyro.sample('theta', dist.Gamma(concentration=1, rate=1/5000))
mu = jnp.dot(X, theta)
numpyro.sample('y', dist.Normal(loc=mu, scale=1), obs=y)
theta = np.zeros(5) # True parameters
theta[0] = 2
theta[1] = 3
X = np.random.randn(20, theta.size)**2 # Design matrix
y = X # theta + np.random.randn(X.shape[0]) # data
rng_key = random.PRNGKey(74674)
rng_key, rng_key_ = random.split(rng_key)
mcmc = MCMC(NUTS(model), num_warmup=500, num_samples=1000, num_chains=6)
mcmc.run(rng_key_, X=X, y=y)
mcmc.print_summary()
I want to include Bernoulli RVs z that choose which theta is active. Then I would like to make inference for these z. Basically, I am trying to do variable selection. The idea is illustrated in the following model (which fails):
def failed_model(y=None, X=None):
n_predictors = X.shape[1]
with numpyro.plate('state', n_predictors):
theta = numpyro.sample('theta', dist.Gamma(concentration=1, rate=1/5000))
z = numpyro.sample('z', dist.Bernoulli(0.1)
mu = jnp.dot(X, theta * z)
numpyro.sample('y', dist.Normal(loc=mu, scale=1), obs=y)
I tried to understand the second example from the [docs][1], but it does not show masking for a random array, but rather for a fixed array.
[1]: https://num.pyro.ai/en/stable/distributions.html
Related
I want to use the Gaussian Process approximation for a simple 1D test function to illustrate a few things. I want to iterate over a few different values for the correlation matrix (since this is 1D it is just a single value) and show what effect different values have on the approximation. My understanding is, that "theta" is the parameter for this. Therefore I want to set the theta value manually and don't want any optimization/changes to it. I thought the constant kernel and the clone_with_theta function might get me what I want but I didn't get it to work. Here is what I have so far:
import numpy as np
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as ConstantKernel
def f(x):
"""The function to predict."""
return x/2 + ((1/10 + x) * np.sin(5*x - 1))/(1 + x**2 * (np.sin(x - (1/2))**2))
# ----------------------------------------------------------------------
# Data Points
X = np.atleast_2d(np.delete(np.linspace(-1,1, 7),4)).T
y = f(X).ravel()
# Instantiate a Gaussian Process model
kernel = ConstantKernel(constant_value=1, constant_value_bounds='fixed')
theta = np.array([0.5,0.5])
kernel = kernel.clone_with_theta(theta)
gp = GaussianProcessRegressor(kernel=kernel, optimizer=None)
# Fit to data using Maximum Likelihood Estimation of the parameters
gp.fit(X, y)
# Make the prediction on the meshed x-axis (ask for MSE as well)
y_pred, sigma = gp.predict(x, return_std=True)
# Plot
# ...
I programmed a simple implementation myself now, which allows to set correlation (here 'b') manually:
import numpy as np
from numpy.linalg import inv
def f(x):
"""The function to predict."""
return x/2 + ((1/10 + x) * np.sin(5*x - 1))/(1 + x**2 * (np.sin(x - (1/2))**2))
def kriging_approx(x,xt,yt,b,mu,R_inv):
N = yt.size
one = np.matrix(np.ones((yt.size))).T
r = np.zeros((N))
for i in range(0,N):
r[i]= np.exp(-b * (xt[i]-x)**2)
y = mu + np.matmul(np.matmul(r.T,R_inv),yt - mu*one)
y = y[0,0]
return y
def calc_R (x,b):
N = x.size
# setup R
R = np.zeros((N,N))
for i in range(0,N):
for j in range(0,N):
R[i][j] = np.exp(-b * (x[i]-x[j])**2)
R_inv = inv(R)
return R, R_inv
def calc_mu_sig (yt, R_inv):
N = yt.size
one = np.matrix(np.ones((N))).T
mu = np.matmul(np.matmul(one.T,R_inv),yt) / np.matmul(np.matmul(one.T,R_inv),one)
mu = mu[0,0]
sig2 = (np.matmul(np.matmul((yt - mu*one).T,R_inv),yt - mu*one))/(N)
sig2 = sig2[0,0]
return mu, sig2
# ----------------------------------------------------------------------
# Data Points
xt = np.linspace(-1,1, 7)
yt = np.matrix((f(xt))).T
# Calc R
R, R_inv = calc_R(xt, b)
# Calc mu and sigma
mu_dach, sig_dach2 = calc_mu_sig(yt, R_inv)
# Point to get approximation for
x = 1
y_approx = kriging_approx(x, xt, yt, b, mu_dach, R_inv)
The goal is to plot two identical dynamical systems that are coupled.
We have:
X = [x0,x1,x2]
U = [u0,u1,u2]
And
Xdot = f(X) + alpha*(U-X)
Udot = f(U) + alpha*(X-U)
So I wish to plot the solution to this grand system on one set of axes (i.e in xyz for example) and eventually change the coupling strength to investigate the behaviour.
import matplotlib.pyplot as plt
from scipy.integrate import odeint
from mpl_toolkits.mplot3d import Axes3D
def couple(s,t,a=0.2,beta=0.2,gamma=5.7,alpha=0.03):
[x,u] = s
[u0,u1,u2] = u
[x0,x1,x2] = x
xdot = np.zeros(3)
xdot[0] = -x1-x2
xdot[1] = x0+a*x1
xdot[2] = beta + x2*(x0-gamma)
udot = np.zeros(3)
udot[0] = -u1-u2
udot[1] = u0+a*u1
udot[2] = beta + u2*(u0-gamma)
sdot = np.zeros(2)
sdot[0] = xdot + alpha*(u-x)
sdot[1] = udot + alpha*(x-u)
return sdot
s_init = [0.1,0.1]
t_init=0; t_final = 300; t_step = 0.01
tpoints = np.arange(t_init,t_final,t_step)
a=0.2; beta=0.2; gamma=5.7; alpha=0.03
y = odeint(couple, s_init, tpoints,args=(a,beta,gamma,alpha), hmax = 0.01)
I imagine that something is wrong with s_init since it should be TWO initial condition vectors but when I try that I get that "odeint: y0 should be one-dimensional." On the other hand when I try s_init to be a 6-vector I get "too many values to unpack (expected two)." With the current setup, I am getting the error
File "C:/Users/Python Scripts/dynsys2019work.py", line 88, in couple
[u0,u1,u2] = u
TypeError: cannot unpack non-iterable numpy.float64 object
Cheers
*Edit: Please note this is basically my first time attempting this kind of thing and will be happy to receive further documentation and references.
The ode definition takes in and returns a 1D vector in scipy odeint, and I think some of your confusion is that you actually have 1 system of ODEs with 6 variables. You have just mentally apportioned it into 2 separate ODEs that are coupled.
You can do it like this:
import matplotlib.pyplot as plt
from scipy.integrate import odeint
import numpy as np
def couple(s,t,a=0.2,beta=0.2,gamma=5.7,alpha=0.03):
x0, x1, x2, u0, u1, u2 = s
xdot = np.zeros(3)
xdot[0] = -x1-x2
xdot[1] = x0+a*x1
xdot[2] = beta + x2*(x0-gamma)
udot = np.zeros(3)
udot[0] = -u1-u2
udot[1] = u0+a*u1
udot[2] = beta + u2*(u0-gamma)
return np.ravel([xdot, udot])
s_init = [0.1,0.1, 0.1, 0.1, 0.1, 0.1]
t_init=0; t_final = 300; t_step = 0.01
tpoints = np.arange(t_init,t_final,t_step)
a=0.2; beta=0.2; gamma=5.7; alpha=0.03
y = odeint(couple, s_init, tpoints,args=(a,beta,gamma,alpha), hmax = 0.01)
plt.plot(tpoints,y[:,0])
I have this simple regression model:
y = a + b * x + c * z + error
with a constraint on parameters:
c = b - 1
There are similar questions posted on SO (like Constrained Linear Regression in Python). However, the constraints' type is lb <= parameter =< ub.
What are the available options to handle this specific constrained linear regression problem?
This is how it can be done using GLM:
import statsmodels
import statsmodels.api as sm
import numpy as np
# Set the link function to identity
statsmodels.genmod.families.links.identity()
OLS_from_GLM = sm.GLM(y, sm.add_constant(np.column_stack(x, z)))
'''Setting the restrictions on parameters in the form of (R, q), where R
and q are constraints' matrix and constraints' values, respectively. As
for the restriction in the aforementioned regression model, i.e.,
c = b - 1 or b - c = 1, R = [0, 1, -1] and q = 1.'''
res_OLS_from_GLM = OLS_from_GLM.fit_constrained(([0, 1.0, -1.0], 1))
print(res_OLS_from_GLM.summary())
There are a few constrained optimization packages in Python such as CVX, CASADI, GEKKO, Pyomo, and others that can solve the problem. I develop Gekko for linear, nonlinear, and mixed integer optimization problems with differential or algebraic constraints.
import numpy as np
from gekko import GEKKO
# Data
x = np.random.rand(10)
y = np.random.rand(10)
z = np.random.rand(10)
# Gekko for constrained regression
m = GEKKO(remote=False); m.options.IMODE=2
a,b,c = m.Array(m.FV,3)
a.STATUS=1; b.STATUS=1; c.STATUS=1
x=m.Param(x); z=m.Param(z)
y = m.Var(); ym=m.Param(y)
m.Equation(y==a+b*x+c*z)
m.Equation(c==b-1)
m.Minimize((ym-y)**2)
m.options.SOLVER=1
m.solve(disp=True)
print(a.value[0],b.value[0],c.value[0])
This gives the solution that may be different when you run it because it uses random values for the data.
-0.021514129645 0.45830726553 -0.54169273447
The constraint c = b - 1 is satisfied with -0.54169273447 = 0.45830726553 - 1. Here is a comparison to other linear regression packages in Python with an without constraints:
import numpy as np
from scipy.stats import linregress
import statsmodels.api as sm
import matplotlib.pyplot as plt
from gekko import GEKKO
# Data
x = np.array([4,5,2,3,-1,1,6,7])
y = np.array([0.3,0.8,-0.05,0.1,-0.8,-0.5,0.5,0.65])
# calculate R^2
def rsq(y1,y2):
yresid= y1 - y2
SSresid = np.sum(yresid**2)
SStotal = len(y1) * np.var(y1)
r2 = 1 - SSresid/SStotal
return r2
# Method 1: scipy linregress
slope,intercept,r,p_value,std_err = linregress(x,y)
a = [slope,intercept]
print('R^2 linregress = '+str(r**2))
# Method 2: numpy polyfit (1=linear)
a = np.polyfit(x,y,1); print(a)
yfit = np.polyval(a,x)
print('R^2 polyfit = '+str(rsq(y,yfit)))
# Method 3: numpy linalg solution
# y = X a
# X^T y = X^T X a
X = np.vstack((x,np.ones(len(x)))).T
# matrix operations
XX = np.dot(X.T,X)
XTy = np.dot(X.T,y)
a = np.linalg.solve(XX,XTy)
# same solution with lstsq
a = np.linalg.lstsq(X,y,rcond=None)[0]
yfit = a[0]*x+a[1]; print(a)
print('R^2 matrix = '+str(rsq(y,yfit)))
# Method 4: statsmodels ordinary least squares
X = sm.add_constant(x,prepend=False)
model = sm.OLS(y,X).fit()
yfit = model.predict(X)
a = model.params
print(model.summary())
# Method 5: Gekko for constrained regression
m = GEKKO(remote=False); m.options.IMODE=2
c = m.Array(m.FV,2); c[0].STATUS=1; c[1].STATUS=1
c[1].lower=-0.5
xd = m.Param(x); yd = m.Param(y); yp = m.Var()
m.Equation(yp==c[0]*xd+c[1])
m.Minimize((yd-yp)**2)
m.solve(disp=False)
c = [c[0].value[0],c[1].value[1]]
print(c)
# plot data and regressed line
plt.plot(x,y,'ko',label='data')
xp = np.linspace(-2,8,100)
slope = str(np.round(a[0],2))
intercept = str(np.round(a[1],2))
eqn = 'LstSQ: y='+slope+'x'+intercept
plt.plot(xp,a[0]*xp+a[1],'r-',label=eqn)
slope = str(np.round(c[0],2))
intercept = str(np.round(c[1],2))
eqn = 'Constraint: y='+slope+'x'+intercept
plt.plot(xp,c[0]*xp+c[1],'b--',label=eqn)
plt.grid()
plt.legend()
plt.show()
I am playing with a simple numpy example and having hard time to understand why associative property of matrix multiplication
ABC = (AB)C = A(BC)
does not exactly hold. I assume the problem is with numeric stability. But how to address it? What is the issue exactly?
Here is my example with linear regression. I use sklearn solution as it gives more divergence between associative groupings:
import numpy as np
np.random.seed(42)
num_samples = 100
M = 1000
sigma = 0.5
X = np.random.binomial(2, 0.4, (num_samples, M))
beta = np.zeros(M)
beta[5] = 1.0
y = X.dot(beta) + sigma*np.random.randn(num_samples)
"standardise y"
y = y - np.mean(y)
y = y/np.std(y)
"center and standardise X"
Xc = X - X.mean(axis=0)
xstd = X.std(axis=0)
mask = xstd > 1e-12
Xc = Xc[:, mask]
from sklearn.linear_model import LinearRegression
lr = LinearRegression()
lr.fit(Xc ,y)
beta_hat_sklearn = lr.coef_
beta_hat_sklearn.T # Xc.T # Xc # beta_hat_sklearn / num_samples
"equivalent < Python3.5"
beta_hat_sklearn.T.dot(Xc.T).dot(Xc).dot(beta_hat_sklearn) / num_samples
# 1.0000000000000009
beta_hat_sklearn.T # (Xc.T # Xc) # beta_hat_sklearn / num_samples
"equivalent < Python3.5"
beta_hat_sklearn.T.dot(Xc.T.dot( Xc )).dot(beta_hat_sklearn )/ num_samples
# 0.89517439485479278
Update
It might be MacOSX specific bug.
How do I numerically solve an ODE in Python?
Consider
\ddot{u}(\phi) = -u + \sqrt{u}
with the following conditions
u(0) = 1.49907
and
\dot{u}(0) = 0
with the constraint
0 <= \phi <= 7\pi.
Then finally, I want to produce a parametric plot where the x and y coordinates are generated as a function of u.
The problem is, I need to run odeint twice since this is a second order differential equation.
I tried having it run again after the first time but it comes back with a Jacobian error. There must be a way to run it twice all at once.
Here is the error:
odepack.error: The function and its Jacobian must be callable functions
which the code below generates. The line in question is the sol = odeint.
import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
from numpy import linspace
def f(u, t):
return -u + np.sqrt(u)
times = linspace(0.0001, 7 * np.pi, 1000)
y0 = 1.49907
yprime0 = 0
yvals = odeint(f, yprime0, times)
sol = odeint(yvals, y0, times)
x = 1 / sol * np.cos(times)
y = 1 / sol * np.sin(times)
plot(x,y)
plt.show()
Edit
I am trying to construct the plot on page 9
Classical Mechanics Taylor
Here is the plot with Mathematica
In[27]:= sol =
NDSolve[{y''[t] == -y[t] + Sqrt[y[t]], y[0] == 1/.66707928,
y'[0] == 0}, y, {t, 0, 10*\[Pi]}];
In[28]:= ysol = y[t] /. sol[[1]];
In[30]:= ParametricPlot[{1/ysol*Cos[t], 1/ysol*Sin[t]}, {t, 0,
7 \[Pi]}, PlotRange -> {{-2, 2}, {-2.5, 2.5}}]
import scipy.integrate as integrate
import matplotlib.pyplot as plt
import numpy as np
pi = np.pi
sqrt = np.sqrt
cos = np.cos
sin = np.sin
def deriv_z(z, phi):
u, udot = z
return [udot, -u + sqrt(u)]
phi = np.linspace(0, 7.0*pi, 2000)
zinit = [1.49907, 0]
z = integrate.odeint(deriv_z, zinit, phi)
u, udot = z.T
# plt.plot(phi, u)
fig, ax = plt.subplots()
ax.plot(1/u*cos(phi), 1/u*sin(phi))
ax.set_aspect('equal')
plt.grid(True)
plt.show()
The code from your other question is really close to what you want. Two changes are needed:
You were solving a different ODE (because you changed two signs inside function deriv)
The y component of your desired plot comes from the solution values, not from the values of the first derivative of the solution, so you need to replace u[:,0] (function values) for u[:, 1] (derivatives).
This is the end result:
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint
def deriv(u, t):
return np.array([u[1], -u[0] + np.sqrt(u[0])])
time = np.arange(0.01, 7 * np.pi, 0.0001)
uinit = np.array([1.49907, 0])
u = odeint(deriv, uinit, time)
x = 1 / u[:, 0] * np.cos(time)
y = 1 / u[:, 0] * np.sin(time)
plt.plot(x, y)
plt.show()
However, I suggest that you use the code from unutbu's answer because it's self documenting (u, udot = z) and uses np.linspace instead of np.arange. Then, run this to get your desired figure:
x = 1 / u * np.cos(phi)
y = 1 / u * np.sin(phi)
plt.plot(x, y)
plt.show()
You can use scipy.integrate.ode. To solve dy/dt = f(t,y), with initial condition y(t0)=y0, at time=t1 with 4th order Runge-Kutta you could do something like this:
from scipy.integrate import ode
solver = ode(f).set_integrator('dopri5')
solver.set_initial_value(y0, t0)
dt = 0.1
while t < t1:
y = solver.integrate(t+dt)
t += dt
Edit: You have to get your derivative to first order to use numerical integration. This you can achieve by setting e.g. z1=u and z2=du/dt, after which you have dz1/dt = z2 and dz2/dt = d^2u/dt^2. Substitute these into your original equation, and simply iterate over the vector dZ/dt, which is first order.
Edit 2: Here's an example code for the whole thing:
import numpy as np
import matplotlib.pyplot as plt
from numpy import sqrt, pi, sin, cos
from scipy.integrate import ode
# use z = [z1, z2] = [u, u']
# and then f = z' = [u', u''] = [z2, -z1+sqrt(z1)]
def f(phi, z):
return [z[1], -z[0]+sqrt(z[0])]
# initialize the 4th order Runge-Kutta solver
solver = ode(f).set_integrator('dopri5')
# initial value
z0 = [1.49907, 0.]
solver.set_initial_value(z0)
values = 1000
phi = np.linspace(0.0001, 7.*pi, values)
u = np.zeros(values)
for ii in range(values):
u[ii] = solver.integrate(phi[ii])[0] #z[0]=u
x = 1. / u * cos(phi)
y = 1. / u * sin(phi)
plt.figure()
plt.plot(x,y)
plt.grid()
plt.show()
scipy.integrate() does ODE integration. Is that what you are looking for?