Related
I have to do a second degree interpolation using an already existing code and changing the values for mine, but for some reason when i go ahead and gragth the interpolation, the fuction suddently stops (it is not continuous). Can someone help me figuring out whats wrong? I believe it has something to do with line 43 (evaluation on new data points), but I am not sure.
Source code:
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-poster')
%matplotlib inline
def divided_diff(x, y):
'''
function to calculate the divided
differences table
'''
n = len(y)
coef = np.zeros([n, n])
# the first column is y
coef[:,0] = y
for j in range(1,n):
for i in range(n-j):
coef[i][j] = \
(coef[i+1][j-1] - coef[i][j-1]) / (x[i+j]-x[i])
return coef
def newton_poly(coef, x_data, x):
'''
evaluate the newton polynomial
at x
'''
n = len(x_data) - 1
p = coef[n]
for k in range(1,n+1):
p = coef[n-k] + (x -x_data[n-k])*p
return p
x = np.array([-5, -1, 0, 2])
y = np.array([-2, 6, 1, 3])
# get the divided difference coef
a_s = divided_diff(x, y)[0, :]
# evaluate on new data points
x_new = np.arange(-5, 2.1, .1)
y_new = newton_poly(a_s, x, x_new)
plt.figure(figsize = (12, 8))
plt.plot(x, y, 'bo')
plt.plot(x_new, y_new)
My code (adjusted for data points (0,0);(6.4,1.9);(10.6,4.3)):
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-poster')
%matplotlib inline
def divided_diff(x, y):
'''
function to calculate the divided
differences table
'''
n = len(y)
coef = np.zeros([n, n])
# the first column is y
coef[:,0] = y
for j in range(1,n):
for i in range(n-j):
coef[i][j] = \
(coef[i+1][j-1] - coef[i][j-1]) / (x[i+j]-x[i])
return coef
def newton_poly(coef, x_data, x):
'''
evaluate the newton polynomial
at x
'''
n = len(x_data) - 1
p = coef[n]
for k in range(1,n+1):
p = coef[n-k] + (x -x_data[n-k])*p
return p
x = np.array([0, 6.4, 10.6])
y = np.array([0, 1.9, 4.3])
# get the divided difference coef
a_s = divided_diff(x, y)[0, :]
# evaluate on new data points
x_new = np.arange(-5, 2.1, .1)
y_new = newton_poly(a_s, x, x_new)
plt.figure(figsize = (12, 8))
plt.plot(x, y, 'bo')
plt.plot(x_new, y_new)
I have a set of data and I have fit a normal distribution to the data using numpy and scipy. But when I try to plot the pdf, I get the following error:
I have tried to change the dtype of Z, but that did not work. Any suggestions will help. Thanks.
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
import numpy as np
import random
from sympy.matrices import Matrix
from sympy import symbols, pprint, N
from scipy.stats import multivariate_normal
from target import true_target_trajectory, target_posiion
def plot_gaussian(X, Y, Z):
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot_surface(X, Y, Z)
plt.show()
def covariance(x, y):
sigma1 = np.std(x, dtype=np.float64)
sigma2 = np.std(y, dtype=np.float64)
cov = np.matrix([[sigma1, sigma1*sigma2], [sigma1*sigma2, sigma2]])
min_eig = np.min(np.real(np.linalg.eigvals(cov)))
if min_eig < 0:
cov -= 10*min_eig * np.eye(*cov.shape)
return cov
def gaussian(x, mu, cov):
rv = multivariate_normal(mu, cov)
return rv.pdf(x)
#plot_gaussian()
vin = 300
qin = 9
x = []
y = []
time = np.linspace(0, 2*np.pi, 100)
for t in (time):
cc = target_posiion(vin, qin, t)
x.append(cc.T[0])
y.append(cc.T[1])
mu = np.array([np.mean(x), np.mean(y)])
cov = covariance(x, y)
X, Y = np.meshgrid(x, y)
pos = np.dstack((X, Y))
Z = gaussian(pos, mu, cov)
plot_gaussian(X, Y, Z)
I tried to reproduce the issue with x = np.linspace(-1, 3, 100) and y = np.linspace(0, 4, 100). But that did not give any error and i got the bell curve as expected.
So i am attaching the code for target position.
The code for target_position:
import random
import numpy as np
from sympy.vector.coordsysrect import CoordSys3D
from sympy.physics.mechanics import dynamicsymbols
from sympy import symbols, sin, pprint, Derivative, Identity, N
from sympy.matrices import Matrix, BlockMatrix, block_collapse
C = CoordSys3D('C')
i, j, k = C.base_vectors()
def evaluate_matrix(m, v_in, q_in, tk):
w, t = symbols('w t')
v0, q = symbols('v0 q')
params = {v0:v_in, q:q_in, t:tk}
return Matrix([[N(m[0].subs(params)), N(m[1].subs(params))]]).T
def true_target_trajectory(v_in, q_in, tk):
w, t = symbols('w t')
v0, q, A = symbols('v0 q A')
r, v, a, x, y = dynamicsymbols('r v a x y')
A = (v0**2)/q
w = q/(2*v0)
x = A*sin(w*t)*i
y = A*sin(2*w*t)*j
r = x + y
r_m = Matrix(r.to_matrix(C)[:2])
v = Derivative(r, t).doit()
v_m = Matrix(v.to_matrix(C)[:2])
a = Derivative(v, t).doit()
a_m = Matrix(a.to_matrix(C)[:2])
x_k = BlockMatrix([[r_m.T, v_m.T, a_m.T]]).T
I = Identity(2)
H = BlockMatrix([[I, I, I]])
z = evaluate_matrix(block_collapse(H*x_k), v_in, q_in, tk)
return z
def target_posiion(v_in, q_in, tk):
sigma = 50
u_k = Matrix([[random.gauss(0,1), random.gauss(0,1)]]).T
z = true_target_trajectory(v_in, q_in, tk)
z_c_k = z + sigma*u_k
return z_c_k
The problem
The problem is that your x and y are lists of type sympy.core.numbers.Float, not regular Python float. Numpy doesn't know how to convert Sympy numeric types, so meshgrid ends up returning X and Y arrays of dtype=object. Down the line, this ends up screwing up the call to ax.plot_surface.
The fix
Just convert x and y to standard Numpy arrays of np.float64 before you pass them into meshgrid:
X, Y = np.meshgrid(np.array(x).astype(float), np.array(y).astype(float))
Once you do that, everything should be fine. Here's the output:
I am trying to make my own implementation of a simple neural network to classify points. I heard about a specific type of activation function that I am interested in testing, the Gaussian. I do not just want to use relus or sigmoids, I am trying to build a network that takes as input about 300 x and y values, then in the first layer computes the Gaussian function on these values with about 50 neurons which each have a separate x and y value as their means (I will keep the sigma constant). Mathematically I anticipate this to look like
exp(- [(x-Mx)^2 + (y-My)^2] / (2 * sigma^2) ) / (sqrt(2*pi*sigma))
then I will perform a weighted sum of these terms over all the neurons in the first layer, add a bias, and pass it through a sigmoid to get my prediction. I will perform this step for each training example and get a list of predictions. I think that I do the forward propagation but I will include the code for that in case someone can spot an obvious error in my implementation. Then I perform the back-propogation. I have tested my updating of the weights and bias, and I believe that they are not the problem. I think that there is something wrong with my implementation of the gradient for the means however because they always cluster to a single point which clearly does not maximize the cost function. I have already tried using a couple of different data sets, and varying some hyper parameters, all to no avail. Can anyone figure out what the problem is?
Here is my code.
# libraries
import matplotlib.patches as patches
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import pdb
# functions
def gaussian(sq_error, sigma):
return ((1/np.sqrt(2*np.pi*sigma**2))) * np.exp(-(sq_error)/(2*sigma**2))
def calc_X1(X0, Mx, My, m, sigma):
X1 = [] # shape will be (10, m)
for ex in range(0, m):
sq_error = (X0[0][ex] - Mx) **2 + (X0[1][ex] - My) **2
X1.append(gaussian(sq_error, sigma))
X1 = np.array(X1)
return X1.T
def sigmoid(Z):
return 1 / (1 + np.exp(-Z))
def calc_X2(W2, X1, b2):
return sigmoid(np.dot(W2, X1) + b2)
def cost(X2, Y, m):
return -1/m * ( np.dot(Y, np.log(X2.T)) + np.dot(1-Y, np.log(1-X2.T))) [0]
def calc_dZ2(X2, Y):
return X2 - Y
def calc_dM(dZ2, W2, X1, sigma, M, m, xOrY, X0):
cur_dM = np.zeros(M.shape)
for i in range(0, m):
# pdb.set_trace()
cur_dM += dZ2[0][i] * float(np.dot(W2, X1.T[i])) * 1/sigma**2 * (X0[xOrY][i] - M)
return cur_dM / m
def train_correct(X2, Y, m):
ct = 0
for i in range(0, m):
if np.round(X2[0][i]) == Y[i]:
ct += 1
return ct / m
# graphing functions
def plot_train_data(X, Y, m, ax):
for ex in range(0, m):
xCur = X[0][ex]
yCur = X[1][ex]
if Y[ex] == 1:
color=(1, 0, 0)
else:
color=(0,0,1)
ax.scatter(xCur, yCur, c=color)
def probability_hash(pr):
return (float(pr), float(np.round(pr)), float(1-pr))
def probability_hash_1d(pr):
return float(pr)
def plot_boundary(Mx, My, sigma, W2, b2, ax):
boundsx = [-5, 5]
boundsy = [-5, 5]
samples = [10, 10]
width = (boundsx[1] - boundsx[0]) / samples[0]
height = (boundsy[1] - boundsy[0]) / samples[1]
pt = np.zeros((2,1))
for x in np.linspace(boundsx[0], boundsx[1], samples[0]):
for y in np.linspace(boundsy[0], boundsy[1], samples[1]):
pt[0][0] = x
pt[1][0] = y
X1_cur = calc_X1(pt, Mx, My, 1, sigma)
X2_cur = calc_X2(W2, X1_cur, b2)
# ax.add_patch(patches.Rectangle((x, y), width, height, facecolor=probability_hash(X2_cur)))
ax.scatter(x, y, c=probability_hash(X2_cur))
def cool_plot_boundary(Mx, My, sigma, W2, b2, ax):
boundsx = [-2, 2]
boundsy = [-2, 2]
samples = [50, 50]
width = (boundsx[1] - boundsx[0]) / samples[0]
height = (boundsy[1] - boundsy[0]) / samples[1]
pt = np.zeros((2,1))
heats = []
xs = np.linspace(boundsx[0], boundsx[1], samples[0])
ys = np.linspace(boundsy[0], boundsy[1], samples[1])
for x in xs:
heats.append([])
for y in ys:
pt[0][0] = x
pt[1][0] = y
X1_cur = calc_X1(pt, Mx, My, 1, sigma)
X2_cur = calc_X2(W2, X1_cur, b2)
heats[-1].append(probability_hash_1d(X2_cur))
# xticks = []
# yticks = []
# for i in range(0, len(xs)):
# if i % 3 == 0:
# xticks.append(round(xs[i], 2))
# for i in range(0, len(ys)):
# if i % 3 == 0:
# yticks.append(round(ys[i], 2))
xticks = []
yticks = []
sns.heatmap(heats, ax=ax, cbar=True, xticklabels=xticks, yticklabels=yticks)
def plot_m(Mx, My, n1, ax):
for i in range(0, n1):
ax.scatter(Mx[i], My[i], c="k")
# initialize parameters
file = "data/disk2.csv"
df = pd.read_csv(file)
sigma = 2
itterations = 10000
learning_rate = 0.9
n0 = 2 # DO NOT CHANGE, formality
X0 = np.row_stack((df["0"], df["1"])) # shape is (2, m)
Y = np.array(df["2"])
m = len(Y)
n1 = 50
Mx = np.random.randn(n1)
My = np.random.randn(n1)
X1 = calc_X1(X0, Mx, My, m, sigma)
n2 = 1 # DO NOT CHANGE, formality
small_number = 0.01
W2 = np.random.randn(1, n1) * small_number
b2 = 0
X2 = calc_X2(W2, X1, b2)
J = cost(X2, Y, m)
Js = []
itters = []
fig = plt.figure()
plotGap = 200
for i in range(0, itterations):
# forward propogation
X1 = calc_X1(X0, Mx, My, m, sigma)
X2 = calc_X2(W2, X1, b2)
J = cost(X2, Y, m)
if i % plotGap == 0:
fig.clear()
costAx = fig.add_subplot(311)
plotAx = fig.add_subplot(312)
pointsAx = fig.add_subplot(313)
cool_plot_boundary(Mx, My, sigma, W2, b2, plotAx)
# plot_boundary(Mx, My, sigma, W2, b2, plotAx)
plot_train_data(X0, Y, m, pointsAx)
Js.append(J)
itters.append(i)
costAx.plot(itters, Js, c="k")
print("cost = " + str(J) + "\ttraining correct = " + str(train_correct(X2, Y, m)))
plot_m(Mx, My, n1, pointsAx)
plt.pause(0.1)
# back propogation
dZ2 = calc_dZ2(X2, Y)
dW2 = np.dot(dZ2, X1.T) / m
db2 = np.sum(dZ2) / m
dMx = calc_dM(dZ2, W2, X1, sigma, Mx, m, 0, X0)
dMy = calc_dM(dZ2, W2, X1, sigma, My, m, 1, X0)
b2 -= learning_rate * db2
W2 -= learning_rate * dW2
Mx -= learning_rate * dMx
My -= learning_rate * dMy
For data I have a csv with a bunch of point locations and labels. You can use this code to generate a similar csv. (Make sure you have a folder called data in the folder you run this from).
# makes data in R2 to learn
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
n = 2
# number of exaples
m = 300
X = []
Y = []
# hyperparamers for data
rApprox = 1
error = 0.4
noise = 0.1
name = "data/disk2"
plt.cla()
for ex in range(0, m):
xCur = np.random.randn(2)
X.append(xCur)
if abs(np.linalg.norm(xCur) + np.random.randn()*noise - rApprox) < error:
Y.append(1)
color="r"
else:
Y.append(0)
color="b"
plt.scatter(xCur[0], xCur[1], c=color)
if abs(np.random.randn()) < 0.01:
plt.pause(0.1)
plt.pause(1)
plt.savefig(name + ".png")
X = np.array(X)
Y = np.array(Y)
df = pd.DataFrame(X)
df[2] = Y
df.to_csv(name + ".csv", index=False)
Thanks for your help.
Substitute this function for the calculate dm function. You must be careful when multiplying, it is not just enough that the dimensions work out.
def calculuate_dMs(X0, X1, X2, Mx, My, W2, dZ2, sigma, m, n1):
# pdb.set_trace()
X0x_big = np.dot(np.ones((n1, 1)), X0[0].reshape(1, m))
X0y_big = np.dot(np.ones((n1, 1)), X0[1].reshape(1, m))
Mx_big = np.dot(Mx.reshape(n1, 1), np.ones((1, m)))
My_big = np.dot(My.reshape(n1, 1), np.ones((1, m)))
W2_big = np.dot(W2.reshape(n1, 1), np.ones((1, m)))
dZ2_big = np.dot(np.ones((n1, 1)), dZ2.reshape(1, m))
dxTemp = np.multiply(np.multiply(np.multiply((X0x_big - Mx_big), X1), W2_big), dZ2_big)
dyTemp = np.multiply(np.multiply(np.multiply((X0y_big - My_big), X1), W2_big), dZ2_big)
return (np.sum(dxTemp, axis=1)/m, np.sum(dyTemp, axis=1)/m)
I am trying to calculate the error rate of the training data I'm using.
I believe I'm calculating the error incorrectly. The formula is as shown:
y is calculated as shown:
I am calculating this in the function fitPoly(M) at line 49. I believe I am incorrectly calculating y(x(n)), but I don't know what else to do.
Below is the Minimal, Complete, and Verifiable example.
import numpy as np
import matplotlib.pyplot as plt
dataTrain = [[2.362761180904257019e-01, -4.108125266714775847e+00],
[4.324296163702689988e-01, -9.869308732049049127e+00],
[6.023323504115264404e-01, -6.684279243433971729e+00],
[3.305079685397107614e-01, -7.897042003779912278e+00],
[9.952423271981121200e-01, 3.710086310489402628e+00],
[8.308127402955634011e-02, 1.828266768673480147e+00],
[1.855495407116576345e-01, 1.039713135916495501e+00],
[7.088332047815845138e-01, -9.783208407540947560e-01],
[9.475723071629885697e-01, 1.137746192425550085e+01],
[2.343475721257285427e-01, 3.098019704040922750e+00],
[9.338350584099475160e-02, 2.316408265530458976e+00],
[2.107903139601833287e-01, -1.550451474833406396e+00],
[9.509966727520677843e-01, 9.295029459100994984e+00],
[7.164931165416982273e-01, 1.041025972594300075e+00],
[2.965557300301902011e-03, -1.060607693351102121e+01]]
def strip(L, xt):
ret = []
for i in L:
ret.append(i[xt])
return ret
x1 = strip(dataTrain, 0)
y1 = strip(dataTrain, 1)
# HELP HERE
def getY(m, w, D):
y = w[0]
y += np.sum(w[1:] * D[:m])
return y
# HELP ABOVE
def dataMatrix(X, M):
Z = []
for x in range(len(X)):
row = []
for m in range(M + 1):
row.append(X[x][0] ** m)
Z.append(row)
return Z
def fitPoly(M):
t = []
for i in dataTrain:
t.append(i[1])
w, _, _, _ = np.linalg.lstsq(dataMatrix(dataTrain, M), t)
w = w[::-1]
errTrain = np.sum(np.subtract(t, getY(M, w, x1)) ** 2)/len(x1)
print('errTrain: %s' % (errTrain))
return([w, errTrain])
#fitPoly(8)
def plotPoly(w):
plt.ylim(-15, 15)
x, y = zip(*dataTrain)
plt.plot(x, y, 'bo')
xw = np.arange(0, 1, .001)
yw = np.polyval(w, xw)
plt.plot(xw, yw, 'r')
#plotPoly(fitPoly(3)[0])
def bestPoly():
m = 0
plt.figure(1)
plt.xlim(0, 16)
plt.ylim(0, 250)
plt.xlabel('M')
plt.ylabel('Error')
plt.suptitle('Question 3: training and Test error')
while m < 16:
plt.figure(0)
plt.subplot(4, 4, m + 1)
plotPoly(fitPoly(m)[0])
plt.figure(1)
plt.plot(fitPoly(m)[1])
#plt.plot(fitPoly(m)[2])
m+= 1
plt.figure(3)
plt.xlabel('t')
plt.ylabel('x')
plt.suptitle('Question 3: best-fitting polynomial (degree = 8)')
plotPoly(fitPoly(8)[0])
print('Best M: %d\nBest w: %s\nTraining error: %s' % (8, fitPoly(8)[0], fitPoly(8)[1], ))
bestPoly()
Updated: This solution uses numpy's np.interp which will connect the points as a kind of "best fit". We then use your error function to find the difference between this interpolated line and the predicted y values for the degree of each polynomial.
import numpy as np
import matplotlib.pyplot as plt
import itertools
dataTrain = [
[2.362761180904257019e-01, -4.108125266714775847e+00],
[4.324296163702689988e-01, -9.869308732049049127e+00],
[6.023323504115264404e-01, -6.684279243433971729e+00],
[3.305079685397107614e-01, -7.897042003779912278e+00],
[9.952423271981121200e-01, 3.710086310489402628e+00],
[8.308127402955634011e-02, 1.828266768673480147e+00],
[1.855495407116576345e-01, 1.039713135916495501e+00],
[7.088332047815845138e-01, -9.783208407540947560e-01],
[9.475723071629885697e-01, 1.137746192425550085e+01],
[2.343475721257285427e-01, 3.098019704040922750e+00],
[9.338350584099475160e-02, 2.316408265530458976e+00],
[2.107903139601833287e-01, -1.550451474833406396e+00],
[9.509966727520677843e-01, 9.295029459100994984e+00],
[7.164931165416982273e-01, 1.041025972594300075e+00],
[2.965557300301902011e-03, -1.060607693351102121e+01]
]
data = np.array(dataTrain)
data = data[data[:, 0].argsort()]
X,y = data[:, 0], data[:, 1]
fig,ax = plt.subplots(4, 4)
indices = list(itertools.product([0,1,2,3], repeat=2))
for i,loc in enumerate(indices, start=1):
xx = np.linspace(X.min(), X.max(), 1000)
yy = np.interp(xx, X, y)
w = np.polyfit(X, y, i)
y_pred = np.polyval(w, xx)
ax[loc].scatter(X, y)
ax[loc].plot(xx, y_pred)
ax[loc].plot(xx, yy, 'r--')
error = np.square(yy - y_pred).sum() / X.shape[0]
print(error)
plt.show()
This prints out:
2092.19807848
1043.9400277
1166.94550318
252.238810889
225.798905379
155.785478366
125.662973726
143.787869281
6553.66570273
10805.6609259
15577.8686283
13536.1755299
108074.871771
213513916823.0
472673224393.0
1.01198058355e+12
Visually, it plots out this:
From here, it's just a matter of saving those errors to a list and finding the minimum.
I may contribute :
def pol_y(x, w):
y = 0; power = 0;
for i in w:
y += i*(x**power);
power += 1;
return y
The M is included implicitly because it is the final index of w. So if w = [0, 0, 1], then pol_y(x, w) is as same as f(x) = x^2.
If you want to map the 1st column of the dataTrain :
get_Y = [pol_y(i, w) for i in x1 ]
The error may be calculated by
vec_error = [(y1[i] - getY[i])**2 for i in range(0, len(y1)];
train_error = np.sum(vec_error)/len(y1);
Hope this helps.
I'm getting hung trying to create a contour plot of three [1, m] numpy matrices. When I try to plot:
plt.contour(reg.theta[0,:], reg.theta[1,:], reg.J[0,:])
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
I think this has something to do with the fact that
In[167]: shape(reg.theta[0,:])
Out[167]: (1, 15000)
and contour wants something along the lines of (15000,)
I've tried to create a meshgrid similar to how I would in matlab using
X , Y = meshgrid(reg.theta[0,:], reg.theta[1,:]
ValueError: total size of new array must be unchanged[1,:])
However, I have no idea how to interpret this.
Please advise stackoverflow!
Here is my complete code, tested on Python 2.7
from numpy import *
import matplotlib.pyplot as plt
class LinearRegression:
def __init__(self, data):
self.data_loc = data
self.max_it = 15000
self.theta = matrix(ones((2, self.max_it)))
self.alpha = .01
self.J = matrix(zeros((1, self.max_it)))
def importData(self):
Data = loadtxt(self.data_loc, delimiter = ',')
rawData = matrix(Data)
x = rawData[:,0]
y = rawData[:,1]
[m,n] = shape(x)
x_0 = matrix(ones((m,1)))
x = concatenate((x_0, x), axis = 1)
return x, y, m, n
def center(self, x):
x = x / mean(x)
return x
def scale(self, x):
x = x/ std(x)
return x
def funct(self, m, b):
return lambda x: m*x + b
def cost(self, x, y, m, mx, b):
f = self.funct(mx, b)
err = power( (f(x) - y), 2).sum()
err = err / (2*len(x))
return err
def gradientDesc(self, x, y, m):
for i in range(0, self.max_it -1):
error = (1.0/m) * transpose(x) * ((x * self.theta[:,i]) - y )
delta = self.theta[:,i] - self.alpha*error
self.J[0,i] = (self.cost( x, y, m, self.theta[1,i], self.theta[0,i]))
self.theta[:, i+1] = delta
print('Calculation Complete')
return self.theta, error
reg = LinearRegression('/users/michael/documents/machine_learning/ex1/ex1data1.txt')
x, y, m ,n = reg.importData()
theta, error = reg.gradientDesc(x,y,m)
print (reg.theta[:,reg.max_it -1])
XX = linspace(0,m,m)
J = reg.theta[0,reg.max_it-1] + reg.theta[1, reg.max_it-1] * XX
plt.plot(XX, J)
plt.scatter(x[:,1],y)`