Can someone enlighten me why the following code to compute the jacobian of kernel matrix doesn't work:
import autograd.numpy as np
# import numpy as np
from autograd import grad
from autograd import jacobian
from numpy import linalg as LA
def kernel(x1,x2,l):
return np.exp(-((x1-x2)**2).sum()/(2*(l**2)))
def kernel_matrixx(top_k_history):
k_t_X_list = []
for i in range(k-1):
# print(kernel(top_k_history[i],observation,l))
k_t_X_list.append(np.expand_dims(np.expand_dims((kernel(top_k_history[0],top_k_history[i+1],l)), axis=0), axis=0))
# print(k_t_X_list[0].item())
# k_t_X = np.expand_dims(np.asarray(k_t_X_list), axis=0)
k_t_X = np.expand_dims(np.expand_dims((kernel(top_k_history[0],top_k_history[0],l)), axis=0), axis=0)
for i in range(k-1):
# temp = np.expand_dims(np.expand_dims(np.asarray(kernel(observation,top_k_history[i+1],l)), axis=0), axis=0)
k_t_X = np.concatenate([k_t_X, k_t_X_list[i]], axis=1)
k_t_X_first = k_t_X
k_t_X_list_list = []
for j in range(k-1):
k_t_X_list = []
for i in range(k-1):
# print(kernel(top_k_history[i],observation,l))
k_t_X_list.append(np.expand_dims(np.expand_dims((kernel(top_k_history[j+1],top_k_history[i+1],l)), axis=0), axis=0))
# print(k_t_X_list[0].item())
# k_t_X = np.expand_dims(np.asarray(k_t_X_list), axis=0)
k_t_X = np.expand_dims(np.expand_dims((kernel(top_k_history[j+1],top_k_history[0],l)), axis=0), axis=0)
for i in range(k-1):
# temp = np.expand_dims(np.expand_dims(np.asarray(kernel(observation,top_k_history[i+1],l)), axis=0), axis=0)
k_t_X = np.concatenate([k_t_X, k_t_X_list[i]], axis=1)
k_t_X_list_list.append(k_t_X)
for i in range(k-1):
k_t_X_first = np.concatenate([k_t_X_first, k_t_X_list_list[i]], axis=0)
return k_t_X_first
k=10
l=19
top_k_history = []
for i in range(10):
top_k_history.append(np.random.rand(10))
jac = jacobian(kernel_matrixx)
jac(top_k_history)
the error I got is:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_15016/2419460232.py in <module>
1 jac = jacobian(kernel_matrixx)
----> 2 jac(top_k_history)
~\Anaconda3\envs\unlearning\lib\site-packages\autograd\wrap_util.py in nary_f(*args, **kwargs)
18 else:
19 x = tuple(args[i] for i in argnum)
---> 20 return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs)
21 return nary_f
22 return nary_operator
~\Anaconda3\envs\unlearning\lib\site-packages\autograd\differential_operators.py in jacobian(fun, x)
57 vjp, ans = _make_vjp(fun, x)
58 ans_vspace = vspace(ans)
---> 59 jacobian_shape = ans_vspace.shape + vspace(x).shape
60 grads = map(vjp, ans_vspace.standard_basis())
61 return np.reshape(np.stack(grads), jacobian_shape)
TypeError: can only concatenate tuple (not "list") to tuple
I am already aware that I cannot create a zero matrix (or identity matrix) then fill in the value with nested for loop. Therefore I create np.array and then concat them. I use the same approach for compute the grad of some other output of the same kernel matrix and it did work so I'm not sure why it didn't work for Jacobian.
Edit: the error now should be reproducible
There is a datatype problem. I your code top_k_history is of type list and contains 10 1D-arrays, each of length 10. If you convert this into 1 2D-array of shape (10, 10), then the error should vanish:
# <original code except the last line>
top_k_history = np.array(top_k_history) # new
jac(top_k_history) # original last line
Related
I am trying to speed up the below operation by doing some sort of matrix/vector-multiplication, can anyone see a nice quick solution?
It should also work for a special case where a tensor has shape 0 (torch.Size([])) but i am not able to initialize such a tensor.
See the image below for the type of tensor i am referring to:
tensor to add to test
def adstock_geometric(x: torch.Tensor, theta: float):
x_decayed = torch.zeros_like(x)
x_decayed[0] = x[0]
for xi in range(1, len(x_decayed)):
x_decayed[xi] = x[xi] + theta * x_decayed[xi - 1]
return x_decayed
def adstock_multiple_samples(x: torch.Tensor, theta: torch.Tensor):
listtheta = theta.tolist()
if isinstance(listtheta, float):
return adstock_geometric(x=x,
theta=theta)
x_decayed = torch.zeros((100, 112, 1))
for idx, theta_ in enumerate(listtheta):
x_decayed_one_entry = adstock_geometric(x=x,
theta=theta_)
x_decayed[idx] = x_decayed_one_entry
return x_decayed
if __name__ == '__main__':
ones = torch.tensor([1])
hundreds = torch.tensor([idx for idx in range(100)])
x = torch.tensor([[idx] for idx in range(112)])
ones = adstock_multiple_samples(x=x,
theta=ones)
hundreds = adstock_multiple_samples(x=x,
theta=hundreds)
print(ones)
print(hundreds)
I came up with the following, which is 40 times faster on your example:
import torch
def adstock_multiple_samples(x: torch.Tensor, theta: torch.Tensor):
arange = torch.arange(len(x))
powers = (arange[:, None] - arange).clip(0)
return ((theta[:, None, None] ** powers[None, :, :]).tril() * x).sum(-1)
It behaves as expected:
>>> x = torch.arange(112)
>>> theta = torch.arange(100)
>>> adstock_multiple_samples(x, theta)
... # the same output
Note that I considered that x was a 1D-tensor, as for your example the second dimension was not needed.
It also works with theta = torch.empty((0,)), and it returns an empty tensor.
This was my piece of code initially :
Here X is the array of data points with dimensions (m x n) where m is number of data points to predict, and n is number of features without the bias term.
y is the data labels with shape (m,)
lambda_ is the regularization term.
from scipy import optimize
def oneVsAll(X,y,num_labels,lambda_):
#used to find the optimal parametrs theta for each label against the others
#X (m,n)
#y (m,)
#num_labels : possible number of labels
#lambda_ : regularization param
#all_theta : trained param for logistic reg for each class
#hence (k,n+1) where k is #labels and n+1 is #features with bias
m,n = X.shape
all_theta = np.array((num_labels,n+1))
X = np.concatenate([np.ones((m,1)),X],axis = 1)
for k in np.arange(num_labels):
#y == k will generate a list with shape of y,but 1 only for index with value same as k and rest with 0
initial_theta = np.zeros(n+1)
options = {"maxiter" : 50}
res = optimize.minimize(lrCostFunction,
initial_theta,args = (X,y==k,lambda_),
jac = True,method = 'CG',
options = options)
all_theta[k] = res.x
return all_theta
lambda_ = 0.1
all_theta = oneVsAll(X,y,num_labels,lambda_)
The error I got was :
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-45-f9501694361e> in <module>()
1 lambda_ = 0.1
----> 2 all_theta = oneVsAll(X,y,num_labels,lambda_)
<ipython-input-44-05a9b582ccaf> in oneVsAll(X, y, num_labels, lambda_)
20 jac = True,method = 'CG',
21 options = options)
---> 22 all_theta[k] = res.x
23 return all_theta
ValueError: setting an array element with a sequence.
Then after debugging, I changed the code to :
from scipy import optimize
def oneVsAll(X,y,num_labels,lambda_):
#used to find the optimal parametrs theta for each label against the others
#X (m,n)
#y (m,)
#num_labels : possible number of labels
#lambda_ : regularization param
#all_theta : trained param for logistic reg for each class
#hence (k,n+1) where k is #labels and n+1 is #features with bias
m,n = X.shape
all_theta = np.array((num_labels,n+1),dtype = "object")
X = np.concatenate([np.ones((m,1)),X],axis = 1)
for k in np.arange(num_labels):
#y == k will generate a list with shape of y,but 1 only for index with value same as k and rest with 0
initial_theta = np.zeros(n+1)
options = {"maxiter" : 50}
res = optimize.minimize(lrCostFunction,
initial_theta,args = (X,y==k,lambda_),
jac = True,method = 'CG',
options = options)
all_theta[k] = res.x
return all_theta
Now the error I am getting is :
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-47-f9501694361e> in <module>()
1 lambda_ = 0.1
----> 2 all_theta = oneVsAll(X,y,num_labels,lambda_)
<ipython-input-46-383fc22e26cc> in oneVsAll(X, y, num_labels, lambda_)
20 jac = True,method = 'CG',
21 options = options)
---> 22 all_theta[k] = res.x
23 return all_theta
IndexError: index 2 is out of bounds for axis 0 with size 2
How can I correct this?
You create all_theta running:
all_theta = np.array((num_labels,n+1),dtype = "object")
This instruction actually creates an array containig just 2 elements
(the shape is (2,)), containing two passed values, whereas you probably
intend to pass the shape of the array to be created.
Change this instruction to:
all_theta = np.empty((num_labels,n+1))
Specification of dtype (in my opinion) is not necessary.
I'm following an excellent medium article: https://towardsdatascience.com/k-medoids-clustering-on-iris-data-set-1931bf781e05 to implement kmedoids from scratch. There is a place in the code where each pixel's distance to the medoid centers is calculated and it is VERY slow. It has numpy.linalg.norm inside a loop. Is there a way to optimize this with numpy.linalg.norm or with numpy broadcasting or scipy.spatial.distance.cdist and np.argmin to do the same thing?
###helper function here###
def compute_d_p(X, medoids, p):
m = len(X)
medoids_shape = medoids.shape
# If a 1-D array is provided,
# it will be reshaped to a single row 2-D array
if len(medoids_shape) == 1:
medoids = medoids.reshape((1,len(medoids)))
k = len(medoids)
S = np.empty((m, k))
for i in range(m):
d_i = np.linalg.norm(X[i, :] - medoids, ord=p, axis=1)
S[i, :] = d_i**p
return S
this is where the slowdown occurs
for datap in cluster_points:
new_medoid = datap
new_dissimilarity= np.sum(compute_d_p(X, datap, p))
if new_dissimilarity < avg_dissimilarity :
avg_dissimilarity = new_dissimilarity
out_medoids[i] = datap
Full code below. All credits to the article author.
# Imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn import datasets
from sklearn.decomposition import PCA
# Dataset
iris = datasets.load_iris()
data = pd.DataFrame(iris.data,columns = iris.feature_names)
target = iris.target_names
labels = iris.target
#Scaling
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
data = pd.DataFrame(scaler.fit_transform(data), columns=data.columns)
#PCA Transformation
from sklearn.decomposition import PCA
pca = PCA(n_components=3)
principalComponents = pca.fit_transform(data)
PCAdf = pd.DataFrame(data = principalComponents , columns = ['principal component 1', 'principal component 2','principal component 3'])
datapoints = PCAdf.values
m, f = datapoints.shape
k = 3
def init_medoids(X, k):
from numpy.random import choice
from numpy.random import seed
seed(1)
samples = choice(len(X), size=k, replace=False)
return X[samples, :]
medoids_initial = init_medoids(datapoints, 3)
def compute_d_p(X, medoids, p):
m = len(X)
medoids_shape = medoids.shape
# If a 1-D array is provided,
# it will be reshaped to a single row 2-D array
if len(medoids_shape) == 1:
medoids = medoids.reshape((1,len(medoids)))
k = len(medoids)
S = np.empty((m, k))
for i in range(m):
d_i = np.linalg.norm(X[i, :] - medoids, ord=p, axis=1)
S[i, :] = d_i**p
return S
S = compute_d_p(datapoints, medoids_initial, 2)
def assign_labels(S):
return np.argmin(S, axis=1)
labels = assign_labels(S)
def update_medoids(X, medoids, p):
S = compute_d_p(points, medoids, p)
labels = assign_labels(S)
out_medoids = medoids
for i in set(labels):
avg_dissimilarity = np.sum(compute_d_p(points, medoids[i], p))
cluster_points = points[labels == i]
for datap in cluster_points:
new_medoid = datap
new_dissimilarity= np.sum(compute_d_p(points, datap, p))
if new_dissimilarity < avg_dissimilarity :
avg_dissimilarity = new_dissimilarity
out_medoids[i] = datap
return out_medoids
def has_converged(old_medoids, medoids):
return set([tuple(x) for x in old_medoids]) == set([tuple(x) for x in medoids])
#Full algorithm
def kmedoids(X, k, p, starting_medoids=None, max_steps=np.inf):
if starting_medoids is None:
medoids = init_medoids(X, k)
else:
medoids = starting_medoids
converged = False
labels = np.zeros(len(X))
i = 1
while (not converged) and (i <= max_steps):
old_medoids = medoids.copy()
S = compute_d_p(X, medoids, p)
labels = assign_labels(S)
medoids = update_medoids(X, medoids, p)
converged = has_converged(old_medoids, medoids)
i += 1
return (medoids,labels)
results = kmedoids(datapoints, 3, 2)
final_medoids = results[0]
data['clusters'] = results[1]
There's a good chance numpy's broadcasting capabilities will help. Getting broadcasting to work in 3+ dimensions is a bit tricky, and I usually have to resort to a bit of trial and error to get the details right.
The use of linalg.norm here compounds things further, because my version of the code won't give identical results to linalg.norm for all inputs. But I believe it will give identical results for all relevant inputs in this case.
I've added some comments to the code to explain the thinking behind certain details.
def compute_d_p_broadcasted(X, medoids, p):
# If a 1-D array is provided,
# it will be reshaped to a single row 2-D array
if len(medoids.shape) == 1:
medoids = medoids.reshape((1,len(medoids)))
# In general, broadcasting n-dim arrays requires that the last
# dim of the first array be a singleton dimension, and that the
# first dim of the second array be a singleton dimension. We can
# quickly accomplish that by slicing with `None` in the appropriate
# places. (`np.newaxis` is a slightly more self-documenting way
# of spelling `None`, but I rarely bother.)
# In this case, the shapes of the other two dimensions also
# have to align in the same way you'd expect for a dot product.
# So we pass `medoids.T`.
diff = np.abs(X[:, :, None] - medoids.T[None, :, :])
# The last tricky bit is to figure out which axis to sum. Right
# now, the array is a 3-dimensional array, with the first
# dimension corresponding to the rows of `X` and the last
# dimension corresponding to the columns of `medoids.T`.
# The middle dimension corresponds to the underlying dimensionality
# of the space; that's what we want to sum for a sum of squares.
# (Or sum of cubes for L3 norm, etc.)
return (diff ** p).sum(axis=1)
def compute_d_p(X, medoids, p):
m = len(X)
medoids_shape = medoids.shape
# If a 1-D array is provided,
# it will be reshaped to a single row 2-D array
if len(medoids_shape) == 1:
medoids = medoids.reshape((1,len(medoids)))
k = len(medoids)
S = np.empty((m, k))
for i in range(m):
d_i = np.linalg.norm(X[i, :] - medoids, ord=p, axis=1)
S[i, :] = d_i**p
return S
# A couple of simple tests:
X = np.array([[ 1.0, 2, 3],
[ 4, 5, 6],
[ 7, 8, 9],
[10, 11, 12]])
medoids = X[[0, 2], :]
np.allclose(compute_d_p(X, medoids, 2),
compute_d_p_broadcasted(X, medoids, 2))
# Returns True
np.allclose(compute_d_p(X, medoids, 3),
compute_d_p_broadcasted(X, medoids, 3))
# Returns True
Of course, these tests don't tell whether this actually gives a significant speedup. You'll have to check that yourself for the relevant use-case. But I suspect it will at least help.
I have a numpy array that is labelled using scipy connected component labelling.
import numpy
from scipy import ndimage
a = numpy.zeros((8,8), dtype=numpy.int)
a[1,1] = a[1,2] = a[2,1] = a[2,2] = a[3,1] = a[3,2] = 1
a[5,5] = a[5,6] = a[6,5] = a[6,6] = a[7,5] = a[7,6] = 1
lbl, numpatches = ndimage.label(a)
I want to apply a custom function (calculation of a specific value) over all labels within the labelled array.
Similar as for instance the ndimage algebra functions:
ndimage.sum(a,lbl,range(1,numpatches+1))
( Which in this case returns me the number of values for each label [6,6]. )
Is there a way to do this?
You can pass an arbitrary function to ndimage.labeled_comprehension, which is roughly equivalent to
[func(a[lbl == i]) for i in index]
Here is the labeled_comprehension-equivalent of ndimage.sum(a,lbl,range(1,numpatches+1)):
import numpy as np
from scipy import ndimage
a = np.zeros((8,8), dtype=np.int)
a[1,1] = a[1,2] = a[2,1] = a[2,2] = a[3,1] = a[3,2] = 1
a[5,5] = a[5,6] = a[6,5] = a[6,6] = a[7,5] = a[7,6] = 1
lbl, numpatches = ndimage.label(a)
def func(x):
return x.sum()
print(ndimage.labeled_comprehension(a, lbl, index=range(1, numpatches+1),
func=func, out_dtype='float', default=None))
# [6 6]
I'm trying to do some bayesian probit code using data augmentation. I can get it to work if I loop over the rows of the output matrix, but I'd like to vectorize it and do it all in one shot (presumably that's faster).
import numpy as np
from numpy import random
import statsmodels.api as sm
from scipy import stats
from scipy.stats import norm, truncnorm
##################################
### Create some simulated data ###
num_leg = 50
num_bills = 20
a = np.random.uniform(-1,1,num_bills).reshape(num_bills, 1)
b = np.random.uniform(-2,2,num_bills).reshape(num_bills, 1)
x = np.random.standard_normal(num_leg).reshape(num_leg, 1)
ystar_base = a + np.dot(b,x.T)
epsilon = np.random.standard_normal(num_leg * num_bills).reshape(num_bills, num_leg)
ystar = ystar_base + epsilon
y = 1*(ystar >0)
### Initialize some stuff I need ###
avec = [0]*num_bills # These are bill parameters
bvec = [0]*num_bills
betavec = [np.matrix(zip(avec,bvec))]
xvec = [0]*num_leg # these are legislator parameters
_ones = np.ones(num_leg)
def init_y(mat): # initialize a latent y matrix
if mat==1: return truncnorm.rvs(0,10000)
else: return truncnorm.rvs(-10000,0)
vectorize_y = np.vectorize(init_y)
latent_y = np.matrix(vectorize_y(y))
burn = 500 # How long to run the MCMC
runs = 500
### define the functions ###
def sample_params(xnow,ynow): # This is the function I'd like to vectorize
if type(xnow) == list:
xnow = np.array(xnow)
if type(ynow) == list:
ynow = np.array(ynow)
ynow = ynow.T #reshape(ynow.shape[0],1)
sigma = np.linalg.inv(np.dot(xnow.T,xnow)) ###This is the line that produces an error###
xy = np.dot(xnow.T,ynow)
mu = np.dot(sigma, xy) # this is just (x'x)inv x'y
return np.random.multivariate_normal(np.array(mu).flatten(), sigma)
vecparams = np.vectorize(sample_params)
def get_mu(xnow, bnow): # getting the updated mean to draw the latent ys
if type(xnow) == list:
xnow = np.array(xnow)
if type(bnow) == list:
bnow = np.array(bnow)
mu = np.dot(xnow,bnow.T)
mu = np.matrix(mu)
return mu
def sample_y(mu, ynow): # generate latent y matrix
if ynow==1:
a, b = (0 - mu),(10000-mu)
else:
a, b = (-10000 - mu),(0-mu)
return truncnorm.rvs(a,b)
vector_sample = np.vectorize(sample_y) # I'd like to be able to do something like this
### Here's the MCMC loop with the internal loop over rows(bills)
for i in range(burn+runs):
this_beta = []
this_x = []
this_y = []
for j in range(num_bills): #I'd like to get rid of this loop
ex = zip(x_ones, x)
newbeta = sample_params(ex, latent_y[j])
this_beta.append(newbeta)
#ex = np.array(zip(x_ones, x))
#this_beta = vecparams(ex, latent_y[:,]) # and call the vectorized function here
betavec.append(this_beta)
#Note, I can vectorize the latent outputs easily enough here
mean = get_mu(ex, betavec[-1])
latent_y = np.matrix(vector_sample(mean, np.matrix(y).T).T.reshape(latent_y.shape[0], latent_y.shape[1]))
### Now a bit of code to check to see if I've recovered what I want ###
test_beta = [zip(*(z)) for z in betavec[burn:]]
test_a = np.array([z[0] for z in test_beta])
test_b = np.array([z[1] for z in test_beta])
amean = test_a.sum(axis = 0)/float(runs)
bmean = test_b.sum(axis = 0)/float(runs)
print 'a mean'
print np.corrcoef([amean, np.array(a)])
print
print 'b mean'
print np.corrcoef([bmean, np.array(b)])
If I comment out the loop and use the commented out lines just above, I get the following error at the line I indicated earlier (the one that defines sigma):
LinAlgError: 0-dimensional array given. Array must be at least two-dimensional