I'm trying to compute circular cross-correlation of two signals with Theano to use it in further calculation of loss that I would optimize over. But I'm not quite sure how to do that.
It is defined as following:
(f * g)[n] = sum_k f[k]g[k+n]
ccc[n] = \sum_k (f*g)[n-kN]
"periodic" summation or like "for each k-th component".
I could do an ordinary correlation and then perform periodic summation, but it's not quite clear how to do that (periodic summation) symbolically (using scan, probably?)
conv2d = T.signal.conv.conv2d
x = T.dmatrix()
y = T.dmatrix()
veclen = x.shape[1]
corr_expr = conv2d(x, y[:, ::-1], image_shape=(1, veclen), border_mode='full')
# circ_corr = T.sum([corr_expr[k::veclen] for k in T.arange(veclen)])
corr = theano.function([x, y], outputs=circ_corr)
corr( np.array([[2, 3, 5]]), np.array([[7, 11, 13]]) )
or use circular cross-correlation theorem and compute as a iFFT(FFT(x)*FFT(y)):
import theano.sandbox.fourier as dft
x = T.dmatrix()
y = T.dvector()
veclen = x.shape[1]
exp = T.real(
dft.ifft(
dft.fft(x, veclen, axis=1)
* dft.fft(y[::-1], y.shape[0], axis=1).reshape((1, -1)),
veclen, axis=1
)
)[:, ::-1]
f = theano.function([x, y], outputs=exp)
f(np.array([[2, 3, 5], [3, 4, 4], [5, 6, 7]]), np.array([7, 11, 13]) )
but in this case I can't actually compute a gradient because gradient for ifft (and all functions that has something to do with complex numbers in general, afaik) is not implemented yet, I guess (aborts with an error: Elemwise{real,no_inplace}.grad illegally returned an integer-valued variable. (Input index 0, dtype complex128))
Here's a working solution I came up with (definitely not optimal as soon as FFT is not used):
def circular_crosscorelation(X, y):
"""
Input:
symbols for X [n, m]
and y[m,]
Returns:
symbol for circular cross corelation of each of row in X with
cc[n, m]
"""
n, m = X.shape
corr_expr = T.signal.conv.conv2d(X, y[::-1].reshape((1, -1)), image_shape=(1, m), border_mode='full')
corr_len = corr_expr.shape[1]
pad = m - corr_len%m
v_padded = T.concatenate([corr_expr, T.zeros((n, pad))], axis=1)
circ_corr_exp = T.sum(v_padded.reshape((n, v_padded.shape[1] / m, m)), axis=1)
return circ_corr_exp[:, ::-1]
X = T.dmatrix()
y = T.dmatrix()
cc = theano.function([X, y], circular_crosscorelation(X, y))
print cc( np.array([[2, 3, 5], [4, 5, 6]]), np.array([[7, 11, 13]]) )
returns
[[ 94. 108. 108.]
[ 149. 157. 159.]]
as expected.
And can be analytically differentiated:
score = T.sum(circ_corr_exp**2)
grad = T.grad(score, x)
g = theano.function([x, y], outputs=grad)
print g( np.array([[2, 3, 5], [4, 5, 6]]), np.array([[7, 11, 13]]) )
>> [[ 6332. 6388. 6500.]
>> [ 9554. 9610. 9666.]]
here's also few more options (through direct circulant calculation) and time-comparation:
def circulant_np(v):
row = np.arange(len(v))
col = -np.arange(len(v))
idx = (row[:, np.newaxis] + col)%len(v)
return v[idx]
print circulant_np(np.array([1, 2, 3, 5]))
def c_corr_np(a, b):
return circulant_np(a).dot(b[::-1])
def circulant_t(v):
row = T.arange(v.shape[0])
col = -T.arange(v.shape[0])
idx = (row.reshape((-1, 1)) + col)%v.shape[0]
return v[idx]
def c_corr_t_f(a, b):
""" 1d correlation using circulant matrix """
return circulant_t(a).dot(b[::-1])
a = T.dvector('a')
b = T.dvector('b')
c_corr_t = theano.function([a, b], c_corr_t_f(a, b))
print c_corr_np(np.array([2, 3, 5]), np.array([7, 11, 13]))
print c_corr_t(np.array([2, 3, 5]), np.array([7, 11, 13]))
print c_corr( np.array([[2, 3, 5]]), np.array([[7, 11, 13]]) )
%timeit c_corr_np(np.array([2, 3, 5]), np.array([7, 11, 13]))
%timeit c_corr_t(np.array([2, 3, 5]), np.array([7, 11, 13]))
%timeit c_corr( np.array([[2, 3, 5]]), np.array([[7, 11, 13]]) ) # = circular_crosscorelation
which gives
10000 loops, best of 3: 30.6 µs per loop
10000 loops, best of 3: 132 µs per loop
10000 loops, best of 3: 149 µs per loop
and inverse cross-corr:
def inverse_circular_crosscorelation(y):
"""
Input:
symbol for y[1, m]
Returns:
symbol for y_inv s.t.
cc( y, y_inv ) = (1, 0 ... 0)
"""
A = circulant_t(y.reshape((-1, )))
b = T.concatenate([T.zeros((y.shape[1] - 1, )), T.ones((1, ))]).reshape((-1, 1))
return T.nlinalg.matrix_inverse(A).dot(b).reshape((1, -1))[:, ::-1]
Related
I am trying to find the minimum number of hops to get from the value of one cell w to the value of another cell v in the following matrix X using Python.
Is there an efficient way to do this?
import numpy as np
from numpy.typing import NDArray
def manhattan_distance(X: NDArray[int], w: int, v: int) -> int:
# something
return distance
X = np.array([
[1, 2, 2, 3, 3],
[1, 2, 2, 4, 4],
[5, 5, 6, 6, 6],
[7, 8, 9, 9, 9],
[10, 10, 10, 10, 10],
]).astype(np.int_)
# Expected result
manhattan_distance(X, 1, 2) # ①
-> 1
manhattan_distance(X, 2, 3) # ②
-> 1
manhattan_distance(X, 3, 6) # ③
-> 2
manhattan_distance(X, 3, 5) # ④
-> 4
I tried to implement it as follows, but it seems to be quite slow.
import numpy as np
from numpy.typing import NDArray
def manhattan_distance(X: NDArray[int], w: int, v: int) -> int:
xx, yy = np.where(X == w)
xx_, yy_ = np.where(X == v)
distance = int(min_dist(xx, xx_) + min_dist(yy, yy_))
return distance
def min_dist(xx, xx_):
min_dist = np.inf
for i in xx:
for j in xx_:
dist = np.sqrt((i - j)**2)
min_dist = dist if dist < min_dist else min_dist
return min_dist
Is there any way to speed this calculation up?
Use cdist to compute all pairwise distances, to find the values' indices use np.argwhere
from scipy.spatial.distance import cdist
import numpy as np
from numpy.typing import NDArray
X = np.array([
[1, 2, 2, 3, 3],
[1, 2, 2, 4, 4],
[5, 5, 6, 6, 6],
[7, 8, 9, 9, 9],
[10, 10, 10, 10, 10],
]).astype(np.int32)
def manhattan_distance(X: NDArray[int], w: int, v: int) -> int:
return cdist(np.argwhere(X == w), np.argwhere(X == v), "cityblock").min()
print(manhattan_distance(X, 1, 2))
print(manhattan_distance(X, 2, 3))
print(manhattan_distance(X, 3, 6))
print(manhattan_distance(X, 3, 5))
Output
1.0
1.0
2.0
4.0
Just adding timings for different matrix sizes to show OP that #Dani Mesejo answer indeed is much faster. For small matrices differences will be small of course.
def manhattan_distance_dani_masejo(X, w: int, v: int) -> int:
return cdist(np.argwhere(X == w), np.argwhere(X == v), "cityblock").min()
def manhattan_distance_ugen(X, w: int, v: int) -> int:
xx, yy = np.where(X == w)
xx_, yy_ = np.where(X == v)
distance = int(min_dist(xx, xx_) + min_dist(yy, yy_))
return distance
def min_dist(xx, xx_):
min_dist = np.inf
for i in xx:
for j in xx_:
dist = np.sqrt((i - j)**2)
min_dist = dist if dist < min_dist else min_dist
return min_dist
def gen_data(matrix_size):
return np.random.randint(0, matrix_size, (matrix_size, matrix_size), dtype=np.int32)
for matrix_size in [5, 50, 500]:
print('Matrix size: {}'.format(matrix_size))
X = gen_data(matrix_size)
print('ugen: ', timeit.timeit("manhattan_distance_ugen(X, np.random.randint(0, matrix_size), np.random.randint(0, matrix_size))",
"from __main__ import manhattan_distance_ugen, X, matrix_size; import numpy as np",
number=10))
print('dani masejo: ', timeit.timeit("manhattan_distance_dani_masejo(X, np.random.randint(0, matrix_size), np.random.randint(0, matrix_size))",
"from __main__ import manhattan_distance_dani_masejo, X, matrix_size; import numpy as np",
number=10))
Result:
Matrix size: 5
ugen: 0.0019634239999999914
dani masejo: 0.0006071939999999776
Matrix size: 50
ugen: 0.093326557
dani masejo: 0.0008874660000000034
Matrix size: 500
ugen: 9.112327058
dani masejo: 0.027754558000001595
I would like to plot a 4d matrix as a 2d matrix with indices:
[i][j][k][l] --> [i * nj + j][ k * nl + l]
I have a working version here.
This is working as I want, but it's not very elegant. I looked into "reshape" but this is not exactly what I'm looking for, or perhaps I am using it incorrectly.
Given a 4d array "r" with shape (100000,4), the relevant snippet I want to replace is:
def transform(i,j,k,l, s1, s2):
return [i * s1 + j, k * s2 + l]
nx = 5
ny = 11
iedges=np.linspace(0,100, nx)
jedges=np.linspace(0, 20, ny)
bins = ( iedges,jedges,iedges,jedges )
H, edges = np.histogramdd(r, bins=bins )
H2 = np.zeros(( (nx-1)*(ny-1),(nx-1)*(ny-1)))
for i in range(nx-1):
for j in range(ny-1):
for k in range(nx-1):
for l in range(ny-1):
x,y = transform(i,j,k,l,ny-1,ny-1)
H2[x][y] = H[i][j][k][l]
In this case the values of H2 will correspond to the values of H, but the entry i,j,k,l will display as i*ny + j, k * ny + l.
Example plot:
Are you sure reshape doesn't work?
I ran your code on a small random r. The nonzero terms of H are:
In [13]: np.argwhere(H)
Out[13]:
array([[0, 9, 3, 1],
[1, 1, 1, 2],
[1, 2, 1, 3],
[2, 2, 2, 3],
[3, 1, 1, 8]])
and for the transformed H2:
In [14]: np.argwhere(H2)
Out[14]:
array([[ 9, 31],
[11, 12],
[12, 13],
[22, 23],
[31, 18]])
And one of the H indices transforms to H2 indices with:
In [16]: transform(0,9,3,1,4,10)
Out[16]: [9, 31]
If I simply reshape H, I get the same array as H2:
In [17]: H3=H.reshape(40,40)
In [18]: np.argwhere(H3)
Out[18]:
array([[ 9, 31],
[11, 12],
[12, 13],
[22, 23],
[31, 18]])
In [19]: np.allclose(H2,H3)
Out[19]: True
So without delving into the details of your code, it looks to me like a simple reshape.
Looks like you can calculate i,j,k,l from x,y? This should be something like:
from functools import partial
def get_ijkl(x, y, s1, s2):
# "Reverse" of `transform`
i, j = divmod(x, s1)
k, l = divmod(y, s2)
return (i, j, k, l)
def get_2d_val(x, y, s1, s2, four_dim_array):
return four_dim_array[get_ijkl(x, y, s1, s2)]
smaller_shape = ((nx-1)*(ny-1), (nx-1)*(ny-1))
Knowing this there are several approaches possible:
numpy.fromfunction:
H3 = np.fromfunction(
partial(get_2d_val, s1=ny-1, s2=ny-1, four_dim_array=H),
shape=smaller_shape,
dtype=int,
)
assert np.all(H2 == H3)
by indexing:
indices_to_take = np.array([
[list(get_ijkl(x, y, ny-1, ny-1)) for x in range(smaller_shape[0])] for y in range(smaller_shape[1])
]).transpose()
H4 = H[tuple(indices_to_take)]
assert np.all(H2 == H4)
as answered by #hpaulj you can simply reshape array and it will be faster. But If you have some different transform and can calculate appropriate "reverse" function then using fromfunction or custom indexing will get useful
I have a function which implements the k-mean algorithm and I want to use it with DataFrames in order to take into account indexes. For the moment I use DataFrame.values and it works. Yet I don't get the indexes of the output.
def cluster_points(X, mu):
clusters = {}
for x in X:
bestmukey = min([(i[0], np.linalg.norm(x-mu[i[0]])) \
for i in enumerate(mu)], key=lambda t:t[1])[0]
try:
clusters[bestmukey].append(x)
except KeyError:
clusters[bestmukey] = [x]
return clusters
def reevaluate_centers(mu, clusters):
newmu = []
keys = sorted(clusters.keys())
for k in keys:
newmu.append(np.mean(clusters[k], axis = 0))
return newmu
def has_converged(mu, oldmu):
return (set([tuple(a) for a in mu]) == set([tuple(a) for a in oldmu]))
def find_centers(X, K):
# Initialize to K random centers
oldmu = random.sample(X, K)
mu = random.sample(X, K)
while not has_converged(mu, oldmu):
oldmu = mu
# Assign all points in X to clusters
clusters = cluster_points(X, mu)
# Reevaluate centers
mu = reevaluate_centers(oldmu, clusters)
return(mu, clusters)
For instance with thus example minimal and sufficient :
import itertools
df = pd.DataFrame(np.random.randint(0,10,size=(10, 5)), index = list(range(10)), columns=list(range(5)))
df.index.name = 'subscriber_id'
df.columns.name = 'ad_id'
I get :
find_centers(df.values, 2)
([array([ 3.8, 3. , 3.6, 2. , 3.6]),
array([ 6.8, 3.6, 5.6, 6.8, 6.8])],
{0: [array([2, 0, 5, 6, 4]),
array([1, 1, 2, 3, 3]),
array([6, 0, 4, 0, 3]),
array([7, 9, 4, 1, 7]),
array([3, 5, 3, 0, 1])],
1: [array([6, 2, 5, 9, 6]),
array([8, 9, 7, 2, 8]),
array([7, 5, 3, 7, 8]),
array([7, 1, 5, 7, 6]),
array([6, 1, 8, 9, 6])]})
I have the values but don't have the indexes.
If you want to get the array of values including the index, you can simply add the index to the columns with reset_index():
values_with_index = df.reset_index().values
Update
If what you want is to have the index on the output, but not use it during the actual clustering, you can do the following. First, pass the actual data frame object to find_centers:
find_centers(df, 2)
Then change cluster_points as follows:
def cluster_points(X, mu):
clusters = {}
for _, x in X.iterrows():
bestmukey = min([(i[0], np.linalg.norm(x-mu[i[0]]))
for i in enumerate(mu)], key=lambda t:t[1])[0]
# You can replace this try/except block with
# clusters.setdefault(bestmukey, []).append(x)
try:
clusters[bestmukey].append(x)
except KeyError:
clusters[bestmukey] = [x]
return clusters
The centers in the output will still be arrays, but the clusters will contain series objects with each row. The name property of each of these series is the index value in the data frame.
I want to do some rolling window calculation in pandas which need to deal with two columns at the same time. I'll take an simple instance to express the problem clearly:
import pandas as pd
df = pd.DataFrame({
'x': [1, 2, 3, 2, 1, 5, 4, 6, 7, 9],
'y': [4, 3, 4, 6, 5, 9, 1, 3, 1, 2]
})
windowSize = 4
result = []
for i in range(1, len(df)+1):
if i < windowSize:
result.append(None)
else:
x = df.x.iloc[i-windowSize:i]
y = df.y.iloc[i-windowSize:i]
m = y.mean()
r = sum(x[y > m]) / sum(x[y <= m])
result.append(r)
print(result)
Is there any way without for loop in pandas to solve the problem? Any help is appreciated
You can use the rolling window trick for numpy arrays and apply it to the array underlying the DataFrame.
import pandas as pd
import numpy as np
def rolling_window(a, window):
shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
strides = a.strides + (a.strides[-1],)
return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
df = pd.DataFrame({
'x': [1, 2, 3, 2, 1, 5, 4, 6, 7, 9],
'y': [4, 3, 4, 6, 5, 9, 1, 3, 1, 2]
})
windowSize = 4
rw = rolling_window(df.values.T, windowSize)
m = np.mean(rw[1], axis=-1, keepdims=True)
a = np.sum(rw[0] * (rw[1] > m), axis=-1)
b = np.sum(rw[0] * (rw[1] <= m), axis=-1)
result = a / b
The result lacks the leading None values, but they should be easy to append (in form of np.nan or after converting the result to a list).
This is probably not what you are looking for, working with pandas, but it will get the job done without loops.
Here's one vectorized approach using NumPy tools -
windowSize = 4
a = df.values
X = strided_app(a[:,0],windowSize,1)
Y = strided_app(a[:,1],windowSize,1)
M = Y.mean(1)
mask = Y>M[:,None]
sums = np.einsum('ij,ij->i',X,mask)
rest_sums = X.sum(1) - sums
out = sums/rest_sums
strided_app is taken from here.
Runtime test -
Approaches -
# #kazemakase's solution
def rolling_window_sum(df, windowSize=4):
rw = rolling_window(df.values.T, windowSize)
m = np.mean(rw[1], axis=-1, keepdims=True)
a = np.sum(rw[0] * (rw[1] > m), axis=-1)
b = np.sum(rw[0] * (rw[1] <= m), axis=-1)
result = a / b
return result
# Proposed in this post
def strided_einsum(df, windowSize=4):
a = df.values
X = strided_app(a[:,0],windowSize,1)
Y = strided_app(a[:,1],windowSize,1)
M = Y.mean(1)
mask = Y>M[:,None]
sums = np.einsum('ij,ij->i',X,mask)
rest_sums = X.sum(1) - sums
out = sums/rest_sums
return out
Timings -
In [46]: df = pd.DataFrame(np.random.randint(0,9,(1000000,2)))
In [47]: %timeit rolling_window_sum(df)
10 loops, best of 3: 90.4 ms per loop
In [48]: %timeit strided_einsum(df)
10 loops, best of 3: 62.2 ms per loop
To squeeze in more performance, we can compute the Y.mean(1) part, which is basically a windowed summation with Scipy's 1D uniform filter. Thus, M could be alternatively computed for windowSize=4 as -
from scipy.ndimage.filters import uniform_filter1d as unif1d
M = unif1d(a[:,1].astype(float),windowSize)[2:-1]
The performance gains are significant -
In [65]: %timeit strided_einsum(df)
10 loops, best of 3: 61.5 ms per loop
In [66]: %timeit strided_einsum_unif_filter(df)
10 loops, best of 3: 49.4 ms per loop
I have numpy ndarrays which could be 3 or 4 dimensional. I'd like to find maximum values and their indices in a moving subarray window with specified strides.
For example, suppose I have a 4x4 2d array and my moving subarray window is 2x2 with stride 2 for simplicity:
[[ 1, 2, 3, 4],
[ 5, 6, 7, 8],
[ 9,10,11,12],
[13,14,15,16]].
I'd like to find
[[ 6 8],
[14 16]]
for max values and
[(1,1), (3,1),
(3,1), (3,3)]
for indices as output.
Is there a concise, efficient implementation for this for ndarray without using loops?
Here's a solution using stride_tricks:
def make_panes(arr, window):
arr = np.asarray(arr)
r,c = arr.shape
s_r, s_c = arr.strides
w_r, w_c = window
if c % w_c != 0 or r % w_r != 0:
raise ValueError("Window doesn't fit array.")
shape = (r / w_r, c / w_c, w_r, w_c)
strides = (w_r*s_r, w_c*s_c, s_r, s_c)
return np.lib.stride_tricks.as_strided(arr, shape, strides)
def max_in_panes(arr, window):
w_r, w_c = window
r, c = arr.shape
panes = make_panes(arr, window)
v = panes.reshape((-1, w_r * w_c))
ix = np.argmax(v, axis=1)
max_vals = v[np.arange(r/w_r * c/w_c), ix]
i = np.repeat(np.arange(0,r,w_r), c/w_c)
j = np.tile(np.arange(0, c, w_c), r/w_r)
rel_i, rel_j = np.unravel_index(ix, window)
max_ix = i + rel_i, j + rel_j
return max_vals, max_ix
A demo:
>>> vals, ix = max_in_panes(x, (2,2))
>>> print vals
[[ 6 8]
[14 16]]
>>> print ix
(array([1, 1, 3, 3]), array([1, 3, 1, 3]))
Note that this is pretty untested, and is designed to work with 2d arrays. I'll leave the generalization to n-d arrays to the reader...