Develop Reference

Speed up multiplication of two dense tensors

I want to perform element wise multiplication between two tensors, where most of the elements are zero.
For two example tensors:
test1 = np.zeros((2, 3, 5, 6))
test1[0, 0, :, 2] = 4
test1[0, 1, [2, 4], 1] = 7
test1[0, 2, 2, :] = 2
test1[1, 0, 4, 1:3] = 5
test1[1, :, 0, 1] = 3
and,
test2 = np.zeros((5, 6, 4, 7))
test2[2, 2, 2, 4] = 4
test2[0, 1, :, 1] = 3
test2[4, 3, 2, :] = 6
test2[1, 0, 3, 1:3] = 1
test2[3, :, 0, 1] = 2
the calulation I need is:
result = test1[..., None, None] * test2[None, None, ...]
In the actual use case I am coding for, the tensors can have more dimensions and much longer lengths in some of the dimensions, so while the multiplication is reasonably quick, I would like to utilise the fact that most of the elements are zero.
My first thought was to make a sparse representation of each tensor.
coords1 = np.nonzero(test1)
shape1 = test1.shape
test1_squished = test1[coords1]
coords1 = np.array(coords1)
coords2 = np.nonzero(test2)
shape2 = test2.shape
test2_squished = test2[coords2]
coords2 = np.array(coords2)
Here there is enough information to perform the multiplication, by comparing the coordinates along the equal axes and multiplying if they are the same.
I have a function for adding a new axis,
def new_axis(coords, shape, axis):
new_coords = np.zeros((len(coords)+1, len(coords[0])))
new_index = np.delete(np.arange(0, len(coords)+1), axis)
new_coords[new_index] = coords
coords = new_coords
new_shape = np.zeros(len(new_coords), dtype=int)
new_shape[new_index] = shape
new_shape[axis] = 1
new_shape = np.array(new_shape)
return coords, new_shape
and for performing the multiplication,
def multiply(coords1, shape1, array1, coords2, shape2, array2): #all inputs should be numpy arrays
if np.array_equal( shape1, shape2 ):
index1 = np.nonzero( ( coords1.T[:, None, :] == coords2.T ).all(-1).any(-1) )[0]
index2 = np.nonzero( ( coords2.T[:, None, :] == coords1.T ).all(-1).any(-1) )[0]
array = array1[index1] * array2[index2]
coords = ( coords1.T[index] ).T
shape = shape1
else:
if len(shape1) == len(shape2):
equal_index = np.nonzero( ( shape1 == shape2 ) )[0]
not_equal_index = np.nonzero( ~( shape1 == shape2 ) )[0]
if np.logical_or( ( shape1[not_equal_index] == 1 ), ( shape2[not_equal_index] == 1 ) ).all():
#if where not equal, one of them = 1 -> can broadcast
# compare dimensions with same length, if equal then multiply corresponding elements
multiply_index1 = np.nonzero(
( coords1[equal_index].T[:, None, :] == coords2[equal_index].T ).all(-1).any(-1)
)[0]
# would like vecotrised version of below
array = []
coords = []
for index in multiply_index1:
multiply_index2 = np.nonzero( ( (coords2[equal_index]).T == (coords1[equal_index]).T[index] ).all(-1) )[0]
array.append( test_squished[index] * test2_squished[multiply_index2] )
temp = np.zeros((6, len(multiply_index2)))
temp[not_equal_index] = ((coords1[not_equal_index].T[index]).T + (coords2[not_equal_index].T[multiply_index2])).T
if len(multiply_index2)==1:
temp[equal_index] = coords1[equal_index].T[index].T[:, None]
else:
temp[equal_index] = np.repeat( coords1[equal_index].T[index].T[:, None], len(multiply_index2), axis=-1)
coords.append(temp)
array = np.concatenate(array)
coords = np.concatenate(coords, axis=-1)
shape = shape1
shape[np.where(shape==1)] = shape2[np.where(shape==1)]
else:
print("error")
else:
print("error")
return array, coords, shape
However the multiply function is very inefficient and so I lose any gain of going to the sparse representation.
Is there an elegant vectorised approach to the multiply function? Or is there a better solution than this sparse tensor idea?
Thanks in advance.

How do I modify this function to return a 4d array instead of 3d?

I created this function that takes in a dataframe to return an ndarrays of input and label.
def transform_to_array(dataframe, chunk_size=100):
grouped = dataframe.groupby('id')
# initialize accumulators
X, y = np.zeros([0, 1, chunk_size, 4]), np.zeros([0,]) # original inpt shape: [0, 1, chunk_size, 4]
# loop over each group (df[df.id==1] and df[df.id==2])
for _, group in grouped:
inputs = group.loc[:, 'A':'D'].values
label = group.loc[:, 'label'].values[0]
# calculate number of splits
N = (len(inputs)-1) // chunk_size
if N > 0:
inputs = np.array_split(
inputs, [chunk_size + (chunk_size*i) for i in range(N)])
else:
inputs = [inputs]
# loop over splits
for inpt in inputs:
inpt = np.pad(
inpt, [(0, chunk_size-len(inpt)),(0, 0)],
mode='constant')
# add each inputs split to accumulators
X = np.concatenate([X, inpt[np.newaxis, np.newaxis]], axis=0)
y = np.concatenate([y, label[np.newaxis]], axis=0)
return X, y
The function returned X of shape (n_samples, 1, chunk_size, 4) and y of shape (n_samples, ).
For examples:
N = 10_000
id = np.arange(N)
labels = np.random.randint(5, size=N)
df = pd.DataFrame(data = np.random.randn(N, 4), columns=list('ABCD'))
df['label'] = labels
df.insert(0, 'id', id)
df = df.loc[df.id.repeat(157)]
df.head()
id A B C D label
0 0 -0.571676 -0.337737 -0.019276 -1.377253 1
0 0 -0.571676 -0.337737 -0.019276 -1.377253 1
0 0 -0.571676 -0.337737 -0.019276 -1.377253 1
0 0 -0.571676 -0.337737 -0.019276 -1.377253 1
0 0 -0.571676 -0.337737 -0.019276 -1.377253 1
To generate the followings:
X, y = transform_to_array(df)
X.shape # shape of input
(20000, 1, 100, 4)
y.shape # shape of label
(20000,)
This function works fine as intended, however, it takes long time to finish execution:
start_time = time.time()
X, y = transform_to_array(df)
end_time = time.time()
print(f'Time taken: {end_time - start_time} seconds.')
Time taken: 227.83956217765808 seconds.
In attempt to improve performance of the function (minimise exec. time), I created the following modified func:
def modified_transform_to_array(dataframe, chunk_size=100):
# group data by 'id'
grouped = dataframe.groupby('id')
# initialize lists to store transformed data
X, y = [], []
# loop over each group (df[df.id==1] and df[df.id==2])
for _, group in grouped:
# get input and label data for group
inputs = group.loc[:, 'A':'D'].values
label = group.loc[:, 'label'].values[0]
# calculate number of splits
N = (len(inputs)-1) // chunk_size
if N > 0:
# split input data into chunks
inputs = np.array_split(
inputs, [chunk_size + (chunk_size*i) for i in range(N)])
else:
inputs = [inputs]
# loop over splits
for inpt in inputs:
# pad input data to have a chunk size of chunk_size
inpt = np.pad(
inpt, [(0, chunk_size-len(inpt)),(0, 0)],
mode='constant')
# add each input split and corresponding label to lists
X.append(inpt)
y.append(label)
# convert lists to numpy arrays
X = np.array(X)
y = np.array(y)
return X, y
At first, it seems like I succeeded reducing time taken:
start_time = time.time()
X2, y2 = modified_transform_to_array(df)
end_time = time.time()
print(f'Time taken: {end_time - start_time} seconds.')
Time taken: 5.842168092727661 seconds.
However, the result is that it changes the shape of the intended returned value.
X2.shape # this should be (20000, 1, 100, 4)
(20000, 100, 4)
y.shape # this is fine
(20000, )
Question
How do I modify modified_transform_to_array() to return the intended array shape (n_samples, 1, chunk_size, 4) since it is much faster?

You can simply reshape the X just before returning it at the end of modified_transform_to_array(), e.g.:
def modified_transform_to_array( ... ):
...
# convert lists to numpy arrays
X = np.array(X)
y = np.array(y)
X = X.reshape((X.shape[0], 1, *X.shape[1:])) # <-- THIS LINE
return X, y
or, equivalently:
X = X.reshape((X.shape[0], 1, X.shape[1], X.shape[2]))
As pointed out in #MSS's answer, you can achieve the same reshaping result also with slicing, by starting from a a slicing where you are selecting the whole array (i.e. X[:, :, :]) and inserting a None (or its more explicit alias np.newaxis) in the position where you want to augment the number of dimensions:
X = X[:, None, :, :]
X = X[:, np.newaxis, :, :]
The last two slicing can be replaced by an Ellipsis ... which essentially produces enough full-axis slicing (i.e. : or slice(None)) to fill the whole array dimensions.
X = X[:, None, ...]
X = X[:, np.newaxis, ...]
You may want to read the relevant section of NumPy's user guide for further explanations on the use of None and Ellipsis in NumPy's slicing.

Add a new axis to your X just before returning it in modified_transform_to_array, e.g.:
def modified_transform_to_array( ... ):
...
# convert lists to numpy arrays
X = np.array(X)
y = np.array(y)
X = X[:, np.newaxis, ...] # <---in this place
# X = X[:, None, :, :]
return X, y

Tensorflow: creating a matrix with variables as elements

In Tensorflow, I'm trying to create the following matrix:
A = [[a, 0], [0,b]]
Where a and b are the parameters I'm trying to solve for.
Here's what I have so far:
a = tf.Variable((1,), name="a", dtype = tf.float64)
b = tf.Variable((1,), name="b", dtype = tf.float64)
const = tf.constant(0,dtype = tf.float64, shape = (1,))
A0 = tf.transpose(tf.stack([a,const]))
A1 = tf.transpose(tf.stack([const,b]))
A = tf.stack([A0,A1])
However the shape of A ends up being (2,1,2) which is wrong (since A0 and B0 both have shapes (1,2))
Is there an easier way to create the matrix object A in Tensorflow, or does anyone know why the shape is getting messed up with what I'm doing?

Well you can create a single variable vector params = tf.Variable((2,), name="ab") and then multiply with the identity matrix tf.eye(2):
A = tf.matmul(tf.expand_dims(params,0), tf.eye(2))

tf.stack increases the rank of the tensor (creating a new axis) and combines them in the new axis. If you want to combine tensors along an existing axis, you should use tf.concat.
a = tf.Variable((1,), name="a", dtype = tf.float64)
b = tf.Variable((1,), name="b", dtype = tf.float64)
const = tf.constant(0,dtype = tf.float64, shape = (1,))
A0 = tf.stack([a, const], axis=1)
A1 = tf.stack([const, b], axis=1) # more clear than tf.transpose
A = tf.concat((A0, A1), axis=0)
A is now shape (2, 2).
To explain, each object is a rank-1 tensor with one element:
A = [1]
const = [0]
stacking gives:
tf.stack((A, const), axis=0) = [[1], [0]] # 2x1 matrix
concatenating gives:
tf.concat((A, const), axis=0) = [1, 0] # 2 vector

How avoid changing numpy array's shape when appending?

i am trying to read 30 images and create a large numpy array from it by appending the numpy array for each image to that one single array so that i can use it later in a flow function for keras.
I have an empty list to which I am appending numpy arrays after doing face detection in a loop, after which I am creating a large numpy array from this list. The problem is that when,I create a numpy array from this list, it changes the shape of my array, which was originally (1,139,139,3), to (30,1,139,139,3). It basically adds the total number of images at the start as I am appending, and I want to get back to original shape. I do not want to use reshape as that might affect the data.
Here is the code:
img_width, img_height = 139, 139
confidence = 0.8
#graph = K.get_session().graph
data1 = []
def get_face(path):
with graph.as_default():
img = io.imread(path)
dets = detector(img, 1)
output = None
for i, d in enumerate(dets):
img = img[d.top():d.bottom(), d.left():d.right()]
img = resize(img, (img_width, img_height))
output = np.expand_dims(img, axis=0)
break
return output
for row in df.itertuples():
data1.append(get_face(row[1]))
data1 = np.array(data1)
print(data1)

As #filippo pointed out you probably want to omit np.expand_dims.
img_width, img_height = 139, 139
confidence = 0.8
#graph = K.get_session().graph
data1 = []
def get_face(path):
with graph.as_default():
img = io.imread(path)
dets = detector(img, 1)
output = None
for i, d in enumerate(dets):
img = img[d.top():d.bottom(), d.left():d.right()]
output = resize(img, (img_width, img_height))
break
return output
for row in df.itertuples():
data1.append(get_face(row[1]))
data1 = np.array(data1)
print(data1)
This piece of code will produce a list of 30 numpy arrays with shapes (139, 139, 3). Calling np.array constructor on that will give you array with shape (30, 139, 139, 3). You should also read documentation for np.stack and np.concatenate. Using the second function you can actually get what you want keeping np.expand_dims if for whatever reason you might need this.

np.array joins the elements of the list on a new front dimension:
In [141]: alist = []
In [142]: for i in range(2):
...: arr = np.zeros((3,4))
...: alist.append(arr)
...:
In [143]: np.array(alist).shape
Out[143]: (2, 3, 4)
expand_dims adds a new dimension:
In [144]: alist = []
In [145]: for i in range(2):
...: arr = np.zeros((3,4))
...: arr = np.expand_dims(arr,0)
...: alist.append(arr)
...:
In [146]: np.array(alist).shape
Out[146]: (2, 1, 3, 4)
concatenate joins on an existing dimension:
In [149]: np.concatenate(alist, axis=0).shape
Out[149]: (2, 3, 4)
An alternative to np.array is np.stack.

Broadcasting mathematical operations with PYMC3/Theano

I think this issue boils down to my lack of understanding with Theano works. I'm in a situation where I want to create a variable that is the result of a subtraction between a distribution and a numpy array. This works fine when I specify the shape parameter as 1
import pymc3 as pm
import numpy as np
import theano.tensor as T
X = np.random.randint(low = -10, high = 10, size = 100)
with pm.Model() as model:
nl = pm.Normal('nl', shape = 1)
det = pm.Deterministic('det', nl - x)
nl.dshape
(1,)
However, this breaks when I specify shape > 1
with pm.Model() as model:
nl = pm.Normal('nl', shape = 2)
det = pm.Deterministic('det', nl - X)
ValueError: Input dimension mis-match. (input[0].shape[0] = 2, input[1].shape[0] = 100)
nl.dshape
(2,)
X.shape
(100,)
I tried transposing X to make it broadcastable
X2 = X.reshape(-1, 1).transpose()
X2.shape
(1, 100)
But now it declares a mismatch at .shape[1] instead of .shape[0]
with pm.Model() as model:
nl = pm.Normal('nl', shape = 2)
det = pm.Deterministic('det', nl - X2)
ValueError: Input dimension mis-match. (input[0].shape[1] = 2, input[1].shape[1] = 100)
I can make this work if I loop over the elements of the distribution
distShape = 2
with pm.Model() as model:
nl = pm.Normal('nl', shape = distShape)
det = {}
for i in range(distShape):
det[i] = pm.Deterministic('det' + str(i), nl[i] - X)
det
{0: det0, 1: det1}
However this feels inelegant and constrains me to use loops for the rest of the model. I was wondering if there was a way to specify this operation so that it could work the same as with distributions.
distShape = 2
with pm.Model() as model:
nl0 = pm.Normal('nl1', shape = distShape)
nl1 = pm.Normal('nl2', shape = 1)
det = pm.Deterministic('det', nl0 - nl1)

You can do
X = np.random.randint(low = -10, high = 10, size = 100)
X = x[:,None] # or x.reshape(-1, 1)
and then
with pm.Model() as model:
nl = pm.Normal('nl', shape = 2)
det = pm.Deterministic('det', nl - X)
In this case the shapes of nl and X will be ((2, 1), (100,)), respectively and then broadcastable.
Notice we get the same behavior with two NumPy arrays (not only one Theano tensor and one NumPy array)
a0 = np.array([1,2])
b0 = np.array([1,2,3,5])
a0 = a0[:,None] # comment/uncomment this line
print(a0.shape, b0.shape)
b0-a0