Develop Reference

How to Do Numpy Like Index Selection in Tensorflow?

Let us use the following code
#!/usr/bin/env python3
# encoding: utf-8
import numpy as np, tensorflow as tf # tf.__version__==2.7.0
sample_array=np.random.uniform(size=(2**10, 120, 20))
to_select=[5, 6, 9, 4]
sample_tensor=tf.convert_to_tensor(value=sample_array)
sample_array[:, :, to_select] # Works okay
sample_tensor[:, :, to_select] # TypeError. How to do this in tensor?
tf.convert_to_tensor(value=sample_tensor.numpy()[:, :, to_select]) # Ugly workaround
Basically, how to get those elements as a tensor of appropriate dimension, just like numpy? I tried tf.slice and tf.gather, but cannot figure out the proper arguments to pass.
I can convert it to numpy and back, but not sure if it will sacrifice the operation's efficiency, and work as part of a custom training loop.

The simplest solution would be to use tf.concat, although it is probably not so efficient:
import numpy as np
import tensorflow as tf
sample_array = np.random.uniform(size=(2, 2, 20))
to_select = [5, 6, 9, 4]
sample_tensor = tf.convert_to_tensor(value = sample_array)
numpy_way = sample_array[:, :, to_select]
tf_way = tf.concat([tf.expand_dims(sample_array[:, :, to_select[i]], axis=-1) for i in tf.range(len(to_select))], axis=-1)
#tf_way = tf.concat([tf.expand_dims(sample_array[:, :, s], axis=-1) for s in to_select], axis=-1)
print(numpy_way)
print(tf_way)
[[[0.81208086 0.03873406 0.89959868 0.97896671]
[0.57569184 0.33659472 0.32566287 0.58383079]]
[[0.59984846 0.43405048 0.42366314 0.25505199]
[0.16180442 0.5903358 0.21302399 0.86569914]]]
tf.Tensor(
[[[0.81208086 0.03873406 0.89959868 0.97896671]
[0.57569184 0.33659472 0.32566287 0.58383079]]
[[0.59984846 0.43405048 0.42366314 0.25505199]
[0.16180442 0.5903358 0.21302399 0.86569914]]], shape=(2, 2, 4), dtype=float64)
A more complicated, but efficient solution would involve using tf.meshgrid and tf.gather_nd. Check this post or this post and finally this. Here is an example based on your question:
to_select = tf.expand_dims(tf.constant([5, 6, 9, 4]), axis=0)
to_select_shape = tf.shape(to_select)
sample_tensor_shape = tf.shape(sample_tensor)
to_select = tf.expand_dims(tf.reshape(tf.tile(to_select, [1, to_select_shape[1]]), (sample_tensor_shape[0], sample_tensor_shape[0] * to_select_shape[1])), axis=-1)
ij = tf.stack(tf.meshgrid(
tf.range(sample_tensor_shape[0], dtype=tf.int32),
tf.range(sample_tensor_shape[1], dtype=tf.int32),
indexing='ij'), axis=-1)
gather_indices = tf.concat([tf.repeat(ij, repeats=to_select_shape[1], axis=1), to_select], axis=-1)
gather_indices = tf.reshape(gather_indices, (to_select_shape[1], to_select_shape[1], 3))
result = tf.gather_nd(sample_tensor, gather_indices, batch_dims=0)
result = tf.reshape(result, (result.shape[0]//2, result.shape[0]//2, result.shape[1]))
tf.Tensor(
[[[0.81208086 0.03873406 0.89959868 0.97896671]
[0.57569184 0.33659472 0.32566287 0.58383079]]
[[0.59984846 0.43405048 0.42366314 0.25505199]
[0.16180442 0.5903358 0.21302399 0.86569914]]], shape=(2, 2, 4), dtype=float64)

Allow or prohibit permutation in tensorflow loss function?

I'm training a network to reconstruct coordinates of specific structures in an image. Until now, my loss function contains three 2D vectors (i.e. 6 variables) for the coordinates which are learned via MSE, and three corresponding classifiers learned via SigmoidFocalCrossEntropy, indicating if there are 0, 1, 2 or 3 of these structures present. I thought it might be beneficial to give tensorflow the information that it is neglectable in which order that the vectors are reconstructed as long as the classifier is still correct. Simple example:
loss(tf.constant([[30, 20, 15, 7, 0, 0, 1, 1, 0]], dtype=tf.float32),
tf.constant([[0, 0, 15, 7, 30, 20, 0, 1, 1]], dtype=tf.float32)) == 0
To implement this I used tf.argsort on the magnitude of each vector:
def sort(tensor):
x = tf.unstack(tensor, axis=-1)
squ = []
for i in range(len(x) // 2):
i *= 2
squ.append(x[i] ** 2 + x[i+1] ** 2)
new = tf.stack(squ, axis=-1)
return tf.argsort(new, axis=-1, direction='ASCENDING',
stable=False, name=None)
and consecutively permuted the tensor:
def permute_tensor_structure(tensor, indices):
c = indices + 6
x = indices * 2
y = indices * 2 + 1
v = tf.transpose([x, y])
v = tf.reshape(v, [indices.shape[0], -1])
perm = tf.concat([v, c], axis=-1)
return tf.gather(tensor, perm, batch_dims=1, name=None, axis=-1)
I did the same for my ground truth and got the network up and running.
Minimal example extracted from my code:
import tensorflow as tf
from tensorflow_addons.losses import SigmoidFocalCrossEntropy
def compute_permute_loss(truth, predict):
l2 = tf.keras.losses.MeanSquaredError()
ce = SigmoidFocalCrossEntropy()
indices = sort(predict[:, 0:6])
indices2 = sort(truth[:, 0:6])
predict = permute_tensor_structure(predict, indices)
truth = permute_tensor_structure(truth, indices2)
L2 = l2(predict[:, 0:6], truth[:, 0:6])
BCE = tf.reduce_sum(ce(truth[:, 6:], predict[:, 6:],))
return 3 * L2 + BCE
class Test(tf.test.TestCase):
def test_permutation_loss(self):
tensor1 = tf.constant(
[[30, 20, 15, 7, 0, 0, 1, 1, 0]],
dtype=tf.float32)
tensor2 = permute_tensor_structure(tensor1, tf.constant([[2, 1, 0]]))
loss = compute_permute_loss(tensor1, tensor2)
self.assertEqual(loss, 0,
msg="loss for only permuted tensors is not zero")
tensor3 = tf.constant(
[[29, 22, 15, 7, 0, 0, 1, 1, 0]],
dtype=tf.float32)
loss = compute_permute_loss(tensor3, tensor2)
self.assertAllClose(loss, (1.0 + 4.0) / 2.0,
msg="loss for values is not rmse")
if __name__ == "__main__":
tf.test.main()
# [ RUN ] Test.test_permutation_loss
# [ OK ] Test.test_permutation_loss
However, I'm afraid the permutation in the loss could backfire and impair tensorflows backpropagation. Maybe somebody already faced a similar problem? Or has deeper knowledge on tensorflows graph building and back propagation? I would be grateful for every suggestion or input.

How to convert numpy code into tensorflow?

I am trying to convert numpy code into tensorflow graph format. But somewhere I am missing an understanding of dimensionality.
Here is numpy code:
def delta_to_boxes3d(deltas, anchors, coordinate='lidar'):
# Input:
# deltas: (N, w, l, 14)(200,240,14)
# feature_map_shape: (w, l)
# anchors: (w, l, 2, 7)(200,240,2,7)
# Ouput:
# boxes3d: (N, w*l*2, 7)
#anchros shape 9200,240,2,7)
anchors_reshaped = anchors.reshape(-1, 7) #(96000,7)
deltas = deltas.reshape(-1, 7) #(96000,7)
anchors_d = np.sqrt(anchors_reshaped[:, 4]**2 + anchors_reshaped[:, 5]**2)
boxes3d = np.zeros_like(deltas)
boxes3d[..., [0, 1]] = deltas[..., [0, 1]] * \
anchors_d[:, np.newaxis] + anchors_reshaped[..., [0, 1]] #in this line I have the problem
boxes3d[..., [2]] = deltas[..., [2]] * \
1.73 + anchors_reshaped[..., [2]] #ANCHOR_H = 1.73
boxes3d[..., [3, 4, 5]] = np.exp(
deltas[..., [3, 4, 5]]) * anchors_reshaped[..., [3, 4, 5]]
boxes3d[..., 6] = deltas[..., 6] + anchors_reshaped[..., 6]
return boxes3d
Here is the code which I have been trying:
def delta_boxes3d():
anchors = tf.placeholder(tf.float32,shape=[None,None,2,7],name="anchor") #check the anchor type later
anchors_reshaped = tf.reshape(anchors,shape=[96000,7])
delta = tf.placeholder(tf.float32,shape=[None,None,14],name="delta")
anchors_d = tf.sqrt(tf.add(tf.pow(anchors_reshaped[:,4],2),tf.pow(anchors_reshaped[:,5],2))) #96000
deltas = tf.reshape(delta,[96000,7])
x_shape = tf.shape(deltas)
boxes3d_ = tf.multiply(deltas[:,0:2],tf.add(tf.expand_dims(anchors_d,-1),anchors_reshaped[:,0:2]))
boxes3d = tf.ones(x_shape[:-1]) + boxes3d_
elta_ = np.random.rand(200,240,14)
anchor_ = np.random.rand(200,240,2,7)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
result = sess.run(boxes3d1,feed_dict={anchors:anchor_,delta:delta_}) #(96000,7) #need to get boxes3d
print(result.shape)
I am getting below error:
ValueError: Dimensions must be equal, but are 96000 and 2 for '{{node add_2}} = AddV2[T=DT_FLOAT](ones, Mul)' with input shapes: [96000], [96000,2].
Could someone help me with this?
Thanks in advance

The error comes from the line boxes3d = tf.ones(x_shape[:-1]) + boxes3d_.
You are trying to add shapes (96000,) and (96000,2), which you can't without expanding dims. If you want to add a scalar, you can do boxes3d = 1 + boxes3d.
In the example above, you want to do multiplication by scalar followed by addition.
Note that in the NumPy example in the highlighted line, you do the multiplication first and then addition. In TensorFlow, you made it the other way around (possibly by mistake).
I rewrote your NumPy example to Tensorflow 2 so that both functions return the same output.
def delta_boxes3d(deltas, anchors):
deltas = tf.constant(deltas)
anchors = tf.constant(anchors)
anchors_reshaped = tf.reshape(anchors, shape=[96000, 7])
anchors_d = tf.sqrt(tf.add(tf.pow(anchors_reshaped[:, 4], 2), tf.pow(anchors_reshaped[:, 5], 2))) # 96000
deltas = tf.reshape(deltas, [96000, 7])
boxes3d_01 = tf.add(tf.multiply(deltas[:, 0:2], tf.expand_dims(anchors_d, -1)), anchors_reshaped[:, 0:2])
boxes3d_2 = deltas[..., 2:3] * 1.73 + anchors_reshaped[..., 2:3]
boxes3d_345 = tf.exp(deltas[..., 3:6]) * anchors_reshaped[..., 3:6]
boxes3d_6 = deltas[..., 6:7] + anchors_reshaped[..., 6:7]
boxes3d = tf.concat([boxes3d_01, boxes3d_2, boxes3d_345, boxes3d_6], axis=-1)
return boxes3d
deltas = np.random.rand(200, 240, 14)
anchors = np.random.rand(200, 240, 2, 7)
print(delta_to_boxes3d(deltas, anchors))
print(delta_boxes3d(deltas, anchors))
You can notice I created smaller arrays first, and then I concatenated them. This is because Tensorflow won't allow me to modify EagerTensors.
Notice the difference between deltas[..., 2] and deltas[..., 2:3]. The second one doesn't reduce the last dimension. They return shapes (96000,), and (96000,1) respectively.

tensor index manipulation with "..."

Hi I'm new to Pytorch and torch tensors. I'm reading yolo_v3 code and encounter this question. I think it relates to tensor indexing with ..., but it's difficult to search ... by google, so I figure to ask it here. The code is:
prediction = (
x.view(num_samples, self.num_anchors, self.num_classes + 5, grid_size, grid_size)
.permute(0, 1, 3, 4, 2)
.contiguous()
)
print (prediction.shape)
# Get outputs
x = torch.sigmoid(prediction[..., 0]) # Center x
y = torch.sigmoid(prediction[..., 1]) # Center y
w = prediction[..., 2] # Width
h = prediction[..., 3] # Height
pred_conf = torch.sigmoid(prediction[..., 4]) # Conf
pred_cls = torch.sigmoid(prediction[..., 5:]) # Cls pred.
My understanding is that the prediction will be a tensor with shape of [batch, anchor, x_grid, y_grid, class]. But what does prediction[..., x] do (x=0,1,2,3,4,5)? Is it similar as numpy indexing of [:, x]? If so the calculation of x, y, w, h, pred_conf and pred_cls don't make sense.

It's call Ellipsis. It indicate unspecified dimensions of ndarray or tensor.
Here, if prediction shape is [batch, anchor, x_grid, y_grid, class] then
prediction[..., 0] # is equivalent to prediction[:,:,:,:,0]
prediction[..., 1] # is equivalent to prediction[:,:,:,:,1]
More
prediction[0, ..., 0] # equivalent to prediction[0,:,:,:,0]
You can also write ... as Ellipsis
prediction[Ellipsis, 0]

how to get covariance matrix in tensorflow?

How could I get covariance matrix in tensorflow? Like numpy.cov() in numpy.
For example, I want to get covariance matrix of tensor A, now I have to use numpy instead
A = sess.run(model.A, feed)
cov = np.cov(np.transpose(A))
Is there anyway to get cov by tensorflow instead of numpy?
It is differnet from the problem how to compute covariance in tensorflow, where their problem is to compute covariance for two vector, while mine is to compute covariance matrix of a matrix(a 2D tensor) effectively using tensorflow API

This is months late but anyway posting for completeness.
import numpy as np
import tensorflow as tf
def tf_cov(x):
mean_x = tf.reduce_mean(x, axis=0, keep_dims=True)
mx = tf.matmul(tf.transpose(mean_x), mean_x)
vx = tf.matmul(tf.transpose(x), x)/tf.cast(tf.shape(x)[0], tf.float32)
cov_xx = vx - mx
return cov_xx
data = np.array([[1., 4, 2], [5, 6, 24], [15, 1, 5], [7,3,8], [9,4,7]])
with tf.Session() as sess:
print(sess.run(tf_cov(tf.constant(data, dtype=tf.float32))))
## validating with numpy solution
pc = np.cov(data.T, bias=True)
print(pc)

Answering from 2019. Tensorflow probability now supports effortless correlation and covariance.
https://www.tensorflow.org/probability/api_docs/python/tfp/stats/covariance
x = tf.random_normal(shape=(100, 2, 3))
y = tf.random_normal(shape=(100, 2, 3))
# cov[i, j] is the sample covariance between x[:, i, j] and y[:, i, j].
cov = tfp.stats.covariance(x, y, sample_axis=0, event_axis=None)
# cov_matrix[i, m, n] is the sample covariance of x[:, i, m] and y[:, i, n]
cov_matrix = tfp.stats.covariance(x, y, sample_axis=0, event_axis=-1)

Equivalent to np.cov(data):
import tensorflow as tf
import numpy as np
data = np.array([[1., 4, 2], [5, 6, 24], [15, 1, 5], [7,3,8], [9,4,7]])
def tf_conv(x):
x = x - tf.expand_dims(tf.reduce_mean(x, axis=1), 1)
fact = tf.cast(tf.shape(x)[1] - 1, tf.float32)
return tf.matmul(x, tf.conj(tf.transpose(x))) / fact
with tf.Session() as sess:
print(sess.run(tf_cov(tf.constant(data, dtype=tf.float32))))

Following up on #Souradeep Nanda, if you experiment with it you'll find that tfp.stats.covariance is only have the value (elementwise) for np.cov(..., rowvar=False), so you will have to multiply by 2 after the calculation. (This applies for v0.11.1, tested on 2x2 matrix).
For 3x3 matrix, the values are NOT EQUIVALENT so perhaps you might want to stay using np.cov. This applies too if you're not using rowvar=False for np.cov() version. I am not sure why.

We can use tfp aka tensorflow-probability to computer cov matrix:
import tensorflow-probability as tfp
x=tf.random.normal(shape=(3,3))
cov = tfp.stats.covariance(x)
## which are same as:
np_cov = np.cov(tf.transpose(x_zero),bias=True)