Develop Reference

Collecting features from network.foward() in TensorFlow

So basically I want to achieve the same goal as in this code but in TensorFlow
def get_function(network, loader):
''' Collect function (features) from the self.network.module.forward_features() routine '''
features = []
for batch_idx, (inputs, targets) in enumerate(loader):
inputs, targets = inputs.to('cpu'), targets.to('cpu')
features.append([f.cpu().data.numpy().astype(np.float16) for f in network.forward_features(inputs)])
return [np.concatenate(list(zip(*features))[i]) for i in range(len(features[0]))]
Are there any clean ways to do this with TensorFlow iterator? Here is the torch code that I want to replicate in TensorFlow. https://pastecode.io/s/b03cpoyv

To answer your question, I just need to ensure that you understand your original torch code properly. So, here's your workflow
class LeNet(nn.Module):
def forward:
# few bunch of layers
return output
def forward_features:
# same as forward function
return [each layer output]
Now, next, you use the torch_get_function method and retrieve all layers output from the forward_features function that is defined in your model. The torch_get_function gives a total of 4 outputs as a list and you pick only the first feature and concate across the batches in the end.
def torch_get_function(network, loader):
features = []
for batch_idx, (inputs, targets) in enumerate(loader):
print('0', network.forward_features(inputs)[0].shape)
print('1', network.forward_features(inputs)[1].shape)
print('2', network.forward_features(inputs)[2].shape)
print('3', network.forward_features(inputs)[3].shape)
print()
features.append([f... for f in network.forward_features(inputs)])
return [np.concatenate(list(zip(*features))[i]) for i in range(len(features[0]))]
for epoch in epochs:
dataset = torchvision.datasets.MNIST...
dataset = torch.utils.data.Subset(dataset, list(range(0, 1000)))
functloader = torch.utils.data.DataLoader(...)
# for x , y in functloader:
# print('a ', x.shape, y.shape)
# a torch.Size([100, 1, 28, 28]) torch.Size([100])
activs = torch_get_function(net, functloader)
print(activs[0].shape)
break
That's why if when I ran your code, I got
# These are the 4 output that returned by forward_features(inputs)
0 torch.Size([100, 10, 12, 12])
1 torch.Size([100, 320])
2 torch.Size([100, 50])
3 torch.Size([100, 10])
...
# In the return statement of forward_features -
# You take only the first index feature and concate across batches.
(1000, 10, 12, 12)
So, the input size of your model is (batch_size, 1, 28, 28) and the final output is like (1000, 10, 12, 12).
Let's do the same in tensorflow, step by step.
import numpy as np
from tqdm import tqdm
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import (Conv2D, Dropout, MaxPooling2D,
Dense, Flatten)
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
x_test = x_test.astype("float32") / 255.0
x_test = np.reshape(x_test, (-1, 28, 28, 1))
dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))
dataset = dataset.shuffle(buffer_size=1024).batch(100)
# it's like torch.utils.data.Subset
dataset = dataset.take(1000)
dataset
<TakeDataset shapes: ((None, 28, 28, 1), (None,)), types: (tf.float32, tf.uint8)>
Let's now build the model. To make it familiar to you, I'm writing in sub-class API.
class LeNet(keras.Model):
def __init__(self, num_classes, input_size=28):
super(LeNet, self).__init__()
self.conv1 = Conv2D(10, (5, 5))
self.conv2 = Conv2D(20, (5, 5))
self.conv2_drop = Dropout(rate=0.5)
self.fc1 = Dense(50)
self.fc2 = Dense(num_classes)
def call(self, inputs, training=None):
x1 = tf.nn.relu(MaxPooling2D(2)(self.conv1(inputs)))
x2 = tf.nn.relu(MaxPooling2D(2)(self.conv2_drop(self.conv2(x1))))
x2 = Flatten()(x2)
x3 = tf.nn.relu(self.fc1(x2))
x4 = tf.nn.softmax(self.fc2(x3), axis=1)
# in tf/keras, when we will call model.fit / model.evaluate
# to train the model only x4 will return
if training:
x4
else: # but when model(input)/model.predict(), we can return many :)
return [x1, x2, x3, x4]
lenet = LeNet(10)
lenet.build(input_shape=(None, 28, 28, 1))
Get the desired features
features = []
for input, target in tqdm(dataset):
# lenet(...) will give 4 output as we model
# but as we're interested on the first index feature...
features.append(lenet(input, training=False)[0])
print(len(features))
features = np.concatenate(features, axis=0)
features.shape
(10000, 12, 12, 10)
In tensorflow, the channel axis is default set to last, as opposed to the torch. In torch, you received (1000, 10, 12, 12) and in tensorflow, it gives you (10000, 12, 12, 10) but you can change it of course, (how). Here is the working colab.

Issue while trying to draw learning curve

I'm trying to draw learning curve on a small data set
Full code here
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam
import keras.backend as K
K.clear_session()
model = Sequential()
model.add(Dense(1, input_shape=(1,)))
model.compile(Adam(lr=0.2), "mean_squared_error")
model.fit(x,y,epochs=50)
iw = model.get_weights()
from keras.utils import to_categorical
yc= to_categorical(y)
from sklearn.model_selection import train_test_split
xtr, xts, ytr, yts = train_test_split(x,yc, test_size=0.3)
train_sizes = (len(xtr) * np.linspace(0.1, 0.99999999, 4)).astype(int)
test_scores = []
for i in train_sizes :
xtrfr, _, yrtfr, _ = train_test_split(xtr,ytr,train_size=i)
model.set_weights(iw)
res = model.fit(xtrfr, yrtfr, epochs=600)
e = model.evaluate(xts,yts)
test_scores.append(e[-1])
plt.plot(train_sizes, test_scores, label="Learning Curve")
plt.legend()
plt.show()
but I'm getting this error
ValueError: Error when checking target: expected dense_1 to have shape (1,) but got array with shape (270,)
I'm guessing there's something wrong with the to_categorical but I can't figure it out ":)

looking at the shapes of your x and y shows that they are a 1 dimensional array:
>>> x.shape
(10000,)
>>> y.shape
(10000,)
however your model expects an array with input_shape=(1,), so first you will have to reshape your data like so:
>>> x = np.array(x, np.float32).reshape((-1, 1))
>>> y = np.array(y, np.float32).reshape((-1, 1))
they will now have this shape:
>>> x.shape
(10000, 1)
>>> y.shape
(10000, 1)
>>> x
and look like this:
>>> x
array([[73.847015],
[68.781906],
[74.11011 ],
...,
[63.867992],
[69.03424 ],
[61.944244]], dtype=float32)
>>> y
array([[241.89357],
[162.31047],
[212.74086],
...,
[128.47531],
[163.85246],
[113.6491 ]], dtype=float32)
an array with an array that has only one element

Resizing images with dynamic shape in tensorflow

I want to resize 3D images with a dynamic shape, for instance go from shape (64,64,64,1) to (128,128,128,1). The idea is to unstack the image along one axis, then use tf.image.resize_images and stack them again.
My issue is that tf.unstack can not handle variable sized inputs. If I run my code I obtain "ValueError: Cannot infer num from shape (?, ?, ?, 1)"
I have considered using tf.split instead, however it expects an integer input. Does anybody know a workaround?
Here is an example:
import tensorflow as tf
import numpy as np
def resize_by_axis(image, dim_1, dim_2, ax):
resized_list = []
# Unstack along axis to obtain 2D images
unstack_img_depth_list = tf.unstack(image, axis = ax)
# Resize 2D images
for i in unstack_img_depth_list:
resized_list.append(tf.image.resize_images(i, [dim_1, dim_2], method=1, align_corners=True))
# Stack it to 3D
stack_img = tf.stack(resized_list, axis=ax)
return stack_img
#X = tf.placeholder(tf.float32, shape=[64,64,64,1])
X = tf.placeholder(tf.float32, shape=[None,None,None,1])
# Get new shape
shape = tf.cast(tf.shape(X), dtype=tf.float32) * tf.constant(2, dtype=tf.float32)
x_new = tf.cast(shape[0], dtype=tf.int32)
y_new = tf.cast(shape[1], dtype=tf.int32)
z_new = tf.cast(shape[2], dtype=tf.int32)
# Reshape
X_reshaped_along_xy = resize_by_axis(X, dim_1=x_new, dim_2=y_new, ax=2)
X_reshaped_along_xyz= resize_by_axis(X_reshaped_along_xy, dim_1=x_new, dim_2=z_new, ax=1)
init = tf.global_variables_initializer()
# Run
with tf.Session() as sess:
sess.run(init)
result = X_reshaped_along_xyz.eval(feed_dict={X : np.zeros((64,64,64,1))})
print(result.shape)

tf.image.resize_images can resize multiple images at the same time, but it does not allow you to pick the batch axis. However, you can manipulate the dimensions of the tensor to put the axis that you want first, so it is used as batch dimension, and then put it back after resizing:
import tensorflow as tf
def resize_by_axis(image, dim_1, dim_2, ax):
# Make permutation of dimensions to put ax first
dims = tf.range(tf.rank(image))
perm1 = tf.concat([[ax], dims[:ax], dims[ax + 1:]], axis=0)
# Transpose to put ax dimension first
image_tr = tf.transpose(image, perm1)
# Resize
resized_tr = tf.image.resize_images(image_tr, [dim_1, dim_2],
method=1, align_corners=True)
# Make permutation of dimensions to put ax in its place
perm2 = tf.concat([dims[:ax] + 1, [0], dims[ax + 1:]], axis=0)
# Transpose to put ax in its place
resized = tf.transpose(resized_tr, perm2)
return resized
In your example:
import tensorflow as tf
import numpy as np
X = tf.placeholder(tf.float32, shape=[None, None, None, 1])
# Get new shape
shape = tf.cast(tf.shape(X), dtype=tf.float32) * tf.constant(2, dtype=tf.float32)
x_new = tf.cast(shape[0], dtype=tf.int32)
y_new = tf.cast(shape[1], dtype=tf.int32)
z_new = tf.cast(shape[2], dtype=tf.int32)
# Reshape
X_reshaped_along_xy = resize_by_axis(X, dim_1=x_new, dim_2=y_new, ax=2)
X_reshaped_along_xyz = resize_by_axis(X_reshaped_along_xy, dim_1=x_new, dim_2=z_new, ax=1)
init = tf.global_variables_initializer()
# Run
with tf.Session() as sess:
sess.run(init)
result = X_reshaped_along_xyz.eval(feed_dict={X : np.zeros((64, 64, 64, 1))})
print(result.shape)
# (128, 128, 128, 1)

Tensorflow - keras - Shape errors with 'strided_slice' (used resized MNIST dataset)

I am going to make some GAN-Model Tester using tf.keras with MNIST hand-writed digits dataset. Because my model is going to be used in 128x128 images, I resized MNIST dataset to 128x128x1. but, the program makes some errors, that I never seen.
(x_train, _), (_, _) = mnist.load_data()
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
x_train = tf.image.resize_images(x_train, [128, 128])
idx = np.random.randint(0, x_train.shape[0], batch_size) # picks some data, count is batch_size=32.
imgs = x_train[idx] # This line made errors
The last line made two errors:
tensorflow.python.framework.errors_impl.InvalidArgumentError: Shape must be rank 1 but is rank 2 for 'strided_slice_1' (op: 'StridedSlice') with input shapes: [60000,128,128,1], [1,32], [1,32], [1].
and,
ValueError: Shape must be rank 1 but is rank 2 for 'strided_slice_1' (op: 'StridedSlice') with input shapes: [60000,128,128,1], [1,32], [1,32], [1].
I think the number '32' means batch_size(=32).
I tried to find about this error, but I couldn't find like this error.
I don't have any ideas to solve this problems (because I started to use keras a week ago, before I used pytorch).

You have more issue with your code above, but the primer cause of error is that tensorflow doesn't support the numpy type advanced slicing.
Actually the error message is because tensorflow try to align your input array in his strided-slices:
An example for the strided-slices:
foo[5:,:,:3] on a 7x8x9 tensor is equivalent to foo[5:7,0:8,0:3].
foo[::-1] reverses a tensor with shape 8.
Unfortunately only basic type indexing is available in Tensorflow currently. Advanced type indexing is under development.
The secondary problem, that your resizing was not proper.
Tensorflow assumes a 3D or 4D input. You tried to pass a 2D image to `tf.image.resize_images(), which doesn't return the required new image dimensions. So we have to reshape the raw images like this:
x_train = x_train.reshape((-1, x_train.shape[1], x_train.shape[1], 1))
only then we can pass them to:
`x_train = tf.image.resize_images(x_train, [128, 128])
It will return then the proper dimensions:
print(x_train.shape)
Out:
(60000, 128, 128, 1)
So summarizing the whole solution, currently you can do it as follows:
import numpy as np
import tensorflow as tf
batch_size = 32
mnist = tf.keras.datasets.mnist
(x_train, _), (_, _) = mnist.load_data()
x_train = x_train.reshape((-1, x_train.shape[1], x_train.shape[1], 1))
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
x_train = tf.image.resize_images(x_train, [128, 128])
idx = np.random.randint(0, x_train.shape[0], batch_size)
imgs = [x_train[i,:,:,:] for i in idx]
which is a pretty messy "solution".
Other, actually a real solution with rearrange the original code we can achieve what we aimed to as a workaround to the tensorflow indexing issue:
import numpy as np
import tensorflow as tf
batch_size = 32
mnist = tf.keras.datasets.mnist
(x_train, _), (_, _) = mnist.load_data()
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
idx = np.random.randint(0, x_train.shape[0], batch_size)
x_train = x_train[idx]
x_train = x_train.reshape((-1, x_train.shape[1], x_train.shape[1], 1))
x_train = tf.image.resize_images(x_train, [128, 128])
print(x_train.shape)
Out:
(32, 128, 128, 1)
That's it!
Alternatively instead of tf.image.resize_images() you can use an additional image tools like skimage.transform.resize() from scikit-image which returns numpy array type data.

Value error while concatenating the numpy arrays

Im loading mnist dataset as follows,
(X_train, y_train), (X_test, y_test) = mnist.load_data()
However since I need to load and train my own dataset, I wrote the little script as follows which will give the exact train and test values
def load_train(path):
X_train = []
y_train = []
print('Read train images')
for j in range(10):
files = glob(path + "*.jpeg")
for fl in files:
img = get_im(fl)
print(fl)
X_train.append(img)
y_train.append(j)
return np.asarray(X_train), np.asarray(y_train)
the pertained model generates a numpy array of size (64, 28, 28, 1) while training. Im concatenating the image_batch from the generated image as follows,
X = np.concatenate((image_batch, generated_images))
However im getting the following error,
ValueError: all the input arrays must have same number of dimensions
img_batch is of size (64, 28, 28)
generated_images is of size (64, 28, 28, 1)
How do I expand the dimension of the img_batch in X_train so as to concatenate with generated_images? or is there any other ways to load the custom images in place of loadmnist?

There is a function in python called np.expand_dims() which can expand the dimension of any array along the axis provided in arguments. In your case use, img_batch = np.expand_dims(img_batch, axis=3).
One other approach would be to use reshape function as suggested by #Ioannis Nasios. img_batch = img_batch.reshape(64,28,28,1)

image_batch = image_batch.reshape(64, 28, 28, 1)