Develop Reference

| ValueError: Input 0 of layer "sequential" is incompatible with the layer: expected shape=(None, 28, 28), found shape=(None, 28, 3)

import numpy as np
import matplotlib.pyplot as plt
import cv2 as cv
import tensorflow as tf
from tensorflow.keras import datasets, layers, models
fashion_mnist = tf.keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
train_images, test_images = train_images / 255, test_images / 255
model = tf.keras.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(10)
])
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.fit(train_images, train_labels, epochs=10)
test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
print('\nTest accuracy:', test_acc)
probability_model = tf.keras.Sequential([model,
tf.keras.layers.Softmax()])
predictions = probability_model.predict(test_images)
predictions[0]
model.save("image_classifier.model")
model = models.load_model("image_classifier.model")
img = cv.imread("shoes.png")
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
plt.imshow(img, cmap = plt.cm.binary)
prediction = model.predict(np.array(img) / 255)
index = np.argmax(prediction)
print(f"Prediction is {class_names[index]}")
when I tried to run this code with new image I get error all the time. I want the model to make a new prediction with new image but always I get scale error. my error is:
`
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
----> 1 prediction = model.predict(np.array(img) / 255)
2 index = np.argmax(prediction)
3 print(class_names[index])
~/opt/anaconda3/lib/python3.9/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
~/opt/anaconda3/lib/python3.9/site-packages/keras/engine/training.py in tf__predict_function(iterator)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
16 except:
17 do_return = False
ValueError: in user code:
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1845, in predict_function *
return step_function(self, iterator)
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1834, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1823, in run_step **
outputs = model.predict_step(data)
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/engine/training.py", line 1791, in predict_step
return self(x, training=False)
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/utils/traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/Users/.../opt/anaconda3/lib/python3.9/site-packages/keras/engine/input_spec.py", line 264, in assert_input_compatibility
raise ValueError(f'Input {input_index} of layer "{layer_name}" is '
`
ValueError: Input 0 of layer "sequential" is incompatible with the layer: expected shape=(None, 28, 28), found shape=(None, 28, 3)

It's only failing on the new image you passed in. Here:
img = cv.imread("shoes.png")
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
plt.imshow(img, cmap = plt.cm.binary)
prediction = model.predict(np.array(img) / 255)
Loading that png image returning an image with three channels: shape [height, width, channels]. Your model is expecting inputs with shape [batch, height, width].
You need to convert the image to greyscale and add a batch dimension:
img = np.array(img)/255
img = np.mean(img, axis=-1]
img = img[np.newaxis, ...]
prediction = model.predict(img)

MultiClass Object Detection and Classification using Fast R-CNN

I'm trying to make model (VGG-16) that uses Fast R-CNN for object detection.
In short, I want to find object on image and put bounding box where object is.
I already tried multiple way's of getting that, but all the time I'm getting some error's, basically most of them are with RoiPoolingLayer and loss function's.
Can you guys guide what I'm doing wrong?
So let me introduce you:
This is my code atm:
import pickle
import numpy
import tensorflow
from keras import Input, Model
from keras.initializers.initializers_v1 import RandomNormal
from keras.layers import Flatten, TimeDistributed, Dense, Dropout
from sklearn.preprocessing import LabelBinarizer
from tensorflow.keras.optimizers import Adam
from tensorflow.python.keras.regularizers import l2
from data import get_data, get_train_data
from rcnn.config import Config
import tensorflow as tf
from tensorflow.keras.layers import Layer
class RoiPoolingConv(Layer):
def __init__(self, pool_size, **kwargs):
self.pool_size = pool_size
super(RoiPoolingConv, self).__init__(**kwargs)
def build(self, input_shape):
self.nb_channels = input_shape[0][3]
super(RoiPoolingConv, self).build(input_shape)
def compute_output_shape(self, input_shape):
return None, None, self.pool_size, self.pool_size, self.nb_channels
def crop_and_resize(self, image, boxes):
box_ind = tf.range(tf.shape(boxes)[0])
box_ind = tf.reshape(box_ind, (-1, 1))
box_ind = tf.tile(box_ind, [1, tf.shape(boxes)[1]])
boxes = tf.keras.backend.cast(
tf.reshape(boxes, (-1, 4)), "float32"
)
box_ind = tf.reshape(box_ind, (1, -1))[0]
result = tf.image.crop_and_resize(image, boxes, box_ind, [self.pool_size, self.pool_size])
result = tf.reshape(result, (tf.shape(image)[0], -1, self.pool_size, self.pool_size, self.nb_channels))
return result
def call(self, x, mask=None):
assert (len(x) == 2)
img = x[0]
rois = x[1]
print(x)
print(img)
print(rois)
x1 = rois[:, 0]
y1 = rois[:, 1]
x2 = rois[:, 2]
y2 = rois[:, 3]
boxes = tf.stack([y1, x1, y2, x2], axis=-1)
print(boxes)
rs = self.crop_and_resize(img, boxes)
print(rs)
return rs
def get_config(self):
config = {'pool_size': self.pool_size}
base_config = super(RoiPoolingConv, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
PROPERTIES = Config()
def prepare_model(
model_path="model\\FastRCNN.h5"
):
roi_input = Input(shape=(None, 4), name="input_2")
model_cnn = tensorflow.keras.applications.VGG16(
include_top=True,
weights='imagenet'
)
model_cnn.trainable = True
x = model_cnn.layers[17].output
x = RoiPoolingConv(7)([x, roi_input])
x = TimeDistributed(Flatten())(x)
softmaxhead = Dense(4096, activation='relu', kernel_initializer=RandomNormal(stddev=0.01), kernel_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))(x)
softmaxhead = Dropout(0.5)(softmaxhead)
softmaxhead = Dense(4096, activation='relu', kernel_initializer=RandomNormal(stddev=0.01), kernel_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))(softmaxhead)
softmaxhead = Dropout(0.5)(softmaxhead)
softmaxhead = Dense(20, activation='softmax', kernel_initializer='zero', name='class_label')(softmaxhead)
bboxhead = Dense(128, activation='relu')(x)
bboxhead = Dense(64, activation='relu')(bboxhead)
bboxhead = Dense(32, activation='relu')(bboxhead)
bboxhead = Dense(4, activation='sigmoid', name='bounding_box')(bboxhead)
model_final = Model(inputs=[model_cnn.input, roi_input], outputs=(bboxhead, softmaxhead))
opt = Adam(learning_rate=0.0001)
losses = {
"class_label": PROPERTIES.CLASS_LABEL_LOSSES,
"bounding_box": PROPERTIES.BOUNDING_BOX_LOSSES
}
lossWeights = {
"class_label": PROPERTIES.LOSS_WEIGHTS,
"bounding_box": PROPERTIES.LOSS_WEIGHTS
}
model_final.compile(
loss=losses,
optimizer=opt,
metrics=["accuracy"],
loss_weights=lossWeights
)
tensorflow.keras.utils.plot_model(
model_final,
"model.png",
show_shapes=True,
show_layer_names=False,
rankdir='TB'
)
model_final.save(model_path)
return model_final
def train_RCNN_VGG(path):
# get voc data
all_data, classes_count, class_mapping = get_data(path)
tr_images, tr_labels_rois, tr_bboxes_rois, tr_bboxes_gt = get_train_data(all_data)
#val_images, val_labels, val_bboxes = get_validation_data(all_data)
# delete unnecessary data
del classes_count
del class_mapping
del all_data
# convert to numpy array
tr_images = numpy.array(tr_images, dtype="float32")
tr_bboxes_rois = numpy.array(tr_bboxes_rois, dtype="float32")
tr_bboxes_gt = numpy.array(tr_bboxes_gt, dtype="float32")
tr_labels_rois = numpy.array(tr_labels_rois)
print(tr_images.shape)
print(tr_bboxes_rois.shape)
print(tr_bboxes_gt.shape)
print(tr_labels_rois.shape)
# same for validation data
#val_images = numpy.array(val_images, dtype="float32")
#val_bboxes = numpy.array(val_bboxes, dtype="float32")
#val_labels = numpy.array(val_labels)
# use label binarizer for signing which class/label if for image
labelBinarizer = LabelBinarizer()
tr_labels_rois = labelBinarizer.fit_transform(tr_labels_rois)
#val_labels = labelBinarizer.fit_transform(val_labels)
classes = len(labelBinarizer.classes_)
# load model, provide number of classes
#model_vgg = load_model_or_construct(classes)
model_vgg = prepare_model()
# define a dictionary to set the loss methods
losses = {
"class_label": PROPERTIES.CLASS_LABEL_LOSSES,
"bounding_box": PROPERTIES.BOUNDING_BOX_LOSSES
}
# define a dictionary that specifies the weights per loss
lossWeights = {
"class_label": PROPERTIES.LOSS_WEIGHTS,
"bounding_box": PROPERTIES.LOSS_WEIGHTS
}
# initialize the optimizer, compile the model, and show the model
opt = Adam(learning_rate=PROPERTIES.LEARNING_RATE)
model_vgg.compile(loss=losses, optimizer=opt, metrics=["accuracy"], loss_weights=lossWeights)
# construct a dictionary for our target training outputs, for our target testing
trainTargets = {
"class_label": tr_labels_rois,
"bounding_box": tr_bboxes_gt
}
#validationTargets = {
# "class_label": val_labels,
# "bounding_box": val_bboxes
#}
# train the network for bounding box regression and class label
H = model_vgg.fit(
[tr_images, tr_bboxes_rois], trainTargets,
# validation_data=(val_images, validationTargets),
batch_size=PROPERTIES.BATCH_SIZE,
epochs=PROPERTIES.EPOCHS,
verbose=PROPERTIES.VERBOSE)
# save model, print summary
model_vgg.save(PROPERTIES.RCNN_MODEL_NAME, save_format=PROPERTIES.RCNN_MODEL_FORMAT)
model_vgg.summary()
# save binarizer
f = open(PROPERTIES.BINARIZER_NAME, "wb")
f.write(pickle.dumps(labelBinarizer))
f.close()
if __name__ == '__main__':
# load rcnn
train_RCNN_VGG(PROPERTIES.DATASET_PATH)
I'm creating RoiPooling Layer, VGG-16 architecture, loading pre-trained weights, making my own output layers, cause I have 20 classes (basing on VOC Data from 2012) that's why first output has 20, second has 4 - cause of bounding box's coordinates.
In train method, you can see I'm printing shape's of data I'm delivering, they are:
(1048, 224, 224, 3)
(1048, 4)
(1048, 4)
(1048,)
First one, it's 1048 images of 224x224 rgb
Second, it's 1048 rois coordinates prepared for 224x224
Third, it's 1048 ground truth's bboxes
Fourth, it's 1048 times 20 labels. Label's are like this: [[0, 0, 0, 0, 0, 0, ... 1, 0, 0,](19's zeros, and one 1 - correct label), [0, ....]]
I was basing on this: https://www.pyimagesearch.com/2020/10/12/multi-class-object-detection-and-bounding-box-regression-with-keras-tensorflow-and-deep-learning/
Currently I have this error:
Traceback (most recent call last):
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\Karol\anaconda3\lib\site-packages\tensorflow\python\framework\func_graph.py", line 1129, in autograph_handler
raise e.ag_error_metadata.to_exception(e)
ValueError: in user code:
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 878, in train_function *
return step_function(self, iterator)
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 867, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 860, in run_step **
outputs = model.train_step(data)
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 809, in train_step
loss = self.compiled_loss(
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 141, in __call__
losses = call_fn(y_true, y_pred)
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 245, in call **
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 1664, in categorical_crossentropy
return backend.categorical_crossentropy(
File "C:\Users\Karol\anaconda3\lib\site-packages\keras\backend.py", line 4994, in categorical_crossentropy
target.shape.assert_is_compatible_with(output.shape)
ValueError: Shapes (None, 20) and (None, None, 20) are incompatible
python-BaseException
So, my question is: What am I missing, is my preprocessing-data incorrect? I'm trying to teach my model recognition 20 classes and pointing where on image this object probably is. But I have to make wrong data delivering I guess.
Just to make something clear, I'm using categorical cross entropy and mean average precision for "class label" and "bounding boxes".
Maybe I'm just using wrong loss function's?
Please help.

Try to use the loss tf.keras.losses.SparseCategoricalCrossEntropy instead, and make sure you have labels in one hot encoding format, for the reasons pointed here:
Getting a ValueError in tensorflow saying that my shapes are incompatible

The explanation of RoiPooligLayer said that
Shape of inputs must be:
[(batch_size, pooled_height, pooled_width, n_channels), for featur map
and (batch_size, num_rois, 4)] for region of interest
but in your work you did not add the batch_size dimension
try with this:
model_cnn.trainable = True
x = model_cnn.layers[17].output
x = np.expand_dims(x, axis=0)
x = RoiPoolingConv(7)([x, roi_input])
x = TimeDistributed(Flatten())(x)

"Got OperatorNotAllowedInGraphError: iterating over tf.Tensor" while not obviously iterating over tensor

OS: Manjaro Linux x64
CUDA: 11.0.3
TF/Keras: 2.4(.1)
Py: 3.8
Hello,
I'm trying to build some kind of W-VAE-GAN. I keep un running into that very same error over and over again and I already had that problem with Keras 2.3, interestingly NOT using TF/K 2.2. Unfortunately I need to use Keras 2.4 because I'm supposed to run my code on our university server with these exact TF/K and CUDA-Versions. At this point I'm just trying to make sure my code works as intended.
The error I get is the following, with the exact lines commented in my code:
...
Epoch 1/20
Traceback (most recent call last):
File "/home/peer/Programmierkram/BA/Metal_GAN/wvaegan.py", line 282, in <module>
gan.fit(gen_trans, batch_size=batch_size, epochs=epochs, callbacks=[callback])
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 1100, in fit
tmp_logs = self.train_function(iterator)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 828, in __call__
result = self._call(*args, **kwds)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 871, in _call
self._initialize(args, kwds, add_initializers_to=initializers)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 725, in _initialize
self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 2969, in _get_concrete_function_internal_garbage_collected
graph_function, _ = self._maybe_define_function(args, kwargs)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3361, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/function.py", line 3196, in _create_graph_function
func_graph_module.func_graph_from_py_func(
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 990, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py", line 634, in wrapped_fn
out = weak_wrapped_fn().__wrapped__(*args, **kwds)
File "/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/framework/func_graph.py", line 977, in wrapper
raise e.ag_error_metadata.to_exception(e)
tensorflow.python.framework.errors_impl.OperatorNotAllowedInGraphError: in user code:
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:795 step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:1259 run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica
return fn(*args, **kwargs)
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:788 run_step **
outputs = model.train_step(data)
/home/peer/Programmierkram/BA/Metal_GAN/wvaegan.py:190 train_step
z_mean, z_log_var, z = self.encoder(clip_img) # <------ WHY NO ERROR HERE?
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:505 __iter__
self._disallow_iteration()
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:498 _disallow_iteration
self._disallow_when_autograph_enabled("iterating over `tf.Tensor`")
/home/peer/Programmierkram/BA/Metal_GAN/venv/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:474 _disallow_when_autograph_enabled
raise errors.OperatorNotAllowedInGraphError(
OperatorNotAllowedInGraphError: iterating over `tf.Tensor` is not allowed: AutoGraph did convert this function. This might indicate you are trying to use an unsupported feature.
What I do not understand is how it comes to this error in the first place, as I successfully executed the VAE-part using TF2.2 without any error like this, and, more importantly, there is no iteration over any tensor obvious to me. Even if I commented out the for-loop in my train_step the same error occurs few lines later in the same context. I have also tried decorating the def train_step() with #tf.function, yet nothing changed.
The code I used is the following:
import os
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import tensorflow.keras.backend as K
from keras.preprocessing.image import ImageDataGenerator
import itertools
import scipy.io
import matplotlib.pyplot as plt
import matplotlib.image as PIL
runOnGPU = 0
if runOnGPU==1:
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="0"
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
except RuntimeError as e:
print(e)
else:
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
path_clipped_train = os.path.join('./sinograms/clip')
path_transparent_train = os.path.join('./sinograms/transparent')
img_width, img_height = 512, 512
bottleneck = 1024 * 2
filters = (1024, 512, 256, 64)
filter_size = (3, 3, 3, 3)
batch_size = 4
epochs = 20
dsc_steps = 1
gp_w = 10.0
beta_v = 2
learning_rate = 125e-5
latent_dim = 2
input_shape = (1, img_width, img_height, 1)
dataset_gen1 = ImageDataGenerator(rescale=1 / 255, dtype="float32")
dataset_gen2 = ImageDataGenerator(rescale=1 / 255, dtype="float32")
gen_trans = dataset_gen1.flow_from_directory(path_transparent_train,
target_size=(img_width, img_height),
color_mode='grayscale',
classes=[''],
class_mode=None,
batch_size=batch_size,
shuffle=False,
)
gen_clip = dataset_gen2.flow_from_directory(path_clipped_train,
target_size=(img_width, img_height),
color_mode='grayscale',
classes=[''],
class_mode=None,
batch_size=batch_size,
shuffle=False,
)
class Sampling(layers.Layer):
def call(self, inputs):
z_mean, z_log_var = inputs
batch = tf.shape(z_mean)[0]
dim = tf.shape(z_mean)[1]
epsilon = tf.keras.backend.random_normal(shape=(batch, dim))
return z_mean + tf.exp(0.5 * z_log_var) * epsilon
def get_encoder():
encoder_inputs = keras.Input(shape=input_shape[1:], name="encoder_input")
enc = encoder_inputs
for (numFilters, szFilters) in zip(filters, filter_size):
enc = layers.Conv2D(numFilters, szFilters, activation='relu', strides=2, padding='same')(enc)
enc = layers.BatchNormalization()(enc)
enc = layers.Dropout(0.2)(enc)
conv_shape = K.int_shape(enc)[1:]
enc = layers.Flatten()(enc)
enc = layers.Dense(bottleneck, activation='relu', name="bottleneck")(enc)
enc = layers.BatchNormalization()(enc)
z_mean = layers.Dense(latent_dim, name="z_mean")(enc)
z_log_var = layers.Dense(latent_dim, name="z_log_var")(enc)
latent_z = Sampling()([z_mean, z_log_var])
encoder_model = keras.models.Model(encoder_inputs, latent_z, name="encoder")
return encoder_model, conv_shape
enc_model, conv_shape = get_encoder()
enc_model.summary()
def get_decoder():
latent_input = keras.Input(shape=(latent_dim,))
dec = layers.Dense(conv_shape[0] * conv_shape[1] * conv_shape[2], activation='relu')(latent_input)
dec = layers.Reshape(conv_shape)(dec)
for (numFilters, szFilters) in zip(reversed(filters), reversed(filter_size)):
dec = layers.Conv2DTranspose(numFilters, szFilters, activation='relu', strides=2, padding='same')(dec)
dec = layers.BatchNormalization()(dec)
dec = layers.Dropout(0.2)(dec)
decoder_outputs = layers.Conv2DTranspose(1, 3, activation='relu', padding='same')(dec)
decoder_model = keras.models.Model(latent_input, decoder_outputs, name="decoder")
return decoder_model
dec_model = get_decoder()
dec_model.summary()
def get_discriminator():
dscr_input = keras.Input(shape=input_shape[1:])
dscr = dscr_input
for numFilters in filters:
dscr = layers.Conv2D(numFilters, kernel_size=5, activation='relu', strides=2, padding='same')(dscr)
dscr = layers.Flatten()(dscr)
dscr = layers.Dense(1, activation="relu", name="dsc_end")(dscr)
discriminator_model = keras.models.Model(dscr_input, dscr, name="discriminator")
return discriminator_model
dsc_model = get_discriminator()
dsc_model.summary()
class GAN(keras.Model):
def __init__(self,
discriminator,
encoder,
decoder,
latent_dim,
dsc_steps=dsc_steps,
gp_w=gp_w,
):
super(GAN, self).__init__()
self.discriminator = discriminator
self.encoder = encoder
self.decoder = decoder
self.latent_dim = latent_dim
self.dsc_steps = dsc_steps
self.gp_w = gp_w
def compile(self,
dsc_optimizer, enc_optimizer, dec_optimizer,
dsc_loss_fn, enc_loss_fn, dec_loss_fn):
super(GAN, self).compile()
self.dsc_optimizer = dsc_optimizer
self.enc_optimizer = enc_optimizer
self.dec_optimizer = dec_optimizer
self.dsc_loss_fn = dsc_loss_fn
self.enc_loss_fn = enc_loss_fn
self.dec_loss_fn = dec_loss_fn
def call(self, data):
ds = self.discriminator(data)
e = self.encoder(data)
d = self.decoder(e)
def gradient_penalty(self, batch_size, ref_img, gen_img):
alpha = tf.random_normal([batch_size, 1, 1, 1], 0.0, 1.0)
diff = gen_img - ref_img
interpolated = ref_img + alpha * diff
with tf.GradientTape() as gp_tape:
gp_tape.watch(interpolated)
pred = self.discriminator(interpolated, training=True)
grads = gp_tape.gradient(pred, [interpolated])[0]
norm = tf.sqrt(tf.reduce_sum(tf.square(grads), axis=[1, 2, 3]))
gp = tf.reduce_mean((norm - 1.0) ** 2)
return gp
#tf.function # doesn't make any difference if decorating with that
def train_step(self, data):
trans_img = data
clip_img = data
batch_size = tf.shape(trans_img)[:1]
for i in range(self.dsc_steps):
with tf.GradientTape() as tape:
z_mean, z_log_var, z = self.encoder(clip_img) # <------ ERROR HERE
gen_img = self.decoder(z)
gen_logits = self.discriminator(gen_img)
ref_logits = self.discriminator(trans_img)
dsc_cost = self.dsc_loss_fn(ref_img=ref_logits, gen_img=gen_logits)
gp = self.gradient_penalty(batch_size, trans_img, gen_img)
dsc_loss = dsc_cost + gp * self.gp_w
dsc_gradient = tape.gradient(dsc_loss, self.discriminator.trainable_variables)
self.dsc_optimizer.apply_gradients(zip(dsc_gradient, self.discriminator.trainable_variables))
with tf.GradientTape() as tape:
z_mean, z_log_var, z = self.encoder(clip_img) # <------ ERROR ALSO HERE IF dsc_steps = 0
gen_img = self.decoder(z)
gen_img_logits = self.discriminator(gen_img)
dec_loss = self.dec_loss_fn(gen_img_logits)
kl_loss = self.kl_loss(z_mean, z_log_var)
enc_gradient = tape.gradient(kl_loss, self.encoder.trainable_variables)
self.enc_optimizer.apply_gradients(zip(enc_gradient, self.encoder.trainable_variables))
dec_gradient = tape.gradient(dec_loss, self.decoder.trainable_variables)
self.dec_optimizer.apply_gradients(zip(dec_gradient, self.decoder.trainable_variables))
return {"dsc_loss": dsc_loss, "KL-Loss": kl_loss, "dec_loss": dec_loss}
class GANMonitor(keras.callbacks.Callback):
def __init__(self, num_img=6, latent_dim=latent_dim):
self.num_img = num_img
self.latent_dim = latent_dim
def on_epoch_end(self, epoch, logs=None):
generated_images = self.model.decoder()
generated_images = (generated_images * 127.5) + 127.5
for i in range(self.num_img):
img = generated_images[i].np()
img = keras.preprocessing.image.array_to_img(img)
img.save("generated_img_{i}_{epoch}.png".format(i=i, epoch=epoch))
encoder_optimizer = keras.optimizers.Adam(learning_rate=0.0002, beta_1=0.5, beta_2=0.9)
decoder_optimizer = keras.optimizers.Adam(learning_rate=0.0002, beta_1=0.5, beta_2=0.9)
discriminator_optimizer = keras.optimizers.Adam(learning_rate=0.0002, beta_1=0.5, beta_2=0.9)
def discriminator_loss(real_img, fake_img):
real_loss = tf.reduce_mean(real_img)
fake_loss = tf.reduce_mean(fake_img)
return fake_loss - real_loss
def generator_loss(fake_img):
return -tf.reduce_mean(fake_img)
def kl_loss(z_mean, z_log_var):
kl_loss = -0.5 * (1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var))
kl_loss = tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
return beta_v * kl_loss
def reconstruction_loss(data, reconstruction):
rec_loss = tf.reduce_mean(
tf.reduce_sum(keras.losses.mse(data, reconstruction), axis=(1, 2))
)
return rec_loss
callback = GANMonitor(num_img=3, latent_dim=latent_dim)
gan = GAN(
discriminator=dsc_model,
encoder=enc_model,
decoder=dec_model,
latent_dim=latent_dim,
dsc_steps=dsc_steps,
)
gan.compile(
dsc_optimizer=discriminator_optimizer,
enc_optimizer=encoder_optimizer,
dec_optimizer=decoder_optimizer,
dsc_loss_fn=discriminator_loss,
enc_loss_fn=kl_loss,
dec_loss_fn=generator_loss,
)
gan.fit(gen_trans, batch_size=batch_size, epochs=epochs, callbacks=[callback])
I'd be very thankful for any help because I haven't been able to find or work any solution to this. I've read that in TF2.5 some errors like this shouldn't occur anymore, but using TF2.5 is not an option.

I found the problem.
In the original Keras example for an VAE, the encoder ends in three layers, namely z_mean, z_log_var and latent_z. While it was possible to access all terminal layers of a model in TF 2.2, as I did in my train_step
z_mean, z_log_var, z = encoder(data)
only (lantent_) z is committed as defined in the encoder model initialization.
By defining the model with
encoder_model = keras.models.Model(encoder_inputs, ([z_mean, z_log_var, latent_z]), name="encoder")
with a list of output layers, all z* are accessible.
I assume that getting multiple variables from a single tensor with a singular output
x1, x2, x3 = model(data)
results in a loop that might look something like:
for i in x:
x{i} = model(data)
which is the only explaination for an iteration over a tensor I can think of.
However, reading code might be useful, I'll try to remember that.

"RuntimeError: expected scalar type Long but found Float" in the forward process

I'm trying to create a model with PyTorch but during the forward, I encounter this issue "RuntimeError: expected scalar type Long but found Float".
Source code:
dataset_loader.py:
import h5py
import torch
import numpy as np
from PIL import Image
from os import listdir
from sklearn.utils import shuffle
WIDTH = 64
HEIGHT = 64
CREATE_DATASET = False
def load_set(path: str):
dataset = []
for f in listdir(path):
dataset.append(np.asarray(Image.open(path + f).resize((WIDTH, HEIGHT)).convert('RGB'), dtype=np.int32).reshape(3, WIDTH, HEIGHT))
return np.array(dataset, dtype=np.int32)
def create_batch(dataset, labels, batch_size):
dataset_result = []
labels_result = []
data_batch = []
label_batch = []
for index in range(len(dataset)):
if len(data_batch) == batch_size:
# dataset_result.append(np.array(data_batch, dtype=np.int32))
# labels_result.append(np.array(label_batch, dtype=np.int32))
dataset_result.append(data_batch)
labels_result.append(label_batch)
data_batch = []
label_batch = []
else:
# data_batch.append(torch.tensor(dataset[index]))
# label_batch.append(torch.tensor(labels[index]))
data_batch.append(np.array(dataset[index], dtype=np.int32))
label_batch.append(np.array(labels[index], dtype=np.int32))
dataset_result = np.array(dataset_result, dtype=np.int32)
labels_result = np.array(labels_result, dtype=np.int32)
return torch.from_numpy(dataset_result), torch.from_numpy(labels_result)
# return dataset_result, labels_result
def create_dataset():
dataset_file = h5py.File("dataset.hdf5", "w")
train_normal = load_set("chest_xray/train/NORMAL/")
dataset_file.create_dataset("train_normal", train_normal.shape, dtype=np.int32, data=train_normal)
train_pneumonia = load_set("chest_xray/train/PNEUMONIA/")
dataset_file.create_dataset("train_pneumonia", train_pneumonia.shape, dtype=np.int32, data=train_pneumonia)
test_normal = load_set("chest_xray/test/NORMAL/")
dataset_file.create_dataset("test_normal", test_normal.shape, dtype=np.int32, data=test_normal)
test_pneumonia = load_set("chest_xray/test/PNEUMONIA/")
dataset_file.create_dataset("test_pneumonia", test_pneumonia.shape, dtype=np.int32, data=test_pneumonia)
val_normal = load_set("chest_xray/val/NORMAL/")
dataset_file.create_dataset("val_normal", val_normal.shape, dtype=np.int32, data=val_normal)
val_pneumonia = load_set("chest_xray/val/PNEUMONIA/")
dataset_file.create_dataset("val_pneumonia", val_pneumonia.shape, dtype=np.int32, data=val_pneumonia)
def load_dataset():
dataset = h5py.File('dataset.hdf5', 'r')
train_set = np.array(list(dataset["train_normal"]) + list(dataset["train_pneumonia"]), dtype=np.int32)
test_set = np.array(list(dataset["test_normal"]) + list(dataset["test_pneumonia"]), dtype=np.int32)
val_set = np.array(list(dataset["val_normal"]) + list(dataset["val_pneumonia"]), dtype=np.int32)
train_labels = [0] * len(dataset["train_normal"]) + [1] * len(dataset["train_pneumonia"])
test_labels = [0] * len(dataset["test_normal"]) + [1] * len(dataset["test_pneumonia"])
val_labels = [0] * len(dataset["val_normal"]) + [1] * len(dataset["val_pneumonia"])
BATCH_SIZE = 32
train_set, train_labels = shuffle(np.array(train_set, dtype=np.int32), np.array(train_labels, dtype=np.int32))
train_set, train_labels = create_batch(train_set, train_labels, BATCH_SIZE)
test_set, test_labels = shuffle(np.array(test_set, dtype=np.int32), np.array(test_labels, dtype=np.int32))
test_set, test_labels = create_batch(test_set, test_labels, BATCH_SIZE)
val_set, val_labels = shuffle(np.array(val_set, dtype=np.int32), np.array(val_labels, dtype=np.int32))
val_set, val_labels = create_batch(val_set, val_labels, BATCH_SIZE)
return train_set, train_labels, test_set, test_labels, val_set, val_labels, BATCH_SIZE
network.py:
import torch.nn as nn
import torch.nn.functional as F
class Network(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels=6, kernel_size=5)
self.conv2 = nn.Conv2d(in_channels=6, out_channels=12, kernel_size=5)
self.fc1 = nn.Linear(in_features=12*5*5, out_features=120)
self.fc2 = nn.Linear(in_features=120, out_features=60)
self.out = nn.Linear(in_features=60, out_features=2)
def forward(self, t):
t = self.conv1(t)
t = F.relu()
t = F.max_pool2d(t, kernel_size=2, stride=2)
t = F.relu(self.conv2(t))
t = F.max_pool2d(t, kernel_size=2, stride=2)
t = t.reshape(-1, 12 * 4 * 4)
t = F.relu(self.fc1(t))
t = F.relu(self.fc2(t))
t = self.out(t)
return t
main.py:
import numpy as np
import torch
import torch.optim as optim
import torch.nn.functional as F
import network
import dataset_loader
import matplotlib.pyplot as plt
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
WIDTH = 64
HEIGHT = 64
NEED_TO_CREATE_DATASET = False
if NEED_TO_CREATE_DATASET:
dataset_loader.create_dataset()
train_set, train_labels, test_set, test_labels, val_set, val_labels, BATCH_SIZE = dataset_loader.load_dataset()
TRAINING_SIZE = len(train_set) * BATCH_SIZE
TESTING_SIZE = len(test_set) * BATCH_SIZE
EPOCHS = 5
LEARNING_RATE = 0.01
network = network.Network().to(device)
optimizer = optim.Adam(network.parameters(), lr=LEARNING_RATE)
training_losses = []
training_accuracies = []
testing_losses = []
testing_accuracies = []
def get_num_correct(preds, labels):
return preds.argmax(dim=1).eq(labels).sum().item()
def train():
network.train()
correct_in_episode = 0
episode_loss = 0
for index, images in enumerate(train_set):
labels = train_labels[index]
print(images.shape)
# exit()
predictions = network(images.type(torch.LongTensor)) # TODO: fix crash "RuntimeError: expected scalar type Long but found Float"
# predictions = network(images)
loss = F.cross_entropy(predictions, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
episode_loss += loss.item()
correct_in_episode += get_num_correct(predictions, labels)
training_losses.append(episode_loss)
training_accuracies.append(correct_in_episode * 100 / TRAINING_SIZE)
print(f"Epoch: {epoch + 1} accuracy: {correct_in_episode * 100 / TRAINING_SIZE:.2f} loss: {episode_loss:.3f}", end="\t")
def test():
network.eval()
episode_loss = 0
correct_in_episode = 0
with torch.no_grad():
for index, images in enumerate(test_set):
labels = test_labels[index]
predictions = network(images)
loss = F.cross_entropy(predictions, labels)
episode_loss = loss.item()
correct_in_episode += get_num_correct(predictions, labels)
testing_losses.append(episode_loss)
testing_accuracies.append(correct_in_episode * 100 / TESTING_SIZE)
print(f'Validation: Accuracy: {correct_in_episode * 100 / TESTING_SIZE:.2f} loss: {episode_loss:.3f}')
for epoch in range(EPOCHS):
train()
test()
fig = plt.figure()
plt.plot(list(range(1, len(training_losses)+1)), training_losses, color='blue')
plt.plot(list(range(1, len(testing_losses)+1)), testing_losses, color='red')
plt.legend(['Train Loss', 'Test Loss'], loc='upper right')
plt.xlabel('number of training examples seen')
plt.ylabel('Loss')
fig = plt.figure()
plt.plot(list(range(1, len(training_accuracies)+1)), training_accuracies, color='blue')
plt.plot(list(range(1, len(testing_accuracies)+1)), testing_accuracies, color='red')
plt.legend(['Train Accuracy', 'Test Accuracy'], loc='upper right')
plt.xlabel('number of training examples seen')
plt.ylabel('Accuracy')
Error:
torch.Size([32, 3, 64, 64])
Traceback (most recent call last):
File "Soluce.py", line 86, in <module>
train()
File "Soluce.py", line 50, in train
predictions = network(images.type(torch.LongTensor)) # TODO: fix crash "RuntimeError: expected scalar type Long but found Float"
File "/home/thytu/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/thytu/Prog/PoC/pool_2021/Day3/Admin-XRAI/Admin/network.py", line 15, in forward
t = self.conv1(t)
File "/home/thytu/.local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
result = self.forward(*input, **kwargs)
File "/home/thytu/.local/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 423, in forward
return self._conv_forward(input, self.weight)
File "/home/thytu/.local/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 419, in _conv_forward
return F.conv2d(input, weight, self.bias, self.stride,
RuntimeError: expected scalar type Long but found Float
Do you have any ideas chat the issue is?
Thanks for your time and your help.
(PS: It's not about Cross-Entropy, it happens before)

I also encountered the same problem. My code is as follows:
my code:
import torch
from torchvision.models import alexnet
model = alexnet(pretrained=True)
input_data = torch.randint(255, size=(1, 3, 224, 224), dtype=torch.long)
outputs = model(input_data)
print(outputs)
very simple operation, but it's error:
Traceback (most recent call last):
File "D:\Program Files\JetBrains\PyCharm 2021.1.3\plugins\python\helpers\pydev\pydevd.py", line 1483, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "D:\Program Files\JetBrains\PyCharm 2021.1.3\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "F:/Python/TF2/main.py", line 9, in <module>
outputs = model(input_data)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torchvision\models\alexnet.py", line 46, in forward
x = self.features(x)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\container.py", line 139, in forward
input = module(input)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\module.py", line 1051, in _call_impl
return forward_call(*input, **kwargs)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\conv.py", line 443, in forward
return self._conv_forward(input, self.weight, self.bias)
File "D:\Anaconda\envs\tf2x\lib\site-packages\torch\nn\modules\conv.py", line 440, in _conv_forward
self.padding, self.dilation, self.groups)
RuntimeError: expected scalar type Long but found Float
and but I think it's the weight problem, but I immediately changed this idea. Because it's only an official model, the problem is located in input data. So I changed the code again:
Changed code:
input_data = torch.rand(size=(1, 3, 224, 224))
outputs = model(input_data)
print(outputs)
and then, it's ok:
tensor([[-1.5024e+00, -1.1394e+00, -3.7661e-01, 1.1497e+00, 1.1878e-01,
-6.6696e-01, 3.8399e-01, -1.0095e+00, -1.3813e+00, -1.4772e+00,
...
Process finished with exit code 0
Therefore, my suggestion is that you can try to change the type of input data.

Error in using model.fit in keras with custom train_step

BATCH_SIZE =32
dataset = tf.data.Dataset.from_tensor_slices((question, ans)).shuffle(1000)
dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)
class EncoderDecoder(tf.keras.Model):
def __init__(self,vocab_input=1000,vocab_output=1000,BATCH_SIZE=32):
super().__init__()
#Encoder
self.encoder_vector = tokens.TextVectorization(vocab_input,output_sequence_length=24)
self.encoder_embedding = tf.keras.layers.Embedding(vocab_input,256)
self.encoder_lstm = tf.keras.layers.LSTM(512,return_sequences=True)
self.attention = tf.keras.layers.Attention()
#Decoder
self.decoder_vector = tokens.TextVectorization(vocab_output,output_sequence_length=20)
self.decoder_embedding = tf.keras.layers.Embedding(vocab_output,256)
self.fc = tf.keras.layers.Dense(vocab_output)
self.decoder_lstm = tf.keras.layers.LSTM(512,return_sequences=True)
def loss_function(real, pred):
mask = tf.math.logical_not(tf.math.equal(real, 0))
loss_ = loss_object(real, pred)
mask = tf.cast(mask, dtype=loss_.dtype)
loss_ *= mask
return tf.reduce_mean(loss_)
def train_step(self,data):
loss =0
input = p[0]
targ = p[1]
input = self.encoder_vector(input)
targ = self.decoder_vector(targ)
with tf.GradientTape() as tape:
input = self.encoder_embedding(input)
enc_output,enc_h,enc_c = self.encoder_lstm(input)
attn_output = self.attention([enc_output,enc_h])
dec_h = tf.concat([tf.expand_dims(attn_output, 1), dec_h], axis=-1)
dec_input = tf.expand_dims([self.decoder_vector(['SOS'])]*BATCH_SIZE,1)
predictions = []
for t in range(1,targ.shape[1]):
dec_input = self.decoder_embedding(dec_input)
dec_output,dec_h,dec_c = self.decoder_lstm(dec_input,initial_states=[dec_h,enc_c])
predictions = self.fc(dec_output)
loss+=loss_function(targ[:,t],predictions)
dec_input = tf.expand_dims(targ[:, t], 1)
gradients = tape.gradient(loss,trainable_variables)
self.optimezer.apply_gradients(zip(gradients,trainable_variables))
self.compiled_metrics.update_state(targ,predictions)
return {m.name:m.result() for m in self.metrics}
model = EncoderDecoder()
model.compile(optimizer = 'adam',metrics = [tf.keras.metrics.SparseCategoricalCrossentropy()])
model.fit(dataset,epochs=10)
Epoch 1/10
---------------------------------------------------------------------------
StagingError Traceback (most recent call last)
<ipython-input-153-573fadf7e010> in <module>()
----> 1 model.fit(dataset,epochs=10)
10 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
966 except Exception as e: # pylint:disable=broad-except
967 if hasattr(e, "ag_error_metadata"):
--> 968 raise e.ag_error_metadata.to_exception(e)
969 else:
970 raise
StagingError: in user code:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:571 train_function *
outputs = self.distribute_strategy.run(
<ipython-input-151-46787da81d42>:33 train_step *
input = self.encoder_vector(input)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py:897 __call__ **
self._maybe_build(inputs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer.py:2416 _maybe_build
self.build(input_shapes) # pylint:disable=not-callable
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/preprocessing/text_vectorization.py:528 build
if self._split is not None and not input_shape[1] == 1: # pylint: disable=g-comparison-negation
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/tensor_shape.py:870 __getitem__
return self._dims[key].value
IndexError: list index out of range
I am trying to override the keras model train_step in my custom class.
question is set of English text questions and ans is their respective answers. They are of random length and are not padded as TextVectorization will add padding. So dataset contains question answer pair. Now I batch this and use model.fit.
The error is when assigning input = data[0] and targ = data[1] in train_step. I need help on how to assign the input and output in the train step for a batch.