Develop Reference

How to adjust layers of Autoencoder for a larger dataset

I got a dataset for motion blur. 29216 good images and 29216 blurry images with a 20% train test split. Initially, I was only using a dataset of 2000 total images so my autoencoder looked like this:
# Below is a custom data loader.
def load_image(file, target_size):
image = tf.keras.preprocessing.image.load_img(file, target_size=target_size)
image = tf.keras.preprocessing.image.img_to_array(image).astype('float32') / 255
return image
clean_frames = []
blurry_frames = []
extensions = ['.jpg', 'jpeg', '.png']
for file in tqdm(sorted(os.listdir(good_frames))):
if any(extension in file for extension in extensions):
file_path = os.path.join(good_frames, file)
clean_frames.append(load_image(file_path, (128,128)))
clean_frames = np.array(clean_frames)
for file in tqdm(sorted(os.listdir(bad_frames))):
if any(extension in file for extension in extensions):
file_path = os.path.join(bad_frames, file)
blurry_frames.append(load_image(file_path, (128,128)))
blurry_frames = np.array(blurry_frames)
print('number of clean frames: ', len(clean_frames))
print('number of blurry frames: ', len(blurry_frames))
# Train test split
x_train, x_test, y_train, y_test = train_test_split(clean_frames, blurry_frames, test_size=0.2, random_state=42)
# Network Parameters
input_shape = (128, 128, 3)
batch_size = 32
kernel_size = 3
latent_dim = 256
inputs = Input(shape = input_shape, name = 'encoder_input')
x = inputs
# Layers of the encoder
x = Conv2D(filters=64, kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
x = Conv2D(filters=128, kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
x = Conv2D(filters=256, kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
shape = K.int_shape(x)
x = Flatten()(x)
latent = Dense(latent_dim, name='latent_vector')(x)
encoder = Model(inputs, latent, name='encoder')
encoder.summary()
# Layers of the decoder
latent_inputs = Input(shape=(latent_dim,), name='decoder_input')
x = Dense(shape[1]*shape[2]*shape[3])(latent_inputs)
x = Reshape((shape[1], shape[2], shape[3]))(x)
x = Conv2DTranspose(filters=256,kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
x = Conv2DTranspose(filters=128,kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
x = Conv2DTranspose(filters=64,kernel_size=kernel_size, strides=2, activation='relu', padding='same')(x)
outputs = Conv2DTranspose(filters=3, kernel_size=kernel_size, activation='sigmoid', padding='same', name='decoder_output')(x)
decoder = Model(latent_inputs, outputs, name='decoder')
autoencoder = Model(inputs, decoder(encoder(inputs)), name='autoencoder')
autoencoder.compile(loss='mse', optimizer='adam',metrics=["acc"])
# Automated Learning Rate reducer
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
cooldown=0,
patience=5,
verbose=1,
min_lr=0.5e-6)
callbacks = [lr_reducer]
# Begins training
history = autoencoder.fit(blurry_frames,
clean_frames,
validation_data=(blurry_frames, clean_frames),
epochs=100,
batch_size=batch_size,
callbacks=callbacks)
This would yield an accuracy of 82% after 100 epochs.
Now that I got more data, I believe I can get better performance. How should I adjust my layers and parameters to make the most of the data that I have?

Tensorflow_callback ValueError: logits and labels must have the same shape ((None, 2) vs (None, 1))

I'm classifying images into two classes using binary crossentropy. So, when I'm trying to wrap my keras model with it, I get the error:
ValueError: logits and labels must have the same shape ((None, 2) vs (None, 1))
Please help. My code is:
import tensorflow as tf
import os
# os.path.join('Train', 'benign', 'malignant')
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
import cv2
import imghdr
from matplotlib import pyplot as plt
data_dir = ['Train', 'Test', 'Validation']
image_exts = ['jpg', 'jpeg', 'png', 'bmp']
# print(os.listdir(data_dir))
# print(os.listdir(os.path.join(data_dir, 'benign')))
'''
img = cv2.imread(os.path.join(data_dir, 'benign', 'image_7.jpg'))
print(img)
print(img.shape)
plt.imshow(img)
plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
'''
for i in data_dir:
for image_class in os.listdir(i):
for image in os.listdir(os.path.join(i, image_class)):
image_path = os.path.join(i, image_class, image)
try:
img = cv2.imread(image_path) # Opening up an image (opens as numpy array)
tip = imghdr.what(image_path)
if tip not in image_exts:
print('Image not in ext list {}'.format(image_path))
os.remove(image_path)
except Exception as e:
print('Issue with image {}'.format(image_path))
import numpy as np
data1 = tf.keras.utils.image_dataset_from_directory('Train', shuffle=True) # Building data pipeline
data2 = tf.keras.utils.image_dataset_from_directory('Test', shuffle=True) # Building data pipeline
data3 = tf.keras.utils.image_dataset_from_directory('Validation', shuffle=True) # Building data pipeline
data_iterator1 = data1.as_numpy_iterator() # Helps in accessing data pipeline
data_iterator2 = data2.as_numpy_iterator()
data_iterator3 = data3.as_numpy_iterator()
batch1 = data_iterator1.next() # Accessing data pipeline
batch2 = data_iterator2.next()
batch3 = data_iterator3.next()
# print(batch3[1])
fig, ax = plt.subplots(ncols=4, figsize=(20,20))
for idx, img in enumerate(batch1[0][:4]):
ax[idx].imshow(img.astype(int))
ax[idx].title.set_text(batch1[1][idx])
fig, ax = plt.subplots(ncols=4, figsize=(20,20))
for idx, img in enumerate(batch2[0][:4]):
ax[idx].imshow(img.astype(int))
ax[idx].title.set_text(batch2[1][idx])
fig, ax = plt.subplots(ncols=4, figsize=(20,20))
for idx, img in enumerate(batch3[0][:4]):
ax[idx].imshow(img.astype(int))
ax[idx].title.set_text(batch3[1][idx])
''' Preprocessing Data '''
# Scaling Data
data1 = data1.map(lambda x,y: (x/255, y))
data1.as_numpy_iterator().next()[0].max()
data2 = data2.map(lambda x,y: (x/255, y))
data2.as_numpy_iterator().next()[0].max()
data3 = data3.map(lambda x,y: (x/255, y))
data3.as_numpy_iterator().next()[0].max()
train_size = int(len(data1))
val_size = int(len(data3))
test_size = int(len(data2))
train = data1.take(train_size)
val = data3.take(val_size)
test = data2.take(test_size)
''' Deep Model '''
# The Network
from tensorflow.keras import Model
from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Dense, Flatten, Dropout, AveragePooling2D, Concatenate
def vgg(input_shape, n_classes):
input = Input(input_shape)
x = Conv2D(64, 3, padding='same', activation='relu')(input)
x = Conv2D(64, 3, padding='same', activation='relu')(x)
x = MaxPool2D(2, strides=2, padding='same')(x)
x = Conv2D(128, 3, padding='same', activation='relu')(x)
x = Conv2D(128, 3, padding='same', activation='relu')(x)
x = MaxPool2D(2, strides=2, padding='same')(x)
x = Conv2D(256, 3, padding='same', activation='relu')(x)
x = Conv2D(256, 3, padding='same', activation='relu')(x)
x = Conv2D(256, 3, padding='same', activation='relu')(x)
x = MaxPool2D(2, strides=2, padding='same')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = MaxPool2D(2, strides=2, padding='same')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = Conv2D(512, 3, padding='same', activation='relu')(x)
x = MaxPool2D(2, strides=2, padding='same')(x)
x = Flatten()(x)
x = Dense(4096, activation='relu')(x)
x = Dense(4096, activation='relu')(x)
output = Dense(n_classes, activation='softmax')(x)
model = Model(input, output)
return model
input_shape = (256, 256, 3)
n_classes = 2
model = vgg(input_shape, n_classes)
model.compile('adam', loss=tf.losses.BinaryCrossentropy(), metrics=['accuracy'])
model.summary()
# Train
logdir='logs'
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=logdir)
hist = model.fit(train, epochs=20, validation_data=val, callbacks=[tensorboard_callback])
print(hist.history)

Val_loss is nan at very first of training

val_loss is NaN at very first of training.
The Structure
def trainingResNet(source_folder):
# Preprocessing
image_gen_train = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range=90, horizontal_flip=True)
image_gen_train.fit(x1_train)
# Load Model
model = Network_frame.ResNet18([2, 2, 2, 2])
optimizer = tf.keras.optimizers.Adam()
model.compile(optimizer=optimizer, loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics=['sparse_categorical_accuracy'])
training_dir = '#%s adam_sca_Undistort' % '5'
checkpoint_save_path = os.path.join('checkpoint', training_dir, 'model.{epoch:02d}-{loss:.2f}.ckpt')
# Callback setting
log_dir = os.path.join('cnnlogs', training_dir)
cp_callback = [
tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=0),
tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_save_path, save_weights_only=True, save_best_only=True, monitor='loss', mode='min'),
tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=5, min_delta=0.005, mode='auto', min_lr=0.0001)]
# Training
history = model.fit([x1_train, x2_train], y_train, batch_size=16, epochs=800,
validation_data=([x1_test, x2_test], y_test), validation_freq=1, callbacks=cp_callback)
Network frame
class ResNetBlock(Model):
def __init__(self, filters, strides=1, residual_path=False):
super(ResNetBlock, self).__init__()
self.filters = filters
self.strides = strides
self.residual_path = residual_path
self.c1 = Conv2D(filters, (3, 3), strides=strides, padding='same', use_bias=False)
self.b1 = BatchNormalization()
self.a1 = Activation('relu')
self.c2 = Conv2D(filters, (3, 3), strides=1, padding='same', use_bias=False)
self.b2 = BatchNormalization()
if residual_path:
self.down_c1 = Conv2D(filters, (1, 1), strides=strides, padding='same', use_bias=False)
self.down_b1 = BatchNormalization()
self.a2 = Activation('relu')
def call(self, inputs):
residual = inputs
x = self.c1(inputs)
x = self.b1(x)
x = self.a1(x)
x = self.c2(x)
y = self.b2(x)
if self.residual_path:
residual = self.down_c1(inputs)
residual = self.down_b1(residual)
out = self.a2(y + residual)
return out
class ResNet18(Model):
def __init__(self, block_list, initial_filters=64):
super(ResNet18, self).__init__()
self.num_blocks = len(block_list)
self.block_list = block_list
self.out_filters = initial_filters
self.c1_1 = Conv2D(self.out_filters, (7, 7), strides=1, padding='same', use_bias=False,
kernel_initializer='he_normal')
self.b1_1 = BatchNormalization()
self.a1_1 = Activation('relu')
self.p1_1 = MaxPool2D(pool_size=(2, 2), strides=1, padding='same')
self.blocks = tf.keras.models.Sequential()
for block_id in range(len(block_list)):
for layer_id in range(block_list[block_id]):
if block_id != 0 and layer_id == 0:
block = ResNetBlock(self.out_filters, strides=2, residual_path=True)
else:
block = ResNetBlock(self.out_filters, residual_path=False)
self.blocks.add(block)
self.out_filters *= 2
self.p2_1 = GlobalAveragePooling2D()
self.f1_1 = Dense(512, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001))
self.d1_1 = Dropout(0.2)
self.f2_1 = Dense(64, activation='relu', )
self.f1_2 = Dense(16, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001))
self.d1_2 = Dropout(0.2)
self.f2_2 = Dense(16, activation='relu', )
self.f1 = Dense(64, activation='relu', )
self.d1 = Dropout(0.2)
self.f2 = Dense(32, activation='relu', )
self.d2 = Dropout(0.2)
self.f3 = Dense(40, activation='softmax')
def call(self, inputs):
x1 = self.c1_1(inputs[0])
x1 = self.b1_1(x1)
x1 = self.a1_1(x1)
x1 = self.p1_1(x1)
x1 = self.blocks(x1)
x1 = self.p2_1(x1)
x1 = self.f1_1(x1)
x1 = self.d1_1(x1)
x1 = self.f2_1(x1)
x2 = self.f1_2(inputs[1])
x2 = self.d1_2(x2)
x2 = self.f2_2(x2)
x = tf.keras.layers.concatenate([x1, x2])
x = self.f1(x)
x = self.d1(x)
x = self.f2(x)
x = self.d2(x)
y = self.f3(x)
return y
The network can be fitted normally, but val_loss is NaN at the first epoch
2864/2875 [============================>.] - ETA: 0s - loss: 13.3955 - sparse_categorical_accuracy: 0.0583
2875/2875 [==============================] - 73s 25ms/sample - loss: 13.3595 - sparse_categorical_accuracy: 0.0584 - val_loss: nan - val_sparse_categorical_accuracy: 0.0668
Epoch 2/800
The network can be fitted normally, but val_loss is NaN at the first epoch
The approaches that have been tried
Use other version of TF (currently 2.1)
Use LeakyReLU instead of ReLU
Decrease lr
Make sure the input does not have NaN
Make sure the target is computable by the loss function (greater than 0)
I would like to ask if there is any way to fix it or
it is possible to reproduce the val_loss calculation to get the location of the NaN if the model checkpoint is saved
Many thanks

VAE reconstructed images are extremely blurry

I am very new to machine learning and have build a VAE from the Keras VAE code example. I only changed a few layers in the model. I trained the model on the Kaggle cats and dogs dataset and then tried to reconstruct a few images. All the reconstructed images looked the same just like these Reconstructed Images. What could be the cause of this? Is it due to a bad model, short training time or do I have a mistake in reconstructing the images?
The encoder model:
latent_dim = 2
encoder_inputs = keras.Input(shape=(328, 328, 3))
x = layers.Conv2D(32, 3, strides=2, padding="same")(encoder_inputs)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization()(x)
x = layers.Conv2D(64, 3,strides=2, padding="same")(x)
x = layers.Activation("relu")(x)
x = layers.BatchNormalization()(x)
x = layers.Conv2D(128, 3,strides=2, padding="same")(x) #neu
x = layers.Activation("relu")(x)
x = layers.BatchNormalization()(x)
x = layers.Flatten()(x)
x = layers.Dense(16, activation="relu")(x)
z_mean = layers.Dense(latent_dim, name="z_mean")(x)
z_log_var = layers.Dense(latent_dim, name="z_log_var")(x)
z = Sampling()([z_mean, z_log_var])
encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name="encoder")
encoder.summary()
The decoder model:
x = layers.Dense(41 * 41 * 128, activation="relu")(latent_inputs)
x = layers.Reshape((41, 41, 128))(x)
x = layers.Conv2DTranspose(128, 3, activation="relu", strides=2, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Conv2DTranspose(64, 3, activation="relu", strides=2, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Conv2DTranspose(32, 3, activation="relu", strides=2, padding="same")(x)
x = layers.BatchNormalization()(x)
decoder_outputs = layers.Conv2DTranspose(3, 3, activation="sigmoid", padding="same")(x)
decoder = keras.Model(latent_inputs, decoder_outputs, name="decoder")
decoder.summary()
The training:
train_data_dir ='/content/PetImages'
nb_train_samples = 200
nb_epoch = 50
batch_size = 32
img_width = 328
img_height = 328
def fixed_generator(generator):
for batch in generator:
yield (batch, batch)
train_datagen = ImageDataGenerator(
rescale=1./255,
)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode=None)
vae = VAE(encoder, decoder)
vae.compile(optimizer=keras.optimizers.Adam())
vae.fit(
fixed_generator(train_generator),
steps_per_epoch=nb_train_samples,
epochs=nb_epoch,
)
And reconstructing the images:
import matplotlib.pyplot as plt
test2_datagen = ImageDataGenerator(rescale=1./255)
test2_generator = test2_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=10,
class_mode=None)
sample_img = next(test2_generator)
z_points = vae.encoder.predict(sample_img)
reconst_images = vae.decoder.predict(z_points)
fig = plt.figure(figsize=(10, 8))
fig.subplots_adjust(hspace=0.1, wspace=0.1)
n_to_show =2
for i in range(n_to_show):
img = sample_img[i].squeeze()
sub = fig.add_subplot(2, n_to_show, i+1)
sub.axis('off')
sub.imshow(img)
for i in range(n_to_show):
img = reconst_images[i].squeeze()
sub = fig.add_subplot(2, n_to_show, i+n_to_show+1)
sub.axis('off')
sub.imshow(img)

Why do I get this graph disconnected error?

I'm trying to create a densenet but when I try to compile the model I get this error message. Here's my model:
from tensorflow import keras
from keras.utils import plot_model
dropoutRate = 0.2
def globalAvgPooling(x):
height = np.shape(x)[2]
width = np.shape(x)[1]
poolSize = [width, height]
return tf.keras.layers.AveragePooling2D(pool_size=poolSize, strides=1)(x)
def concatenation(layers):
return tf.keras.layers.concatenate(layers, axis=3)
class DenseNet():
def __init__(self, filters, numBlocks, numClasses, training):
self.filters = filters
self.numBlocks = numBlocks
self.training = training
self.numClasses = numClasses
self.model = self.denseNet()
def bottleneckLayer(self, inputX):
x = tf.keras.layers.BatchNormalization()(inputs=inputX, training=self.training)
x = tf.keras.activations.relu(x)
x = tf.keras.layers.Conv2D(use_bias=False, filters=self.filters, kernel_size=1, strides=1, padding='same')(x)
x = tf.layers.dropout(inputs=x, rate=dropoutRate, training=self.training)
x = tf.keras.layers.BatchNormalization()(inputs=inputX, training=self.training)
x = tf.keras.activations.relu(x)
x = tf.keras.layers.Conv2D(use_bias=False, filters=self.filters, kernel_size=3, strides=1, padding='same')(x)
x = tf.layers.dropout(inputs=x, rate=dropoutRate, training=self.training)
return x
def denseBlock(self, inputX, numLayers):
concatLayers = list()
concatLayers.append(inputX)
x = self.bottleneckLayer(inputX=inputX)
concatLayers.append(x)
for i in range(self.numBlocks - 1):
x = concatenation(concatLayers)
x = self.bottleneckLayer(inputX=x)
concatLayers.append(x)
x = concatenation(concatLayers)
return x
def transitionLayer(self, inputX):
x = tf.keras.layers.BatchNormalization()(inputs=inputX, training=self.training)
x = tf.keras.activations.relu(x)
x = tf.keras.layers.Conv2D(use_bias=False, filters=self.filters, kernel_size=1, strides=1, padding='same')(x)
x = tf.layers.dropout(inputs=x, rate=dropoutRate, training=self.training)
x = tf.keras.layers.AveragePooling2D(pool_size=[2,2], strides=2, padding='valid')(x)
return x
def denseNet(self):
inputs = keras.Input(shape=(32,32,3))
x = tf.keras.layers.Conv2D(use_bias=False, filters=self.filters, kernel_size=7, strides=1, padding='same')(inputs)
x = self.denseBlock(inputX=x, numLayers=6) #Dense block 1 with 6 layers
x = self.transitionLayer(inputX=x)
x = self.denseBlock(inputX=x, numLayers=12) #Dense block 2 with 12 layers
x = self.transitionLayer(inputX=x)
x = self.denseBlock(inputX=x, numLayers= 48) #Dense block 3 with 48 layers
x = self.transitionLayer(inputX=x)
x = self.denseBlock(inputX=x, numLayers=32) #Dense block 4 with 32 layers (final block)
x = globalAvgPooling(x=x)
x = tf.keras.layers.Softmax()(x)
outputs = tf.keras.layers.Dense(units=self.numClasses)(x)
model = keras.Model(inputs=inputs, outputs=outputs)
return x
tf.compat.v1.disable_eager_execution()
growthK = 24
numBlock = 2
cameraModel = DenseNet(filters=growthK, numBlocks=numBlock, numClasses=4, training=True).model
Here is the error message I get:
ValueError: Graph disconnected: cannot obtain value for tensor
Tensor("dropout_109/dropout/mul_1:0", shape=(?, 32, 32, 24), dtype=float32)
at layer "concatenate_48". The following previous layers were accessed without issue:
['input_6', 'conv2d_120', 'batch_normalization_115', 'tf_op_layer_Relu_115', 'conv2d_122', 'dropout_108']
What am I doing wrong?

The error comes from the line:
x = tf.keras.layers.BatchNormalization()(inputs=inputX, training=self.training)
as I think you have a bad inputs parameter in it.
It should looks like:
x = tf.keras.layers.BatchNormalization()(inputs=x, training=self.training)
That is why value the Graph disconnected.