Develop Reference

Keras: Overfitting Model?

I am trying to create a binary image classification model using the malaria dataset from NIH (National Library of Medicine) which contains approximately 27,000 images of each class (infected/uninfected).
There seems to be overfitting and I have tried to play around with different batch sizes, steps per epoch/validation steps, using different hidden layers and adding callbacks etc. The graph always shows a straight line that is either dramatically increasing or decreasing, rather than increasing steadily with some decreases as it learns (which from my understanding, is how it should be). Below is an example, most turn out somewhat similar to this.
I'm new to deep learning, I have read a lot about overfitting and trying to find a solution. But I think there must be something I'm doing wrong and/or misunderstanding. If someone is able to spot something that doesn't look right and is able to point me in the right direction, it would be greatly appreciated!
from keras.layers import MaxPooling2D, Conv2D, Flatten, Dense, Dropout
from keras_preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from keras.models import Sequential
import matplotlib.pyplot as plt
import constants as c
import numpy as np
import keras
# Clear session and instantiate model
keras.backend.clear_session()
model = Sequential()
# Load images & labels
cells = np.load(c.cells_path)
labels = np.load(c.labels_path)
# Shuffle the entire dataset
n = np.arange(cells.shape[0])
np.random.shuffle(n)
# Update numpy files with shuffled data
cells = cells[n]
labels = labels[n]
# Split the dataset into train/validation/test
train_x, test_x, train_y, test_y = train_test_split(cells, labels, test_size=1 - c.train_ratio, shuffle=False)
val_x, test_x, val_y, test_y = train_test_split(test_x, test_y, test_size=c.test_ratio / (c.test_ratio + c.val_ratio),
shuffle=False)
# The amount of images in each set
print('Training data shape: ', train_x.shape)
print('Validation data shape: ', val_x.shape)
print('Testing data shape: ', test_x.shape)
# Neural network
model.add(Conv2D(32, (3, 3), input_shape=c.input_shape, activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(units=1, activation='sigmoid'))
# Compile the model
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
# Data augmentation
train_datagen = ImageDataGenerator(rescale=1. / 255,
rotation_range=20,
width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.05,
zoom_range=0.05,
horizontal_flip=True,
fill_mode='nearest')
validation_datagen = ImageDataGenerator(rescale=1. / 255)
testing_datagen = ImageDataGenerator(rescale=1. / 255)
training_dataset = train_datagen.flow(train_x, train_y, batch_size=32)
validation_dataset = validation_datagen.flow(val_x, val_y, batch_size=32)
testing_dataset = validation_datagen.flow(val_x, val_y, batch_size=32)
# Add callbacks to prevent overfitting
es = EarlyStopping(monitor='accuracy',
min_delta=0,
patience=2,
verbose=0,
mode='max')
rlrop = ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=0.5,
min_lr=0.001)
checkpoint = ModelCheckpoint("Model.h5")
# Perform backpropagation and update weights in model
history = model.fit_generator(training_dataset,
epochs=50,
validation_data=validation_dataset,
callbacks=[es, checkpoint, rlrop])
# Save model & weights
model.save_weights("Model_weights.h5")
model.save("Model.h5")
# Plot accuracy graph
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.show()

Doesn't seem to be a case of overfitting. Without looking much at it, would do the following:
leave the filters at 32 in the fist layer and gradually double on every following convolutional layer.
Since the variations in the images aren't that significant lower the Dropout rate.
Oddly enough, this is something I built when I was first trying Tensorflow 2.0, you can check it here.

Keras.predict always gives "1."as the output

I trained a binary classifier distinguish clear MNIST images from blurry images. All images are 28*28*1 grayscale digits and I have 40000 for training, 10000 for validating and 8000 for testing. My code looks like:
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
import cv2
import numpy as np
import glob
from PIL import Image
img_width, img_height = 28, 28#all MNIST images are of size (28*28)
train_data_dir = '/Binary Classifier/data/train'#train directory generated by train_cla
validation_data_dir = '/Binary Classifier/data/val'#validation directory generated by val_cla
train_samples = 40000
validation_samples = 10000
epochs = 20
batch_size = 16
if K.image_data_format() == 'channels_first':
input_shape = (1, img_width, img_height)
else:
input_shape = (img_width, img_height, 1)
#build a sequential model to train data
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
train_datagen = ImageDataGenerator(#train data generator
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
val_datagen = ImageDataGenerator(rescale=1. / 255)#validation data generator
train_generator = train_datagen.flow_from_directory(#train generator
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary',color_mode = 'grayscale')
validation_generator = val_datagen.flow_from_directory(#validation generator
validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary',color_mode = 'grayscale')
model.fit_generator(#fit the generator to train and validate the model
train_generator,
steps_per_epoch=train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=validation_samples // batch_size)
#model.save_weights('output.h5')#save the output as HDF5 file
filelist = glob.glob('/Binary Classifier/data/image_data/*.png')
x = np.array([np.array(Image.open(fname)) for fname in filelist])
x = np.expand_dims(x, axis=3)
ones=model.predict(x)
But my output prediction in ones[] are all [1.] while the accuracy in training is actually really high(almost perfect). Does anyone know why?
Edit: I think I may get more help if I can show my image data. Basically the MNIST image in the directory is either a (clear) or a (blurry). All are (28*28*1) grayscale images whose format is .png. There are 40000 digits in '/Binary Classifier/data/train' for training, 10000 digits in '/Binary Classifier/data/val' for validation and 58000 digits in '/Binary Classifier/data/image_data/ for testing.

Some suggestions:
Pull data directly from one of your generators and test on that. Treat the generator like you would a list in a for loop to get image/label pairs out. This should sort out any differences in the way you are obtaining data and its formatting (e.g. channel order).
Check how many examples you have in each subdirectory of train/ and val/.
Change your metric to binary_accuracy since you are posing the problem as a binary classification problem (network only has one output).

Keras training model with images

My first go at training a model on a dataset, this is the data from a pandas dataset
In [16]: exists.head()
Out[16]:
id url \
1 0a58358a2afd3e4e http://lh6.ggpht.com/-igpT6wu0mIA/ROV8HnUuABI/...
2 6b2bb500b6a38aa0 http://lh6.ggpht.com/-vKr5G5MEusk/SR6r6SJi6mI/...
3 b399f09dee9c3c67 https://lh3.googleusercontent.com/-LOW2cjAqubA...
4 19ace29d77a5be66 https://lh5.googleusercontent.com/-tnmSXwQcWL8...
5 2c9c54b62f0a6a37 https://lh5.googleusercontent.com/-mEaSECO7D-4...
landmark_id exists filename
1 6651 True training_images/0a58358a2afd3e4e.jpg
2 11284 True training_images/6b2bb500b6a38aa0.jpg
3 8429 True training_images/b399f09dee9c3c67.jpg
4 6231 True training_images/19ace29d77a5be66.jpg
5 10400 True training_images/2c9c54b62f0a6a37.jpg
it shows the training image in filename and the classification name in landmark_id
This is the way I've written the model to train it
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(3, 150, 150)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(5))
model.add(Dense(activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, verbose=0, mode='auto')
checkpointer = ModelCheckpoint(filepath="best_weights.hdf5", verbose=0, save_best_only=True) # save best model
model.compile(loss='binary_crossentropy',
optimizer='adam',
callbacks=[monitor,checkpointer],
verbose=0,
epochs=1000,
metrics=['accuracy'])
batch_size = 16
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255)
# this is a generator that will read pictures found in
# subfolers of 'data/train', and indefinitely generate
# batches of augmented image data
train_generator = train_datagen.flow_from_directory(
'training_images', # this is the target directory
target_size=(150, 150), # all images will be resized to 150x150
batch_size=batch_size,
class_mode='binary') # since we use binary_crossentropy loss, we need binary labels
# this is a similar generator, for validation data
validation_generator = test_datagen.flow_from_directory(
'test_images',
target_size=(150, 150),
batch_size=batch_size,
class_mode='binary')
model.fit_generator(
train_generator,
steps_per_epoch=2000 // batch_size,
epochs=50,
validation_data=validation_generator,
validation_steps=800 // batch_size)
model.load_weights('best_weights.hdf5') # load weights from best model
model.save('last_model.h5')
I don't know how I'm supposed to put the labels to the image while training. So when it trains and loops through the images in the training_images folder.

Samuel,
Your FitGenerator is getting the training input labels from the flow_from_directory method. This method uses the folder structure to determine the training categories. Since your class is binary, and you have a single sigmoid output, I'm assuming that you are doing a Hot Dog - Not Hot Dog type of classification where you just want a single probability value.
The other hint for me that you care about the single probability that something is a category or not is that your loss function is binary_crossentropy.
Check your training data folder. Look at how the data is organized. This should be set up such that the data is organized correctly.
You seem to be hinting that you want to create multiple labels (e.g., car, boat, train). If this is the case, you will want to create those folders under train and validate and put the images in the respective folder. You will need to change several things about your model if you do this though. Your loss, output layer size, and output layer activations will change accordingly.

How to check what features are extracted while training and testing a CNN model for image classification?

I'm using CNN for training and testing images of seeds. I want to know:
What features are getting extracted at every layer?
Is there any way to represent it in a graphical or image format?
How do I define my classifier to extract only specific features?
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
# dimensions of our images.
img_width, img_height = 150, 150
train_data_dir = 'Train_Walnut_Seed/train'
validation_data_dir = 'Train_Walnut_Seed/validation'
nb_train_samples = 70
nb_validation_samples = 9
epochs = 50
batch_size = 16
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size)
model.save('first_try_walnut.h5')
The above code is for training the classifier using CNN. how to visually represent the output at each layer while training.
Also how to deploy my trained model into a protocolbuffer(.pb) file for using it in my android project

I believe the best way, or at least the best way I know of to extract useful features would be using an autoencoder.
Check out this article from the Keras blog.
Cheers,
Gabriel

I know this probably isn't an issue anymore, but I just thought I'd add this in case it's useful to someone else. As the features output by a CNN aren't really human-readable it is difficult to inspect them. One way is to use t-SNE which gives a visual indication of which embedded representations of the images are close to each other. Another way to do this is using a 'heat map' which shows in more detail which parts of an image are activating parts of the CNN. This post has a nice explanation of some of these techniques: http://cs231n.github.io/understanding-cnn/
Getting a classifier to focus on certain features is difficult - either you need to change the network architecture or use image pre-processing to accentuate the features you want the network to focus on. I'm afraid I can't really give more details on that.

Keras multi-class model with wrong dimensions

New to Keras, trying to reimplement this following binary image classification example from: https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html
It works for binary classification for me.
Rebuilding it for a 3-class classification I am getting the following dimensions mismatch error:
60 epochs=50,
61 validation_data=validation_generator,
---> 62 validation_steps=250 // batch_size)
ValueError: Error when checking target: expected activation_50 to have shape (None, 1) but got array with shape (16, 3)
This is my current implementation:
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
K.set_image_dim_ordering('th')
batch_size = 16
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255)
# this is a generator that will read pictures found in
# subfolers of 'data/train', and indefinitely generate
# batches of augmented image data
train_generator = train_datagen.flow_from_directory(
'F://train_data//', # this is the target directory
target_size=(150, 150), # all images will be resized to 150x150
batch_size=batch_size,
class_mode='categorical') # since we use binary_crossentropy loss, we need binary labels
# this is a similar generator, for validation data
validation_generator = test_datagen.flow_from_directory(
'F://validation_data//',
target_size=(150, 150),
batch_size=batch_size,
class_mode='categorical')
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(3, 150, 150)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_first"))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('softmax')) # instead of sigmoid
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['accuracy'])
# another loss: sparse_categorical_crossentropy
model.fit_generator(
train_generator,
steps_per_epoch=1800 // batch_size,
epochs=50,
validation_data=validation_generator,
validation_steps=250 // batch_size)
So far I have changed the activation function of the output layer from sigmoid to softmax. Changed class_mode from binary to categorical. Can't seem to find the problem.
Also, I am aware of similar questions on StackOverflow:
Multi-Output Multi-Class Keras Model
Train multi-class image classifier in Keras
Multi-class classification using keras
But none of the solutions helped me.

You need to change the final Dense layer to model.add(Dense(3)). Softmax activation expects the units in the Dense layer to match the number of classes.
Also, if you are going to use loss='sparse_categorical_crossentropy', remember to change class_mode into 'sparse'. Your current setting, class_mode='categorical', should be used with loss='categorical_crossentropy'.