I attempted to fix it after seeing this [post][1] by adding dropout, but it didn't work. And I'm still getting consistent accuracy, so any assistance would be greatly appreciated.
import os
os.environ['KAGGLE_CONFIG_DIR'] = "/content"
!kaggle datasets download -d jakeshbohaju/brain-tumor
!unzip \*.zip -d brain_tumor_dataset
!rm -rf yes
!rm -rf no
!rm -rf *.zip
# Commented out IPython magic to ensure Python compatibility.
import pandas as pd
import numpy as np
import os
import tensorflow as tf
import cv2
from tensorflow import keras
from tensorflow.keras import layers, Input
from keras.layers import InputLayer, MaxPooling2D, Flatten, Dense, Conv2D, Dropout, BatchNormalization
from keras.losses import BinaryCrossentropy
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions, ResNet50
from tensorflow.keras.optimizers import Adam, SGD
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from PIL.Image import open
from matplotlib import pyplot as plt
import matplotlib.image as mpimg
import random
# %matplotlib inline
# Constants
IMAGE_DATASET = "/content/brain_tumor_dataset/Brain Tumor/Brain Tumor"
IMAGE_DATASET_RAW = '/content/brain_tumor_dataset/Brain Tumor/Brain Tumor'
WORKING_FOLDER = "/content/brain_tumor_dataset/working"
IMG_HEIGHT = 224
IMG_WIDTH = 224
EPOCHS = 100
# # Image3202
plt.figure(figsize=(20,20))
test_folder="/content/brain_tumor_dataset/Brain Tumor/Brain Tumor/Image100.jpg"
img=mpimg.imread(test_folder)
print(img.size)
ax=plt.subplot(1,5,4)
# # ax.title.set_text(file)
plt.imshow(img)
# We will import the csv file containing the features and the classes of the images
cortex_df = pd.read_csv("/content/brain_tumor_dataset/Brain Tumor.csv")
cortex_df.head()
plt.figure(figsize=(20,20))
test_folder="/content/brain_tumor_dataset/Brain Tumor/Brain Tumor"
for i in range(5):
file = random.choice(os.listdir(test_folder))
image_path= os.path.join(test_folder, file)
img=mpimg.imread(image_path)
ax=plt.subplot(1,5,i+1)
ax.title.set_text(file)
plt.imshow(img)
dataset_df = pd.DataFrame()
dataset_df["Image"] = cortex_df["Image"]
dataset_df["Class"] = cortex_df["Class"]
path_list = []
for img_path in os.listdir(IMAGE_DATASET):
path_list.append( os.path.join(IMAGE_DATASET,img_path))
path_dict = {os.path.splitext(os.path.basename(x))[0]: x for x in path_list}
dataset_df["paths"] = cortex_df["Image"].map(path_dict.get)
dataset_df["pixels"] = dataset_df["paths"].map(lambda x:np.asarray(open(x).resize((IMG_HEIGHT,IMG_WIDTH))))
dataset_df.head()
image_list = []
for i in range(len(dataset_df)):
brain_image = dataset_df["pixels"][i].astype(np.float32)
brain_image /= 255
image_list.append(brain_image)
X = np.array(image_list)
print(X.shape)
y = np.array(dataset_df.Class)
#y.shape
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
print('The shape of the X_train :'+' '+str(X_train.shape))
print('The size of the X_train :'+' '+str(X_train.shape[0]))
print('The shape of the X_test :'+' '+str(X_test.shape))
print('The size of the X_test:'+' '+str(X_test.shape[0]))
def model(input_shape):
# res_conv = ResNet50(include_top=False, weights="imagenet", input_tensor=None, input_shape=input_shape, pooling=None)
model = Sequential()
model.add(Input(shape=input_shape))
model.add(Conv2D(16, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(16, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Dropout(0.25))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last", padding='same'))
model.add(Conv2D(32, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(32, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Dropout(0.25))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last", padding='same'))
model.add(Conv2D(64, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(64, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(64, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(64, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Dropout(0.25))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last", padding='same'))
model.add(Conv2D(128, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(128, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(128, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(128, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Dropout(0.25))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last", padding='same'))
model.add(Conv2D(256, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(256, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(256, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Conv2D(256, kernel_size=3, strides=(2, 2), padding="same", activation="relu", kernel_initializer="he_normal"))
model.add(Dropout(0.25))
model.add(MaxPooling2D(pool_size=(2, 2), data_format="channels_last", padding='same'))
model.add(Flatten())
model.add(Dense(256, activation="relu"))
model.add(Dropout(0.5))
model.add(Dense(128, activation="relu"))
model.add(Dropout(0.4))
model.add(Dense(1, activation="sigmoid")) # Never use sigmoid for binary classification
return model
model = model(input_shape = (IMG_HEIGHT, IMG_WIDTH, 3))
model.summary()
# optimizer = Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name="Adam",)
optimizer = SGD(learning_rate=0.01)
loss_fn = BinaryCrossentropy(from_logits=True)
model.compile(optimizer=optimizer, loss=loss_fn, metrics=['accuracy'])
# Training the model
history = model.fit(x=X_train, y=y_train, epochs=EPOCHS, batch_size=10)
loss = history.history["loss"]
acc = history.history["accuracy"]
epoch = np.arange(EPOCHS)
plt.plot(epoch, loss)
# plt.plot(epoch, val_loss)
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.title('Training Loss')
plt.legend(['train', 'val'])
epoch = np.arange(EPOCHS)
plt.plot(epoch, acc)
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.title('Training Accuracy');
eval_score = model.evaluate(X_test, y_test)
print("Test loss:", eval_score[0])
print("Test accuracy:", eval_score[1])
some of Outputs
Epoch 70/100
301/301 [==============================] - 4s 13ms/step - loss: 0.6864 - accuracy: 0.5577
Epoch 71/100
301/301 [==============================] - 4s 13ms/step - loss: 0.6867 - accuracy: 0.5577
Epoch 72/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6866 - accuracy: 0.5577
Epoch 73/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6866 - accuracy: 0.5577
Epoch 74/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6867 - accuracy: 0.5577
Epoch 75/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6868 - accuracy: 0.5577
Epoch 76/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6869 - accuracy: 0.5577
Epoch 77/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6867 - accuracy: 0.5577
Epoch 78/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6866 - accuracy: 0.5577
Epoch 79/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6867 - accuracy: 0.5577
Epoch 80/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6864 - accuracy: 0.5577
Epoch 81/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6866 - accuracy: 0.5577
Epoch 82/100
301/301 [==============================] - 4s 12ms/step - loss: 0.6866 - accuracy: 0.5577
Epoch 83/100
301/301 [==============================] - 4s 13ms/step - loss: 0.6867 - accuracy: 0.5577
Epoch 84/100
301/301 [==============================] - 4s 13ms/step - loss: 0.6867 - accuracy: 0.557
I tried a few other techniques, such as adding Epoch or dropout, but the accuracy remained the same?
[1]: Keras model gets constant loss and accuracy
One problem in your case is that you are using from_logits = True simultaneously with the sigmoid activation function.
The logit is the unnormalised prediction of a model; in other words, it is the prediction of the network before one applies sigmoid or softmax.
The default parameters are the following:
tf.keras.losses.BinaryCrossentropy(
from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,
name='binary_crossentropy'
)
If you use from_logits=True, then you have to change this line here: model.add(Dense(1), which in fact translates to linear activation.
Then your network will start learning.
Another suggestion would be to decrease the learning_rate, to an initial one of 0.0001.
Related
I am building a denoising autoencoder (DAE) to denoise respiratory signals. I pass through the model both noisy and clean versions of the signal (in frame sizes as multiples of 1024).
I've set up my model up as follows:
class NoiseReducer(tf.keras.Model):
def __init__(self):
super().__init__()
self.encoder = tf.keras.Sequential([
# Input(shape=(window_size, 1)),
Masking(mask_value=np.nan, input_shape=(window_size, 1)),
Conv1D(filters=128, kernel_size=32, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu'),
Dense(128, activation='elu'),
Conv1D(filters=32, kernel_size=16, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu'),
Conv1D(filters=16, kernel_size=8, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu')
])
self.decoder = tf.keras.Sequential([
Conv1DTranspose(filters=16, kernel_size=8, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu')
Conv1DTranspose(filters=32, kernel_size=16, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu'),
Dense(128, activation='elu'),
Conv1DTranspose(filters=128, kernel_size=32, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', kernel_initializer='glorot_normal', activation='elu'),
Conv1D(filters=1, kernel_size=2, strides=1, kernel_constraint=max_norm(max_norm_value), padding='same', activation='sigmoid')
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
dae = NoiseReducer()
adam_optimizer=tf.keras.optimizers.Adam(
learning_rate=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
sgd_optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)
dae.compile(optimizer=sgd_optimizer, loss='mean_squared_error', metrics='accuracy')
history = dae.fit(X_noisy_train,
X_clean_train,
epochs=epochs,
batch_size=batch_size,
shuffle=False,
validation_split=0.3,
callbacks=[tb_callback]
)
RESULTS:
13/13 [==============================] - 16s 1s/step - loss: 0.2185 - accuracy: 0.8272 - val_loss: 0.2143 - val_accuracy: 0.8288
Epoch 2/100
13/13 [==============================] - 12s 898ms/step - loss: 0.2120 - accuracy: 0.8272 - val_loss: 0.2082 - val_accuracy: 0.8288
Epoch 3/100
13/13 [==============================] - 12s 908ms/step - loss: 0.2057 - accuracy: 0.8272 - val_loss: 0.2017 - val_accuracy: 0.8288
Epoch 4/100
13/13 [==============================] - 12s 906ms/step - loss: 0.1997 - accuracy: 0.8272 - val_loss: 0.1956 - val_accuracy: 0.8288
Epoch 5/100
13/13 [==============================] - 12s 907ms/step - loss: 0.1938 - accuracy: 0.8272 - val_loss: 0.1898 - val_accuracy: 0.8288
When running the model, the accuracy and validation accuracy is stuck at around 0.827 for both and does not change at all throughout the epochs (100 in total) suggesting that the model isn't learning anything. The MSE is however descreasing with epochs.
For my datasets I have set any nan values to 0
In terms of solutions I have implemented the following changes to my model but to no success:
Increased filter length of conv1D layers
Tested different learning rates for both SGD and Adam
Test with ELU (Exponential Linear Unit) activation instead of RELU
Use more int. dense layers with more neurons.
Using glorot (commonly known as Xavier) initializer
Use SGD instead of Adam
Change window size with different multiples of 1024
None of these seem to change the accuracy. After model completion and reconstructing the signal (from the noisy) I get a straight line cutting through 0.345 illustrating that the model has not learnt anything and can not reconstruct the signal.
What other strategies/alleys should I explore around this?
I'm trying to create a binary classifier that can differentiate between MRIs of alzheimer's patients and healthy individuals.
These are the stats so far:
1032 training images
400 validation images
Running a simple model as shown below
I have both the raw 160x160 images as well as the images after edge detection
Model:
model = Sequential([
Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same', input_shape=(160,160,3)),
MaxPool2D(pool_size=(2, 2), strides=2),
Flatten(),
Dense(units=2, activation='softmax')
])
As you can see - it's very simple, something I've done purposefully to try and remedy the issue of overfitting.
Output:
11/11 [==============================] - 2s 194ms/step - loss: 0.7604 - accuracy: 0.5155 - val_loss: 0.7081 - val_accuracy: 0.5000
Epoch 2/20
11/11 [==============================] - 2s 185ms/step - loss: 0.6885 - accuracy: 0.5223 - val_loss: 0.6942 - val_accuracy: 0.4839
Epoch 3/20
11/11 [==============================] - 2s 185ms/step - loss: 0.6802 - accuracy: 0.5854 - val_loss: 0.6985 - val_accuracy: 0.4931
Epoch 4/20
11/11 [==============================] - 2s 185ms/step - loss: 0.6717 - accuracy: 0.5932 - val_loss: 0.6996 - val_accuracy: 0.4677
Epoch 5/20
11/11 [==============================] - 2s 195ms/step - loss: 0.6512 - accuracy: 0.6175 - val_loss: 0.7124 - val_accuracy: 0.5115
Epoch 6/20
11/11 [==============================] - 2s 185ms/step - loss: 0.6345 - accuracy: 0.6476 - val_loss: 0.7073 - val_accuracy: 0.5253
Epoch 7/20
11/11 [==============================] - 2s 185ms/step - loss: 0.6118 - accuracy: 0.6680 - val_loss: 0.6920 - val_accuracy: 0.5207
Epoch 8/20
11/11 [==============================] - 2s 185ms/step - loss: 0.5817 - accuracy: 0.7068 - val_loss: 0.6964 - val_accuracy: 0.5207
Epoch 9/20
11/11 [==============================] - 2s 184ms/step - loss: 0.5528 - accuracy: 0.7272 - val_loss: 0.7123 - val_accuracy: 0.5161
Epoch 10/20
11/11 [==============================] - 2s 193ms/step - loss: 0.5239 - accuracy: 0.7417 - val_loss: 0.7397 - val_accuracy: 0.5392
Epoch 11/20
11/11 [==============================] - 2s 186ms/step - loss: 0.5106 - accuracy: 0.7427 - val_loss: 0.7551 - val_accuracy: 0.5461
Epoch 12/20
11/11 [==============================] - 2s 197ms/step - loss: 0.4920 - accuracy: 0.7650 - val_loss: 0.7402 - val_accuracy: 0.5438
Epoch 13/20
11/11 [==============================] - 2s 190ms/step - loss: 0.4741 - accuracy: 0.7835 - val_loss: 0.7564 - val_accuracy: 0.5507
Epoch 14/20
11/11 [==============================] - 2s 188ms/step - loss: 0.4591 - accuracy: 0.7767 - val_loss: 0.7445 - val_accuracy: 0.5300
Epoch 15/20
11/11 [==============================] - 2s 185ms/step - loss: 0.4486 - accuracy: 0.7767 - val_loss: 0.7712 - val_accuracy: 0.5415
Epoch 16/20
11/11 [==============================] - 2s 185ms/step - loss: 0.4503 - accuracy: 0.7806 - val_loss: 0.7446 - val_accuracy: 0.5346
Epoch 17/20
11/11 [==============================] - 2s 188ms/step - loss: 0.4404 - accuracy: 0.7670 - val_loss: 0.7669 - val_accuracy: 0.5553
Epoch 18/20
11/11 [==============================] - 2s 184ms/step - loss: 0.4169 - accuracy: 0.8078 - val_loss: 0.7804 - val_accuracy: 0.5576
Epoch 19/20
11/11 [==============================] - 2s 184ms/step - loss: 0.3987 - accuracy: 0.7971 - val_loss: 0.7846 - val_accuracy: 0.5507
Epoch 20/20
11/11 [==============================] - 2s 192ms/step - loss: 0.3977 - accuracy: 0.7981 - val_loss: 0.8060 - val_accuracy: 0.5461
Things I've tried so far:
resizing the image to a smaller input
adding dropout layers
using preprocessed images where it's just the edges shown
ensuring both classes in both training and validation datasets are evenly distributed
changing learning rate
reducing number of parameters to be of the same magnitude of the number of training images i have
I am literally out of ideas, I'm not sure how to move forward with this so I would appreciate any tips or advice.
All my code:
# Use ImageDataGenerator to create 3 lots of batches
train_batches = ImageDataGenerator(
rescale=1/255).flow_from_directory(directory=train_path,
target_size=(80,80), classes=['cn', 'ad'], batch_size=100,
color_mode="rgb")
valid_batches = ImageDataGenerator(
rescale=1/255).flow_from_directory(directory=valid_path,
target_size=(80,80), classes=['cn', 'ad'], batch_size=100,
color_mode="rgb")
# test_batches = ImageDataGenerator(
# rescale=1/255).flow_from_directory(directory=test_path,
# target_size=(224,224), classes=['cn', 'ad'], batch_size=10,
# color_mode="rgb")
imgs, labels = next(train_batches)
# Test to see normalisation has occurred properly
print(imgs[1][8])
# Define method to plot MRIs
def plotImages(images_arr):
fig, axes = plt.subplots(1, 10, figsize=(20,20))
axes = axes.flatten()
for img, ax in zip( images_arr, axes):
ax.imshow(img)
ax.axis('off')
plt.tight_layout()
plt.show()
# Plot a sample of MRIs
plotImages(imgs)
# # Define the model
# # VGG16
# model = Sequential()
# model.add(Conv2D(input_shape=(160,160,3),filters=64,kernel_size=(3,3),padding="same", activation="relu"))
# model.add(Conv2D(filters=64,kernel_size=(3,3),padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Flatten())
# model.add(Dense(units=1024,activation="relu"))
# model.add(Dense(units=128,activation="relu"))
# model.add(Dense(units=2, activation="softmax"))
# # Model from the paper
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(160,160,3)),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
## Model from Dr Paul
# static_conv_layer=Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same')
#
# model = Sequential([
# Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same', input_shape=(32,32,3)),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# static_conv_layer,
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
# This model hits around 75% train acc, 54% val acc
model = Sequential([
Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same', input_shape=(80,80,3)),
MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
Flatten(),
Dense(units=2, activation='softmax')
])
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(160,160,3)),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
## Basic model with dropouts
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(224,224,3)),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.2),
# Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.3),
# Flatten(),
# Dense(units=1, activation='sigmoid')
# ])
# Summarise each layer of the model
print(model.summary())
# Compile and train the model
model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])
model.fit(x=train_batches,
steps_per_epoch=len(train_batches),
validation_data=valid_batches,
validation_steps=len(valid_batches),
epochs=20,
verbose=1
)
EDIT:
This paper seems to be doing much better than me and completing a very similar task, it may be useful to look at the methodology for:
Things you can try.
It's a nice way to start with transfer learning. Using image net weights helps you to train just last layers and give much better accuracy.
Adding early stopping and learning rate reduction with validation accuracy as constraint.
Taking advantage of ImageDataGenerator and add much more data augmentation techniques.
Make your model much deeper and also try different optimizer(RMSprop), run for more epochs with early stopping.
Add callbacks and plot training validation accuracy graphs with respective to learning rate to see which lr proves best for the data.
It looks like your model is overfitting due to a lack of data. You can do some data augmentation to increase how many images you have. If you don't care about your aspect ratio you can warp the images, if you don't always need the full image you can crop it and you can rotate it if orientation is not important. These things can dramatically increase your dataset size and help mitigate overfitting.
Here is an example from the tensorflow documentation:
batch_size = 32
AUTOTUNE = tf.data.experimental.AUTOTUNE
def prepare(ds, shuffle=False, augment=False):
# Resize and rescale all datasets
ds = ds.map(lambda x, y: (resize_and_rescale(x), y),
num_parallel_calls=AUTOTUNE)
if shuffle:
ds = ds.shuffle(1000)
# Batch all datasets
ds = ds.batch(batch_size)
# Use data augmentation only on the training set
if augment:
ds = ds.map(lambda x, y: (data_augmentation(x, training=True), y),
num_parallel_calls=AUTOTUNE)
# Use buffered prefecting on all datasets
return ds.prefetch(buffer_size=AUTOTUNE)
Also, here is a great video to watch from the TensorFlow developers youtube channel which explains the idea of image augmentation and shows an example of how to implement it.
I have to identify a retinal disease with CNN. I have 1400 images, 700 each class. My classes are (0 - no PDR) and (1 - PDR). I'm trying to make a model to identify if an input retina have the disease in level 4 or not.
I'm making a follow manipulation with my images and rezise all to 256x256:
ImageCV[index] = cv2.addWeighted(ImageCV[index],4, cv2.GaussianBlur(ImageCV[index],(0,0), 256/30), -4, 128)
And it made the follow with my imgs:
https://imgur.com/X1p9G1c
Then, when I train my model I got a very high accuracy (like 99....) but when I try to predict some test images, it fails.. for example, I've putted 10 PDR examples in test folder and tries to predict them (all must be 1).. this is the result:
[[0.]]
[[0.]]
[[1.]]
[[0.]]
[[0.]]
[[0.]]
[[1.]]
[[0.]]
[[0.]]
[[0.]]
This is my model:
visible = Input(shape=(256,256,3))
conv1 = Conv2D(16, kernel_size=(3,3), activation='relu', strides=(1, 1))(visible)
conv2 = Conv2D(16, kernel_size=(3,3), activation='relu', strides=(1, 1))(conv1)
bat1 = BatchNormalization()(conv2)
conv3 = ZeroPadding2D(padding=(1, 1))(bat1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(32, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(pool1)
conv5 = Conv2D(32, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(conv4)
bat2 = BatchNormalization()(conv5)
pool2 = MaxPooling2D(pool_size=(1, 1))(bat2)
conv6 = Conv2D(64, kernel_size=(3,3), activation='relu',strides=(1, 1), padding='valid')(pool2)
conv7 = Conv2D(64, kernel_size=(3,3), activation='relu',strides=(1, 1), padding='valid')(conv6)
bat3 = BatchNormalization()(conv7)
conv7 = ZeroPadding2D(padding=(1, 1))(bat3)
pool3 = MaxPooling2D(pool_size=(1, 1))(conv7)
conv8 = Conv2D(128, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(pool3)
conv9 = Conv2D(128, kernel_size=(2,2), activation='relu', strides=(1, 1), padding='valid')(conv8)
bat4 = BatchNormalization()(conv9)
pool4 = MaxPooling2D(pool_size=(1, 1))(bat4)
flat = Flatten()(pool4)
output = Dense(1, activation='sigmoid')(flat)
model = Model(inputs=visible, outputs=output)
opt = optimizers.adam(lr=0.001, decay=0.0)
model.compile(optimizer= opt, loss='binary_crossentropy', metrics=['accuracy'])
data, labels = ReadImages(TRAIN_DIR)
test, lt = ReadImages(TEST_DIR)
data = np.array(data)
labels = np.array(labels)
test = np.array(test)
lt = np.array(lt)
np.random.permutation(len(data))
np.random.permutation(len(labels))
np.random.permutation(len(test))
np.random.permutation(len(lt))
model.fit(data, labels, epochs=7, validation_data = (test,lt))
model.save('model.h5')
And this is predict.py
model = load_model('model.h5')
for filename in os.listdir(r'v/'):
if filename.endswith(".jpg") or filename.endswith(".ppm") or filename.endswith(".jpeg"):
ImageCV = cv2.resize(cv2.imread(os.path.join(TEST_DIR) + filename), (256,256))
ImageCV = cv2.addWeighted(ImageCV,4, cv2.GaussianBlur(ImageCV,(0,0), 256/30), -4, 128)
cv2.imshow('image', ImageCV)
cv2.waitKey(0)
cv2.destroyAllWindows()
ImageCV = ImageCV.reshape(-1,256,256,3)
print(model.predict(ImageCV))
What do I could to do to improve my predictions at all?
I strongly appreciate your help
UPDATE
Well, I tried to do all of were said in answers but still'nt working...
this is my code now:
visible = Input(shape=(256,256,3))
conv1 = Conv2D(16, kernel_size=(3,3), activation='relu', strides=(1, 1))(visible)
conv2 = Conv2D(32, kernel_size=(3,3), activation='relu', strides=(1, 1))(conv1)
bat1 = BatchNormalization()(conv2)
conv3 = ZeroPadding2D(padding=(1, 1))(bat1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv3)
drop1 = Dropout(0.30)(pool1)
conv4 = Conv2D(32, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(drop1)
conv5 = Conv2D(64, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(conv4)
bat2 = BatchNormalization()(conv5)
pool2 = MaxPooling2D(pool_size=(1, 1))(bat2)
drop1 = Dropout(0.30)(pool2)
conv6 = Conv2D(128, kernel_size=(3,3), activation='relu', padding='valid', kernel_regularizer=regularizers.l2(0.01))(pool2)
conv7 = Conv2D(128, kernel_size=(2,2), activation='relu', strides=(1, 1), padding='valid')(conv6)
bat3 = BatchNormalization()(conv7)
pool3 = MaxPooling2D(pool_size=(1, 1))(bat3)
drop1 = Dropout(0.30)(pool3)
flat = Flatten()(pool3)
drop4 = Dropout(0.50)(flat)
output = Dense(1, activation='sigmoid')(drop4)
model = Model(inputs=visible, outputs=output)
opt = optimizers.adam(lr=0.001, decay=0.0)
model.compile(optimizer= opt, loss='binary_crossentropy', metrics=['accuracy'])
data, labels = ReadImages(TRAIN_DIR)
test, lt = ReadImages(TEST_DIR)
data = np.array(data)
labels = np.array(labels)
perm = np.random.permutation(len(data))
data = data[perm]
labels = labels[perm]
#model.fit(data, labels, epochs=8, validation_data = (np.array(test), np.array(lt)))
aug = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15,
horizontal_flip=True)
# train the network
model.fit_generator(aug.flow(data, labels, batch_size=32),
validation_data=(np.array(test), np.array(lt)), steps_per_epoch=len(data) // 32,
epochs=7)
And this is the return:
Epoch 1/7
43/43 [==============================] - 1004s 23s/step - loss: 1.8090 - acc: 0.9724 - val_loss: 1.7871 - val_acc: 0.9861
Epoch 2/7
43/43 [==============================] - 1003s 23s/step - loss: 1.8449 - acc: 0.9801 - val_loss: 1.4828 - val_acc: 1.0000
Epoch 3/7
43/43 [==============================] - 1092s 25s/step - loss: 1.5704 - acc: 0.9920 - val_loss: 1.3985 - val_acc: 1.0000
Epoch 4/7
43/43 [==============================] - 1062s 25s/step - loss: 1.5219 - acc: 0.9898 - val_loss: 1.3167 - val_acc: 1.0000
Epoch 5/7
43/43 [==============================] - 990s 23s/step - loss: 2.5744 - acc: 0.9222 - val_loss: 2.9347 - val_acc: 0.9028
Epoch 6/7
43/43 [==============================] - 983s 23s/step - loss: 1.6053 - acc: 0.9840 - val_loss: 1.3299 - val_acc: 1.0000
Epoch 7/7
43/43 [==============================] - 974s 23s/step - loss: 1.6180 - acc: 0.9801 - val_loss: 1.5181 - val_acc: 0.9861
I'd have added dropouts, reduce the model layers, put data augmentation, and doesn't work at all (all the predictions returns 0)...
Please anyone can help in it.
It seems you having issues with overfitting. I have kind of a personal dilemma in here whether this is off-topic or not because the approach one can give it's somewhat opinion-based, but here I go:
First off, if you need to regularize an overfitting network, you want to use dropout starting at 0.25 and checking if that improves the model. Data augmentation is a must when dealing with overfitting, together with batch normalization (which you are applying).
If this still not solves your overfitting problems, then you should try working on your network architecture to tweak it in order to make it generalize better. Did you check a simple of the inputs being used for train and test?
TLDR: Try dropout and data augmentation, if it doesn't work and your data is correct, you might have to work on improving the architecture to make a better generalizing model.
EDIT: The consensus on approaching this kind of models is to have it overfit at first, with a decent accuracy and then work towards generalizing it without losing accuracy if possible.
Your model is very big for your training set size, leading to overfitting. Use less layers
You're overfitting i.e. have a high variance problem. Try adding some soft dropout (0.2-0.3) at the end of your convolution blocks. You can also add a couple of Dense layers with decreasing unit numbers before the output layer and put more dropout layers in between those (0.5+).
You should also implement more data augmentation such as rotations, flips, random noise, random brightness, etc. Check out Keras' documentation for the ImageDataGenerator class.
I'm running Keras with a tensorflow-backend.
I try to predict images.
My model looks like this:
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(50, 50, 1)))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(1, activation='softmax'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
Running my code, it produces this output over 10 epochs:
Epoch 1/10
24946/24946 [==============================] - 36s 1ms/sample - loss: 7.9693 - acc: 0.5001
Epoch 2/10
24946/24946 [==============================] - 35s 1ms/sample - loss: 7.9693 - acc: 0.5001
...
Epoch 9/10
24946/24946 [==============================] - 30s 1ms/sample - loss: 7.9693 - acc: 0.5001
Epoch 10/10
24946/24946 [==============================] - 30s 1ms/sample - loss: 7.9693 - acc: 0.5001
1/1 [==============================] - 0s 36ms/step
[[1.]]
Anyhow, I do not understand why the accuracy is always 0.5001 over all 10 epochs.
My question is: Why does the accuracy not change within any epoch?
This part of your code makes no sense:
model.add(Dense(1, activation='softmax'))
Softmax with only one neuron with always produce output of constant 1.0, due to the normalization. If you want to do binary classification, you should use a sigmoid activation at the output.
I have created the following model with Keras. The dataset is MNIST.
'''
conv - relu - conv- relu - pool -
conv - relu - conv- relu - pool -
conv - relu - conv- relu - pool -
affine - relu - dropout - affine - dropout - softmax
'''
model = Sequential()
model.add(Conv2D(16, kernel_size=(3, 3),
padding='same',
input_shape=input_shape))
model.add(Activation('relu'))
model.add(Conv2D(16, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(50, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Dropout(0.5))
model.add(Activation('softmax'))
The following is the result:
60000/60000 [==============================] - 10s - loss: 1.2707 - acc: 0.5059 - val_loss: 0.0881 - val_acc: 0.9785
Epoch 2/20
60000/60000 [==============================] - 9s - loss: 0.9694 - acc: 0.5787 - val_loss: 0.0449 - val_acc: 0.9873
...
Epoch 19/20
60000/60000 [==============================] - 9s - loss: 0.8530 - acc: 0.6004 - val_loss: 0.0282 - val_acc: 0.9937
Epoch 20/20
60000/60000 [==============================] - 9s - loss: 0.8564 - acc: 0.5982 - val_loss: 0.0383 - val_acc: 0.9910
Test loss: 0.0382921607383
Test accuracy: 0.991
Why is the training accuracy so low, while the validation accururacy is so high?
The dropout on your last Dense layer removes half of your 10 neurons for your classes by random. Your last layer can only by accurate half of the times because in general half of the neurons are missing.
Try to remove that and I assume you get even values.