Extremely High Loss with Keras VGG16 transfer learning Image Classification - python

I'm retraining VGG16 and fine tuning the top 2 convolutional blocks for an image classification task. The retraining itself finished rather uneventfully with mediocre accuracy. The fine tuning program is currently running and it seems like it will take a couple of days at the least. However, the loss is ludicrously high at around 440 after 3 epochs, while accuracy hovers around 0.4. Please confirm whether there is any major flaw in the model so that I can kill the program. I have used some data augmentation and l1_l2 regularisation during the fine tuning stage. The data set is a small set of about 8000 images of tumors, divided into 8 classes. Thus, the data is both small and extremely unrelated to the imagenet images. This is my code for the transfer learning:
import numpy as np
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Dense
from keras import applications
from keras.utils.np_utils import to_categorical
import math
img_width, img_height= 224, 224
weight_path= 'bottleneck_fc_model.h5'
train_dir= 'Cancer_Data/Train'
validation_dir= 'Cancer_Data/Validate'
epochs= 150
batch_size= 128
def save_bottleneck_features():
datagen = ImageDataGenerator(rescale= 1./255)
model = applications.VGG16(include_top=False, weights='imagenet')
print 'Extracting Bottleneck Training Features'
generator = datagen.flow_from_directory(
train_dir,
target_size= (img_width, img_height),
batch_size= batch_size,
class_mode= None,
shuffle= False)
nb_train_samples= len(generator.filenames)
predict_size_train= int(math.ceil(nb_train_samples / batch_size))
bottleneck_feature_train =model.predict_generator(generator,predict_size_train)
np.save(open('bottleneck_feature_train.npy', 'w'), bottleneck_feature_train)
print "Bottleneck Training Features Saved"
print "Extracting Bottleneck Validation Features"
generator2 = datagen.flow_from_directory(
validation_dir,
target_size= (img_width, img_height),
batch_size= batch_size,
class_mode= None,
shuffle= False)
nb_validation_samples= len(generator2.filenames)
predict_size_validation= int(math.ceil(nb_validation_samples / batch_size))
bottleneck_feature_validation = model.predict_generator(generator2, predict_size_validation)
np.save(open('bottleneck_feature_validation.npy', 'w'), bottleneck_feature_validation)
print "Bottleneck Validation Features Saved"
def train_top_model():
datagen_top = ImageDataGenerator(rescale=1./255)
generator_top = datagen_top.flow_from_directory(
train_dir,
target_size= (img_width,img_height),
batch_size=batch_size,
class_mode='categorical',
shuffle=False)
nb_classes = len(generator_top.class_indices)
np.save('class_indices.npy', generator_top.class_indices)
train_data = np.load('bottleneck_feature_train.npy')
train_labels= to_categorical(generator_top.classes, num_classes= nb_classes)
generator_top2 = datagen_top.flow_from_directory(
validation_dir,
target_size=(img_width,img_height),
batch_size=batch_size,
class_mode=None,
shuffle=False)
validation_data = np.load('bottleneck_feature_validation.npy')
validation_labels= to_categorical(generator_top2.classes, num_classes= nb_classes)
model = Sequential()
model.add(Flatten(input_shape=train_data.shape[1:]))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation= 'softmax'))
model.compile(optimizer= 'adam', loss= 'categorical_crossentropy', metrics= ['accuracy'])
model.save_weights(weight_path)
model.fit(train_data, train_labels, epochs= epochs, batch_size= batch_size,
validation_data= (validation_data, validation_labels))
(eval_loss, eval_accuracy) = model.evaluate(
validation_data, validation_labels, batch_size=batch_size, verbose=1)
print("[INFO] accuracy: {:.2f}%".format(eval_accuracy * 100))
print("[INFO] Loss: {}".format(eval_loss))
save_bottleneck_features()
train_top_model()
The code for fine tuning the model is:
from keras import applications
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.regularizers import l1_l2
from keras.models import Sequential, Model
from keras.layers import Dropout, Flatten, Dense
import PIL
import math
weight_path = 'fine_tuned.h5'
top_model_weight_path = 'top_model.h5'
img_width, img_height = 224, 224
train_dir = 'Cancer_Data/Train'
validation_dir = 'Cancer_Data/Validate'
epochs = 200
batch_size = 128
nb_train_samples = 6454
nb_validation_samples = 1464
base_model =applications.VGG16(weights= 'imagenet', include_top= False, input_shape=(224,224,3))
print "Model Loaded."
top_model= Sequential()
top_model.add(Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(Dense(256, activation='relu', kernel_regularizer= l1_l2(l1=0.01, l2= 0.01)))
top_model.add(Dense(8, activation= 'softmax'))
top_model.load_weights(top_model_weight_path)
model= Model(inputs= base_model.input, outputs= top_model(base_model.output))
for layer in model.layers[:18]:
layer.trainable=False
model.compile(optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
loss='categorical_crossentropy', metrics=['accuracy'])
train_datagen= ImageDataGenerator(
rescale=1./255,
shear_range=0.3,
zoom_range=0.3,
horizontal_flip=True)
test_datagen= ImageDataGenerator(rescale=1./255)
train_generator= train_datagen.flow_from_directory(
train_dir,
target_size=(img_height,img_width),
batch_size=batch_size,
class_mode='categorical')
validation_generator= test_datagen.flow_from_directory(
validation_dir,
target_size=(img_height,img_width),
batch_size=batch_size,
class_mode='categorical')
model.save(weight_path)
model.fit_generator(
train_generator,
steps_per_epoch = int(math.ceil(nb_train_samples / batch_size)),
epochs=epochs,
validation_data=validation_generator,
validation_steps = int(math.ceil(nb_validation_samples / batch_size)))
I'm fairly new to python and Keras so I haven't been able to figure out whether I have done something wrong or whether there is some structural issue with the model itself. What changes can I make and what techniques can I use to reduce the loss if it is the latter?

VGG 16 was not trained on data in range [0,1] rather in range [-128,128] (Approximately) obtained by subtracting corresponding mean pixels from respective color channels. Also, VGG 16 requires data in BGR format.
Use preprocessing as below (taken from keras source code)
def preprocess:
if data_format == 'channels_first':
if x.ndim == 3:
# 'RGB'->'BGR'
x = x[::-1, ...]
# Zero-center by mean pixel
x[0, :, :] -= 103.939
x[1, :, :] -= 116.779
x[2, :, :] -= 123.68
else:
x = x[:, ::-1, ...]
x[:, 0, :, :] -= 103.939
x[:, 1, :, :] -= 116.779
x[:, 2, :, :] -= 123.68
else:
# 'RGB'->'BGR'
x = x[..., ::-1]
# Zero-center by mean pixel
x[..., 0] -= 103.939
x[..., 1] -= 116.779
x[..., 2] -= 123.68
return x

Related

Accuracy is zero for cifar10 dataset with Keras Sequential Model

My accuracy is zero for all the 15 epochs in spite of using multiple Conv2D and Max Pooling Layers. I am using ImageDataGenerator for Data Augmentation.
Complete code is given below:
# importing all the required libraries
import tensorflow as tf
from tensorflow.keras.layers import Dense, Conv2D, Flatten, MaxPool2D, Dropout
from tensorflow.keras.models import Sequential
from tensorflow.keras.datasets import cifar10
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
# Loading the Data from the in built library
(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()
# Normalize the Pixel Data
train_images = train_images/255.0
test_images = test_images/255.0
# Instantiate the Image Data Generator Class with the Data Augmentation
datagen = ImageDataGenerator(width_shift_range = 0.2, height_shift_range = 0.2,
rotation_range = 20, horizontal_flip = True,
vertical_flip = True, validation_split = 0.2)
# Apply the Data Augmentation to the Training Images
datagen.fit(train_images)
# Create the Generator for the Training Images
train_gen = datagen.flow(train_images, train_labels, batch_size = 32,
subset = 'training')
# Create the Generator for the Validation Images
val_gen = datagen.flow(train_images, train_labels, batch_size = 8,
subset = 'validation')
num_classes = 10
# One Hot Encoding of Labels using to_categorical
train_labels = to_categorical(train_labels, num_classes)
test_labels = to_categorical(test_labels, num_classes)
img_height = 32
img_width = 32
# Building the Keras Model
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))
model.add(MaxPool2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPool2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
#model.add(Dropout(rate = 0.2))
model.add(Dense(units = num_classes, activation = 'softmax'))
model.summary()
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam',
metrics = ['accuracy'])
steps_per_epoch = len(train_images) * 0.8//32
history = model.fit(train_gen, validation_data = val_gen,
steps_per_epoch = steps_per_epoch, epochs = 15)
Your problem is you ran this code
train_gen = datagen.flow(train_images, train_labels, batch_size = 32,
subset = 'training')
# Create the Generator for the Validation Images
val_gen = datagen.flow(train_images, train_labels, batch_size = 8,
subset = 'validation')
but only after this did you convert the labels to categorical. So take the code
num_classes = 10
# One Hot Encoding of Labels using to_categorical
train_labels = to_categorical(train_labels, num_classes)
test_labels = to_categorical(test_labels, num_classes)
and place it PRIOR to the train_gen and val_gen code. On a finer point you have the code
datagen.fit(train_images)
You only need to fit the generator if you have any of the parameters
featurewise_center, samplewise_center, featurewise_std_normalization, or
samplewise_std_normalization set to true.
Transform your label to one hot right before the .flow.
...
# One Hot Encoding of Labels using to_categorical
train_labels = to_categorical(train_labels, num_classes)
test_labels = to_categorical(test_labels, num_classes)
# Create the Generator for the Training Images
train_gen = datagen.flow(train_images, train_labels, batch_size = 32,
subset = 'training')
# Create the Generator for the Validation Images
val_gen = datagen.flow(train_images, train_labels, batch_size = 8,
subset = 'validation')
...

Problem with logits and labels size. Tensorflow

I try to train top layer separate from base model. All is working with generating features with model.predict_generator like
bottleneck_features_train = model.predict_generator(
train_generator, predict_size_train)
np.save(save_dir + 'bottleneck_features_train.npy', bottleneck_features_train)
train_data = np.load(mtx.save_dir + 'bottleneck_features_train.npy')
model.fit(train_data, ....
)
But now I got huge dataset and can't load all data in memory so I use generator flow_from_directory
def create_generator(root_path, batch_size):
datagen = ImageDataGenerator(rescale=1. / 255)
generator = datagen.flow_from_directory(
root_path,
target_size=(224, 224),
batch_size=batch_size,
class_mode="categorical",
shuffle=True)
return generator
train_generator = create_generator(mtx.train_data_dir, mtx.batch_size)
and than
model.fit(train_generator...
class_mode in flow_from_directory is "categorical" and loss function too(categorical_crossentropy)
layers is
model = Sequential()
model.add(Flatten(input_shape=(7, 7, 512)))
model.add(Dense(512, activation="relu"))
model.add(Dropout(0.7))
model.add(Dense(num_classes, activation='softmax'))
but when I run training I get
logits and labels must be broadcastable: logits_size=[24,32] labels_size=[4,32]
As I understand it's something wrong with shapes in layers or how are features/labels encoded.
Update 1:
Also it's working when batch_size in flow_from_directory is set with 1. But accuracy is very low than.
try
model.add(Flatten(input_shape=(224,224,3)))

Wrong prediction on images

In short
I'm going my first steps with Keras using my dataset.
When I try to load a model and test on test-images - I obtain only one class despite the previous output on training states the accuracy was 77%. Could you point out, what is misconfigured in the prediction code?
Detailed description:
The dataset are marked biological cells in gray-scale, size 64x64. There are 2 types of cells p and c.
I created the dataset directories as proposed in cat/dogs example.
train, test, val irectories with panca, canca in each. canca, canca directories contain images.
e.g.:
train-panca-<images>
train-canca-<images>
I've derived the code mainly also from cat/dog example and changed the image size.
After running the code, I obtain the output:
...
80/80 [==============================] - 8s 101ms/step - loss: 0.2995 - acc: 0.8910 - val_loss: 0.5150 - val_acc: 0.7560
Using TensorFlow backend.
Finished saving
Test loss: 0.46428223699331284
Test accuracy: 0.7759999990463257
This looks reasonable and promising - Accuracy of 77% with very litle data and short training time (ca. 3 minutes) is incredible.
I also tested replacing the test directory by train directory and obtain the accuracy of > 90%. So the model looks functional.
However, when I try to load it and test on test-images - I obtain only one class.
train.py
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
# dimensions of the images.
img_width, img_height = 64, 64
train_dir = 'c:/tmp/anca_pc/train'
val_dir = 'c:/tmp/anca_pc/val'
test_dir = 'c:/tmp/anca_pc/test'
nb_train_samples = 2000
nb_validation_samples = 500
nb_test_samples = 500
epochs = 5
batch_size = 25
#Is right got gray-scale images? Shouldn't be 1 instead of 3
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
#Define model as proposed in keras tutorials
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
datagen = ImageDataGenerator( rescale=1. / 255 )
train_generator = datagen.flow_from_directory(
train_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
validation_generator = datagen.flow_from_directory(
val_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
test_generator = datagen.flow_from_directory(
test_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size)
#Save model architecture
model_json = model.to_json()
json_file = open("anca_pc.json", "w")
json_file.write(model_json)
json_file.close()
#Save model weights
model.save_weights("anca_pc.h5")
print("Finished saving")
score = model.evaluate_generator(test_generator, nb_test_samples // batch_size)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
print('Scode', score)
predict.py
import numpy as np
from keras.models import model_from_json
from keras.preprocessing import image
from keras.applications.vgg16 import preprocess_input
import matplotlib.pyplot as plt
from scipy.misc import toimage
import os
classes = ['panca', 'canca']
directory = 'C:/tmp/anca_pc/test/'
json_file = open("anca_pc.json", "r")
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
loaded_model.load_weights("anca_pc.h5")
loaded_model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
for c in classes:
aktdir=directory+c
print(aktdir)
for filename in os.listdir(aktdir):
fn=os.path.join(aktdir, filename)
#print(fn)
img = image.load_img(fn, target_size=(64, 64))
#plt.imshow(img)
#plt.show()
x = image.img_to_array(img)
x = x.astype('float32')
x /= 255
x = np.expand_dims(x, axis=0)
#x = preprocess_input(x)
prediction = loaded_model.predict(x)
#print(prediction)
print(c + " : " + classes[np.argmax(prediction)], " : ", prediction)
EDIT
Score output of model.predict_generator(test_generator, nb_test_samples // batch_size) run with 3 epochs
Test loss: 0.5996998563408852
Test accuracy: 0.7060000032186509
Score [[0.72468185]
[0.07171335]
[0.06702321]
[0.04176971]
[0.76247555]
[0.07743845]
[0.07435916]
[0.9960306 ]
[0.9270018 ]
[0.04746262]
[0.05305129]
[0.9993339 ]
[0.9986149 ]
[0.63527316]
[0.08033804]
[0.3816172 ]
[0.97601706]
[0.83666223]
[0.7226989 ]
[0.5351326 ]
[0.8407803 ]
[0.6953097 ]
[0.89651984]
[0.44985726]
[0.30889446]
[0.16406931]
[0.6346773 ]
[0.13678996]
[0.51343983]
[0.97438985]
[0.9981396 ]
[0.5485193 ]
[0.05270131]
[0.8029713 ]
[0.3295382 ]
[0.1865853 ]
[0.94497275]
[0.07609159]
[0.67434824]
[0.18562992]
[0.53442085]
[0.06662691]
[0.0388172 ]
[0.8763066 ]
[0.9875164 ]
...
np.argmax(prediction) will always return 0, because prediction contains only one value.
Since you have a binary output, you need to replace np.argmax with something like this:
def get_class(prediction):
return 1 if prediction > 0.5 else 0

Keras Val_acc is good but prediction for same data is poor

I am using Keras for a CNN two class classification. While training my val_acc is above 95 percent. But when I predict result for the same validation data the acc is less than 60 percent, is that even possible? This is my Code:
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
from keras.callbacks import TensorBoard
from keras.preprocessing import image
import matplotlib.pyplot as plt
import numpy as np
np.random.seed(1337) # for reproducibility
%matplotlib inline
img_width, img_height = 230,170
train_data_dir = 'data/Train'
validation_data_dir = 'data/Validation'
nb_train_samples = 13044
nb_validation_samples = 200
epochs =14
batch_size = 32
if K.image_data_format() == 'channels_first':
input_shape = (1, img_width, img_height)
else:
input_shape = (img_width, img_height, 1)
model = Sequential()
model.add(Convolution2D(32, (3, 3),data_format='channels_first' , input_shape=(1,230,170)))
convout1 = Activation('relu')
model.add(convout1)
convout2 = MaxPooling2D(pool_size=(2,2 ), strides= None , padding='valid', data_format='channels_first')
model.add(convout2)
model.add(Convolution2D(32, (3, 3),data_format='channels_first'))
convout3 = Activation('relu')
model.add(convout3)
model.add(MaxPooling2D(pool_size=(2, 2), data_format='channels_first'))
model.add(Convolution2D(64, (3, 3),data_format='channels_first'))
convout4 = Activation('relu')
model.add(convout4)
convout5 = MaxPooling2D(pool_size=(2, 2), data_format='channels_first')
model.add(convout5)
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
train_datagen = ImageDataGenerator(rescale=1. / 255,
shear_range=0,
zoom_range=0.2,
horizontal_flip=False,
data_format='channels_first')
test_datagen = ImageDataGenerator(rescale=1. / 255,
data_format='channels_first')
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary',
color_mode= "grayscale",
shuffle=True
)
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary',
color_mode= "grayscale",
shuffle=True
)
model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
shuffle=True
)
Epoch 37/37
407/407[==============] - 1775s 4s/step - loss: 0.12 - acc: 0.96 - val_loss: 0.02 - val_acc: 0.99
#Prediction:
test_data_dir='data/test'
validgen = ImageDataGenerator(horizontal_flip=False, data_format='channels_first')
test_gen = validgen.flow_from_directory(
test_data_dir,
target_size=(img_width, img_height),
batch_size=1,
class_mode='binary',
shuffle=False,
color_mode= "grayscale")
preds = model.predict_generator(test_gen)
In the below output about 7 images belong to class 0. I tried the same for all 100 images of the class 0 validation data and only 15 images were predicted as class 0 and remaining was predicted as class 1
Found 10 images belonging to 1 classes.
[[ 1.]
[ 1.]
[ 1.]
[ 1.]
[ 1.]
[ 1.]
[ 1.]
[ 0.]
[ 0.]
[ 1.]]
You are not scaling your test images by 1./255 as you have in your training and validation images. Ideally, the statistics of your test data should be similar to the training data.
So, I have decided to post the answer I had posted in Quora but with the essential part as advised. I too had a similar problem as this one and I hope my answer can help someone else as well. I decided to research on the Internet and came across this answer by cjbayron.
What helped me solve a similar issue was that I had the following in my code for training the model:
import keras
import os
from keras import backend as K
import tensorflow as tf
import random as rn
import numpy as np
os.environ['PYTHONHASHSEED'] = '0'
np.random.seed(70)
rn.seed(70)
tf.set_random_seed(70)
/******* code for my model ******/
#very important here to save session after completing model.fit
model.fit_generator(train_batches, steps_per_epoch=4900, validation_data=valid_batches,validation_steps=1225, epochs=40, verbose=2, callbacks=callbacks_list)
saver = tf.train.Saver()
sess = keras.backend.get_session()
saver.save(sess, 'gdrive/My Drive/KerasCNN/model/keras_session/session.ckpt')
the saved session will generate the following files as well:
/keras_session/checkpoint
/keras_session/session.ckpt.data-00000-of-00001
/keras_session/session.ckpt.index
/keras_session/session.ckpt.meta
I downloaded all these files from my Google Drive as well and placed them in a local directory. You might notice that there appears to be no file named session.ckpt only but is being used in saver.restore(). This is okay. Tensorflow kinda works it out. It will not bring an error.
During model.load_model()
So in my Pycharm, I loaded the model as follows:
model=load_model('C:\\Users\\Username\\PycharmProjects\\MyProject\\mymodel\\mymodel.h5')
saver = tf.train.Saver()
sess = keras.backend.get_session()
saver.restore(sess,'C:\\Users\\Username\\PycharmProjects\\MyProject\\mymodel\\keras_session\\session.ckpt')
/***** then predict the images as you wish ******/
pred = model.predict_classes(load_image(os.path.join(test_path, file)))
It is important to place the restore code as shown i.e. after loading the model.
Once I did this, I tried predicting same images I used for training and validation and this time round, the model wrongly predicted around 2 images per class. Now I was sure that my model was okay and I went ahead to predict with my test images i.e. images it had not seen before and it performed very well.

VGG16 Keras fine tuning: low accuracy

I've already asked similar question here, but now I have slightly different problem, therefore asking new question.
I decided to use slightly different approach instead of proposed among answers in the referenced question to train, and then fine-tune model.
Update: I've replaced old question provided here with more suitable version
Here is my sequence of actions:
Build VGG16 model and drop top layer (call it no-top model)
Generate bottleneck features using no-top model
Train a separate fully-connected model using bottleneck features
Build new VGG16 model, drop top layers, and attach pretrained top-model
Train concatenated model on dogs/cats data
And here is a code I use to implement aforementioned sequence of actions:
import warnings
warnings.simplefilter('ignore', UserWarning)
warnings.simplefilter('ignore', DeprecationWarning)
from __future__ import print_function
from itertools import izip_longest as zip_longest
from pprint import pformat as pf
from pprint import pprint as pp
import os
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.layers import Conv2D, MaxPooling2D, MaxPooling2D, ZeroPadding2D
from keras.layers import Dropout, Flatten, Dense, InputLayer, Lambda
from keras.models import Sequential, Model, load_model
from keras.utils.data_utils import get_file
from keras.optimizers import SGD
import keras.backend as K
import numpy as np
RANDOM_STATE = 1
IMAGE_WIDTH = 224
IMAGE_HEIGHT = 224
BATCH_SIZE = 4
VGG_MEAN = np.array([123.68, 116.779, 103.939]).reshape((3, 1, 1))
VGG16_WEIGHTS_PATH = 'http://www.platform.ai/models/vgg16.h5'
DATA_ROOT = os.path.join(os.path.expanduser('~'), 'data', 'dogscats')
TRAIN_DIR = os.path.join(DATA_ROOT, 'train')
VALID_DIR = os.path.join(DATA_ROOT, 'valid')
SAMPLES_DIR = os.path.expanduser('~/dogscats_samples')
np.random.seed(RANDOM_STATE)
K.set_image_dim_ordering('th')
def get_batches(dirname, gen=ImageDataGenerator(), shuffle=True,
batch_size=BATCH_SIZE, class_mode='categorical'):
return gen.flow_from_directory(
os.path.join(SAMPLES_DIR, dirname),
target_size=(IMAGE_WIDTH, IMAGE_HEIGHT),
class_mode=class_mode,
shuffle=shuffle,
batch_size=batch_size)
def vgg_preprocess(x):
x = x - VGG_MEAN
return x[:, ::-1]
def conv_block(model, n_layers, n_filters, name='block'):
for i in range(n_layers):
model.add(ZeroPadding2D((1, 1), name='%s_padding_%s' % (name, i)))
model.add(Conv2D(n_filters, (3, 3), activation='relu', name='%s_conv2d_%s' % (name, i)))
model.add(MaxPooling2D((2, 2), strides=(2, 2), name='%s_maxpool' % name))
def fc_block(model, name='block'):
model.add(Dense(4096, activation='relu', name=name + '_dense'))
model.add(Dropout(0.5))
def build_vgg_16():
model = Sequential()
input_shape = (3, IMAGE_WIDTH, IMAGE_HEIGHT)
model.add(InputLayer(input_shape=input_shape))
model.add(Lambda(vgg_preprocess))
conv_block(model, n_layers=2, n_filters=64, name='block1')
conv_block(model, n_layers=2, n_filters=128, name='block2')
conv_block(model, n_layers=3, n_filters=256, name='block3')
conv_block(model, n_layers=3, n_filters=512, name='block4')
conv_block(model, n_layers=3, n_filters=512, name='block5')
model.add(Flatten())
fc_block(model)
fc_block(model)
model.add(Dense(1000, activation='softmax'))
return model
def train_finetuned_model():
file_path = get_file('vgg16.h5', VGG16_WEIGHTS_PATH, cache_subdir='models')
print('Building VGG16 (no-top) model to generate bottleneck features')
vgg16_notop = build_vgg_16()
vgg16_notop.load_weights(file_path)
for _ in range(6):
vgg16_notop.pop()
vgg16_notop.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
train_batches = get_batches('train', shuffle=False, class_mode=None)
train_labels = np.array([0]*1000 + [1]*1000)
bottleneck_train = vgg16_notop.predict_generator(train_batches, steps=2000 // BATCH_SIZE)
valid_batches = get_batches('valid', shuffle=False, class_mode=None)
valid_labels = np.array([0]*400 + [1]*400)
bottleneck_valid = vgg16_notop.predict_generator(valid_batches, steps=800 // BATCH_SIZE)
print('Training top model on bottleneck features')
top_model = Sequential()
top_model.add(Flatten(input_shape=bottleneck_train.shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(1, activation='sigmoid'))
top_model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
top_model.fit(bottleneck_train, train_labels,
batch_size=32, epochs=50,
validation_data=(bottleneck_valid, valid_labels))
print('Concatenate new VGG16 (without top layer) with pretrained top model')
vgg16_fine = build_vgg_16()
vgg16_fine.load_weights(file_path)
for _ in range(6):
vgg16_fine.pop()
vgg16_fine.add(Flatten(name='top_flatten'))
vgg16_fine.add(Dense(256, activation='relu', name='top_dense'))
vgg16_fine.add(Dropout(0.5, name='top_dropout'))
vgg16_fine.add(Dense(1, activation='sigmoid', name='top_sigmoid'))
for i, layer in enumerate(reversed(top_model.layers), 1):
pretrained_weights = layer.get_weights()
vgg16_fine.layers[-i].set_weights(pretrained_weights)
for layer in vgg16_fine.layers[:26]:
layer.trainable = False
vgg16_fine.compile(optimizer=SGD(lr=1e-4, momentum=0.9),
loss='binary_crossentropy',
metrics=['accuracy'])
print('Train concatenated model on dogs/cats dataset sample')
train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1./255)
train_batches = get_batches('train', gen=train_datagen, class_mode='binary')
valid_batches = get_batches('valid', gen=test_datagen, class_mode='binary')
vgg16_fine.fit_generator(train_batches,
steps_per_epoch=2000 // BATCH_SIZE,
epochs=50,
validation_data=valid_batches,
validation_steps=800 // BATCH_SIZE)
return vgg16_fine
final_model = train_finetuned_model()
But the problem is that model's accuracy drastically dropped. After 50 epochs, its accuracy is around 50%. Therefore, probably I've done something wrong.
Maybe something wrong with parameters, i.e. learning rate, batch size, etc.?
Your fully connected layers look totally different from the original VGG architecture.
# yours
Flatten()
Dense(256, activation='relu')
Dense(1, activation='sigmoid')
# original
Flatten()
Dense(4096, activation='relu')
Dense(4096, activation='relu')
Dense(2, activation='softmax')
Two points.
The last layer should be 2-class-softmax instead of sigmoid. The
accuracy is not computed as you expect if you use sigmoid, I guess.
Complexity (number of neurons and layers) seems to be too low.
Well, not sure if it is a right solution, but I was able to increase accuracy at least up to 70% with this code (probably the main reason is decreased learning rate and more epochs):
def train_finetuned_model(lr=1e-5, verbose=True):
file_path = get_file('vgg16.h5', VGG16_WEIGHTS_PATH, cache_subdir='models')
if verbose:
print('Building VGG16 (no-top) model to generate bottleneck features.')
vgg16_notop = build_vgg_16()
vgg16_notop.load_weights(file_path)
for _ in range(6):
vgg16_notop.pop()
vgg16_notop.compile(optimizer=RMSprop(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
if verbose:
print('Bottleneck features generation.')
train_batches = get_batches('train', shuffle=False, class_mode=None, batch_size=BATCH_SIZE)
train_labels = np.array([0]*1000 + [1]*1000)
train_bottleneck = vgg16_notop.predict_generator(train_batches, steps=2000 // BATCH_SIZE)
valid_batches = get_batches('valid', shuffle=False, class_mode=None, batch_size=BATCH_SIZE)
valid_labels = np.array([0]*400 + [1]*400)
valid_bottleneck = vgg16_notop.predict_generator(valid_batches, steps=800 // BATCH_SIZE)
if verbose:
print('Training top model on bottleneck features.')
top_model = Sequential()
top_model.add(Flatten(input_shape=train_bottleneck.shape[1:]))
top_model.add(Dense(4096, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(4096, activation='relu'))
top_model.add(Dropout(0.5))
top_model.add(Dense(2, activation='softmax'))
top_model.compile(optimizer=RMSprop(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
top_model.fit(train_bottleneck, to_categorical(train_labels),
batch_size=32, epochs=10,
validation_data=(valid_bottleneck, to_categorical(valid_labels)))
if verbose:
print('Concatenate new VGG16 (without top layer) with pretrained top model.')
vgg16_fine = build_vgg_16()
vgg16_fine.load_weights(file_path)
for _ in range(6):
vgg16_fine.pop()
vgg16_fine.add(Flatten(name='top_flatten'))
vgg16_fine.add(Dense(4096, activation='relu'))
vgg16_fine.add(Dropout(0.5))
vgg16_fine.add(Dense(4096, activation='relu'))
vgg16_fine.add(Dropout(0.5))
vgg16_fine.add(Dense(2, activation='softmax'))
vgg16_fine.compile(optimizer=RMSprop(lr=lr), loss='categorical_crossentropy', metrics=['accuracy'])
if verbose:
print('Loading pre-trained weights into concatenated model')
for i, layer in enumerate(reversed(top_model.layers), 1):
pretrained_weights = layer.get_weights()
vgg16_fine.layers[-i].set_weights(pretrained_weights)
for layer in vgg16_fine.layers[:26]:
layer.trainable = False
if verbose:
print('Layers training status:')
for layer in vgg16_fine.layers:
print('[%6s] %s' % ('' if layer.trainable else 'FROZEN', layer.name))
vgg16_fine.compile(optimizer=RMSprop(lr=1e-6), loss='binary_crossentropy', metrics=['accuracy'])
if verbose:
print('Train concatenated model on dogs/cats dataset sample.')
train_datagen = ImageDataGenerator(rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1./255)
train_batches = get_batches('train', gen=train_datagen, class_mode='categorical', batch_size=BATCH_SIZE)
valid_batches = get_batches('valid', gen=test_datagen, class_mode='categorical', batch_size=BATCH_SIZE)
vgg16_fine.fit_generator(train_batches, epochs=100,
steps_per_epoch=2000 // BATCH_SIZE,
validation_data=valid_batches,
validation_steps=800 // BATCH_SIZE)
return vgg16_fine
I guess there is a way to achieve much better results with fine-tuning (up to 98%), but I wasn't able to achieve it with provided code.

Categories

Resources