Wrong Confusion Matrix Display - CCN Image Keras model - python

am new to Deep Learning and using CNNs. I am trying to predict the quality of coffee seeds using images where good quality is Coffee AA and the other is Poor Quality. I built the model tried to evaluate its performance by using a Confusion Matrix on a test data(test_ds) but it seems to give me a one column prediction instead of a diagonal form. I don't know where the issue is coming from whether its how i built the model or how am trying to build the confusion matrix. Kindly help. The rest of the code is here https://colab.research.google.com/drive/1cNgAbCy8e4lG-dCepVxIpFx1Wfeof5wb?usp=sharing
Here is part of the Code:
data = tf.keras.preprocessing.image_dataset_from_directory(
"/data/train2 cofi",
shuffle=True,
image_size= (IMAGE_SIZE,IMAGE_SIZE),
batch_size= BATCH_SIZE)
img_height = 256
img_width = 256
batch_size = 32
test_ds = tf.keras.preprocessing.image_dataset_from_directory(
"/data/test2 cofi",
image_size=(img_height, img_width),
batch_size=batch_size)
#preparing the model layers
input_shape = (BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, CHANNELS)
n_classes = 1
model = models.Sequential([
resize_and_rescale,
data_augmentation,
layers.Conv2D(32, (3,3), activation='relu', input_shape = input_shape), # 32-layers, (3,3)-filter size,activation function
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, kernel_size = (3,3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Flatten(),
layers.Dense(64, activation='relu'),
layers.Dense(n_classes, activation='sigmoid') # softmax helps to normalize the probabilities in the dense layer
])
model.build(input_shape=input_shape)
#Training the model
metrics = [TruePositives(name='TruePov'), FalsePositives(name='FalsePov'),
TrueNegatives(name='TrueNeg'), FalseNegatives(name='FalseNeg'),
BinaryAccuracy(name='accuracy'), Precision(name='Precision'), Recall(name='recall'), AUC(name='auc')]
# Using the adam optimizer to track the gradient descent in the training process
model.compile(
optimizer= Adam(learning_rate = 0.01),
loss = BinaryCrossentropy(),
metrics=metrics
)
history = model.fit(
train_ds,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
verbose=1,
validation_data=val_ds
)
# Building a Confusion Matrix
from sklearn.metrics import confusion_matrix, classification_report
import seaborn as sns
sns.set_style('darkgrid')
classes=test_ds.class_names # ordered list of class names
class_names = ['Arabica AA', 'Poor Arabica Quality']
ytrue=[]
for images, label in test_ds:
for e in label:
ytrue.append(classes[e]) # list of class names associated with each image file in test dataset
ypred=[]
errors=0
count=0
preds=model.predict(test_ds, verbose=1) # predict on the test data
for i, p in enumerate(preds):
count +=1
index=np.argmax(p) # get index of prediction with highest probability
klass=classes[index]
ypred.append(klass)
if klass != ytrue[i]:
errors +=1
acc= (count-errors)* 100/count
msg=f'there were {count-errors} correct predictions in {count} tests for an accuracy of {acc:6.2f} % '
print(msg)
ypred=np.array(ypred)
ytrue=np.array(ytrue)
if len(classes)<= 2: # if more than 30 classes plot is not useful to cramed
# create a confusion matrix
cm = confusion_matrix(ytrue, ypred )
length=len(classes)
if length<8:
fig_width=5
fig_height=5
else:
fig_width= int(length * .4)
fig_height= int(length * .4)
plt.figure(figsize=(fig_width, fig_height))
sns.heatmap(cm, annot=True, vmin=0, fmt='g', cmap='Blues', cbar=False)
plt.xticks(np.arange(length)+.3, classes, rotation= 90)
plt.yticks(np.arange(length)+.3, classes, rotation=0)
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.title("Confusion Matrix")
plt.show()
clr = classification_report(ytrue, ypred, target_names=class_namz)
print("Classification Report:\n----------------------\n", clr)
The output of the Confusion Matrix
enter image description here

Related

Getting wrong order of the Confusion Matrix using-CNN(Binary Classification for image dataset)

Am learning how to create CNN for a Binary classification task 0/1. I trained the model and tried to evaluate my model on the test data. My confusion Matrix gives me one column results such as TruePositive and False Negatives only!!
[[100 0] but was expecting something like this >> [[100 0]
[100 0]] [0 100]]
Actual Image of CM
Am not sure where my issue is coming from, whether its the way i structured the CNN layers or how i tried to plot the Confusion Matrix.
Here's look at the Code: Please guide How best can i create a confusion matrix!!
# loading the train data(data)
data = tf.keras.preprocessing.image_dataset_from_directory("data/train cofi",shuffle=True,
image_size= (IMAGE_SIZE,IMAGE_SIZE),
atch_size= BATCH_SIZE)
# the data
input_shape = (BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, CHANNELS)
n_classes = 1
model = models.Sequential([
resize_and_rescale,
data_augmentation,
layers.Conv2D(32, (3,3), activation='relu', input_shape = input_shape), # 32-layers, (3,3)-filter size,activation function
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, kernel_size = (3,3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Conv2D(64, (3, 3), activation='relu'),
layers.MaxPooling2D((2, 2)),
layers.Flatten(),
layers.Dense(64, activation='relu'),
layers.Dense(n_classes, activation='sigmoid') # softmax helps to normalize the probabilities in the dense layer
])
model.build(input_shape=input_shape)
# training the model on tran-data
model.compile(optimizer= Adam(learning_rate = 0.01),loss = BinaryCrossentropy(),metrics=metrics)
history = model.fit(train_ds, epochs=EPOCHS, batch_size=BATCH_SIZE, verbose=1, validation_data=val_ds)
Plotting the Confusion
from sklearn.metrics import confusion_matrix, classification_report
import seaborn as sns
sns.set_style('darkgrid')
test_3 = tf.keras.preprocessing.image_dataset_from_directory("/data/test2_cofi",
image_size= (IMAGE_SIZE,IMAGE_SIZE),
batch_size= BATCH_SIZE)
class_names = ['Arabica AA', 'Poor Arabica Quality']
classes = test_3.class_names #ordered list of class names
ytrue=[]
for images, label in test_ds:
for e in label:
ytrue.append(classes[e]) # list of class names associated with each image file in test dataset
ypred=[]
errors=0
count=0
preds=model1.predict(test_3, verbose=1) # predict on the test data
I suspect the use of np.argmax(p) below to be the issue in plotting the CM rather supposed to use a threshold like 0.5 If so, how do i change it incase its the issue!!
for i, p in enumerate(preds):
count +=1
index=np.argmax(p) # get index of prediction with highest probability
klass=classes[index]
ypred.append(klass)
if klass != ytrue[i]:
errors +=1
acc= (count-errors)* 100/count
msg=f'there were {count-errors} correct predictions in {count} tests for an accuracy of {acc:6.2f} % '
print(msg)

not sure why my CNN model isn't getting past 0.03 acc

I am using a CNN model to classify the spectrograms of different audio files.
this is the model, I created
input_shape = (128, 216, 1)
def create_model():
model = models.Sequential()
model.add(layers.Conv2D(32, (5, 5), strides = (1,1), name = 'conv0', input_shape = input_shape))
model.add(layers.BatchNormalization(axis = 3, name = 'bn0'))
model.add(layers.Activation('relu'))
model.add(layers.MaxPooling2D((2, 2), name='max_pool'))
model.add(layers.Conv2D(64, (3, 3), strides = (1,1), name='conv1'))
model.add(layers.Activation('relu'))
model.add(layers.AveragePooling2D((3, 3), name='avg_pool'))
model.add(layers.GlobalAveragePooling2D())
model.add(layers.Dense(300, activation='relu', name='rl'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(50, activation='softmax', name='sm'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics='acc')
return model
I'm using this to get the spectrograms from a file
for filename in pbar(os.listdir(save_file + r"spec-samples/")):
image_raw = tf.io.read_file(save_file + r"spec-samples/" + filename)
image = tf.image.decode_image(image_raw)
X.append(image)
and this to get the labels from a csv file
def reload_labels():
y = []
for data in tqdm(df.iterrows(), desc='Progress'):
y.append(data[1][1])
return y
when I fit the model via
cp_callback = tf.keras.callbacks.ModelCheckpoint(
checkpoint, monitor='accuracy', verbose=1, save_best_only=True, mode='max')
es_callback = tf.keras.callbacks.EarlyStopping(
monitor='loss', patience=3, verbose=1)
csv_callback = tf.keras.callbacks.CSVLogger(save_file + "logs.csv", separator=',', append=False)
history = model.fit(
X_, y_,
epochs=20,
validation_data=(X_test, y_test),
callbacks=[cp_callback, es_callback, csv_callback])
the highest accuracy I get is 0.026 which is obviously not great
this is an example of one of the spectrograms
spectrogram
what am i doing wrong?
I am a bit confused by your code. If I understand it you are feeding the network with images of spectrograms and I assume binary labels. The top layer of your model is
model.add(layers.Dense(50, activation='sigmoid', name='sm')
where does the value 50 come from? I would expect the last layer to have 1 node.
Also are you sure you have maintained the order of the images X to line up with the labels Y since they are coming from 2 different sources.
If you have 50 categories (classes) then the top layer should use activation softmax

How can I save the output of a convolution layer with and without a dilation rate as images

I want to save a image file to see about difference using convolution layer with dilation rate and without that.
Of course I can search images about that, but I want to see difference of my dataset.
Is there a special function? or Can I make code with opencv like a keras layers?
I am using
python 3.8 and
tf 2.4.0
You can use matplotlib and a custom Callback to save the feature maps of a CNN layer after every epoch. Here is a working example:
import tensorflow as tf
import pathlib
import matplotlib.pyplot as plt
dataset_url = "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz"
data_dir = tf.keras.utils.get_file('flower_photos', origin=dataset_url, untar=True)
data_dir = pathlib.Path(data_dir)
batch_size = 32
normalized_ds = tf.keras.utils.image_dataset_from_directory(
data_dir,
validation_split=0.2,
subset="training",
seed=123,
image_size=(180, 180),
batch_size=batch_size)
model = tf.keras.Sequential([
tf.keras.layers.Rescaling(1./255, input_shape=(180, 180, 3)),
tf.keras.layers.Conv2D(16, 3, padding='same', activation='relu', dilation_rate=(3, 3)),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Conv2D(32, 3, padding='same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Conv2D(64, 3, padding='same', activation='relu'),
tf.keras.layers.MaxPooling2D(),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(5)
])
class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, data, sample_size):
self.data = data
self.sample_size = sample_size
def on_epoch_end(self, epoch, logs=None):
images, _ = next(iter(self.data.take(self.sample_size)))
image = tf.expand_dims(images[0], axis=0)
output = first_cnn_layer(image)
plt.imsave('image{}.png'.format(epoch), images[0].numpy()/255)
ix = 1
for _ in range(4):
for _ in range(4):
ax = plt.subplot(4, 4, ix)
ax.set_xticks([])
ax.set_yticks([])
plt.imshow(output[0, :, :, ix-1], cmap='gray')
ix += 1
plt.savefig('filters{}.png'.format(epoch))
first_cnn_layer = model.layers[1]
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
history = model.fit(normalized_ds, epochs=2, callbacks=[CustomCallback(normalized_ds, 1)])
With dilation_rate=(1, 1) at epoch 1:
Original image
                  
Feature maps
With dilation_rate=(3, 3) at epoch 1:
Original image
                  
Feature maps

Getting very poor accuracy on stanford_dogs dataset

I'm trying to train a model on the stanford_dogs dataset to classify 120 dog breeds but my code is acting strange.
I downloaded the image data from http://vision.stanford.edu/aditya86/ImageNetDogs/images.tar
Then ran the following code to split each folder of the breeds into training and testing folders:
dataset_dict = {}
source_path = 'C:/Users/visha/Downloads/stanford_dogs/dataset'
dir_root = os.getcwd()
dataset_folders = [x for x in os.listdir(os.path.join(dir_root, source_path)) if os.path.isdir(os.path.join(dir_root, source_path, x))]
for category in dataset_folders:
dataset_dict[category] = {'source_path': os.path.join(dir_root, source_path, category),
'train_path': create_folder(new_path='C:/Users/visha/Downloads/stanford_dogs/train',
folder_type='train',
data_class=category),
'validation_path': create_folder(new_path='C:/Users/visha/Downloads/stanford_dogs/validation',
folder_type='validation',
data_class=category)}
dataset_folders = [x for x in os.listdir(os.path.join(dir_root, source_path)) if os.path.isdir(os.path.join(dir_root, source_path, x))]
for key in dataset_dict:
print("Splitting Category {} ...".format(key))
split_data(source_path=dataset_dict[key]['source_path'],
train_path=dataset_dict[key]['train_path'],
validation_path=dataset_dict[key]['validation_path'],
split_size=0.7)
I fed the images through the network after some image augmentation and used sigmoid activation in the final layer and categorical_crossentropy loss.
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.optimizers import RMSprop
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(16, (3, 3), activation='relu', input_shape=(150, 150, 3)),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),
tf.keras.layers.MaxPooling2D(2, 2),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dense(120, activation='softmax')
])
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
TRAINING_DIR = 'C:/Users/visha/Downloads/stanford_dogs/train'
train_datagen = ImageDataGenerator(rescale=1./255,rotation_range=40,width_shift_range=0.2,height_shift_range=0.2,
shear_range=0.2,zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')
train_generator = train_datagen.flow_from_directory(TRAINING_DIR,
batch_size=10,
class_mode='categorical',
target_size=(150, 150))
VALIDATION_DIR = 'C:/Users/visha/Downloads/stanford_dogs/validation'
validation_datagen = ImageDataGenerator(rescale=1./255, rotation_range=40,width_shift_range=0.2, height_shift_range=0.2,
shear_range=0.2,zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')
validation_generator = validation_datagen.flow_from_directory(VALIDATION_DIR,
batch_size=10,
class_mode='categorical',
target_size=(150, 150))
history = model.fit(train_generator,
epochs=10,
verbose=1,
validation_data=validation_generator)
But the code is not working as intended. The val_accuracy after 10 epochs is something like 4.756.
For validation data you should not do any image augmentation, just do rescale. In validation flow_from_directory set shuffle=False. Be advised that the Stanford Dog Data set is very difficult. To achieve a reasonable degree of accuracy you will need a much more complex model. I recommend you consider transfer learning using the Mobilenet model. The code below shows how to do that.
base_model=tf.keras.applications.mobilenet.MobileNet( include_top=False,
input_shape=(150,150,3) pooling='max', weights='imagenet',dropout=.4)
x=base_model.output
x=keras.layers.BatchNormalization(axis=-1, momentum=0.99, epsilon=0.001 )(x)
x = Dense(1024, activation='relu')(x)
x=Dropout(rate=.3, seed=123)(x)
output=Dense(120, activation='softmax')(x)
model=Model(inputs=base_model.input, outputs=output)
model.compile(Adamax(lr=.001),loss='categorical_crossentropy',metrics=
['accuracy'] )
I forgot to mention that Mobilenet was trained on Images with pixel values in the range -1 to +1. So in ImageDataGenerator include the code
preprocessing_function=tf.keras.applications.mobilenet.preprocess_input
this scales the pixels so you do not need the code
rescale=1./255
or alternatively set
rescale=1/157.5-1
which will rescale the values between -1 and +1

Confusion Matrix with convolution Neural Network

I'm new to using neural network and machine learning. I've tried to get the result from my Convolution Neural Network in form of confusion Matrix.
Currently, I got some problems in how to make it visually more easier to understand and how to make it work.
batch_size = 60
# Image Processing Stage
train_data = ImageDataGenerator(rescale=1./255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
test_data = ImageDataGenerator(rescale=1./255)
training_set = train_data.flow_from_directory('dataset/train_data', target_size=(128, 128), batch_size=batch_size, class_mode='binary')
test_set = test_data.flow_from_directory('dataset/test_data', target_size=(128, 128), batch_size=batch_size, class_mode='binary')
# Starting Convolution Neural Network
start_cnn = Sequential()
start_cnn.add(Conv2D(64, (3, 3), input_shape=(128, 128, 3), activation='relu', kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001))) # 3*3*3*32+32
start_cnn.add(Conv2D(64, (3, 3), activation='relu', kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001)))
start_cnn.add(MaxPooling2D(pool_size=(2, 2)))
for i in range(0, 2):
start_cnn.add(Conv2D(32, (3, 3), activation='relu', kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001)))
start_cnn.add(MaxPooling2D(pool_size=(2, 2)))
for i in range(0, 2):
start_cnn.add(Conv2D(32, (3, 3), activation='relu', kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001)))
start_cnn.add(MaxPooling2D(pool_size=(2, 2)))
start_cnn.add(Dropout(0.25))
# Flattening
start_cnn.add(Flatten())
# Step 4 - Full connection
start_cnn.add(Dense(activation="relu", units=32, kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001)))
start_cnn.add(Dropout(0.5))
start_cnn.add(Dense(activation="sigmoid", units=1, kernel_regularizer=l2(0.0001), bias_regularizer=l2(0.0001)))
start_cnn.summary()
start_cnn.load_weights("weights.best.hdf5")
# Compiling the CNN
start_cnn.compile(Adam(learning_rate=0.0001), loss='binary_crossentropy', metrics=['accuracy'])
history = start_cnn.fit(training_set, steps_per_epoch=85, epochs=1, validation_data=test_set, verbose=2, shuffle = False) # step per epoch is 5065/batch size callbacks=callbacks_list,
This is a brief on how my code layout is organized. Please, if there is any comment or concern please do tell me, I am new and would like to learn as much as possible.
Here is the confusion matrix code I did
y_pred = start_cnn.predict_generator(test_set, steps = np.ceil(test_set.samples / test_set.batch_size), verbose=0, workers=0)
y_pred = np.where(y_pred>0.5, 1, 0)
print('Confusion Matrix')
print(confusion_matrix(test_set.classes, y_pred))
print('Classification Report')
print(classification_report(test_set.classes, y_pred))
labels = ['melanoma', 'non-melanoma']
cm = confusion_matrix(test_set.classes, y_pred)
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(cm)
plt.title('Confusion matrix')
fig.colorbar(cax)
ax.set_xticklabels([''] + labels)
ax.set_yticklabels([''] + labels)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()
As you can see in the image here. The value only appears in the two box on the left side but not on the right side at all. Also, I would like to use plot_confusion_matrix too, but I can't seems to find a way to get it working at all.
enter image description here
Please help!! Thank you for your time

Categories

Resources