Develop Reference

ValueError: Input to `.fit()` should have rank 4. Got array with shape: (704, 227, 227)

I am running my LeNet code with LFW, but when I run it, I am getting the following error message:
Traceback (most recent call last):
File "C:\Users\JoshG\PycharmProjects\LeNet\LeNet.py", line 134, in <module>
train_generator.fit(xtrain)
`ValueError: Input to `.fit()` should have rank 4. Got array with shape: (704, 227, 227`)
Here is part of the code that it is getting the error? What is the fix for something like this?
# Import the packages
from matplotlib import pyplot
import matplotlib.pyplot as plt
from pipeline.nn.conv import mini_lenet
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, \
f1_score
from pipeline.callbacks import train
from keras.layers import Conv2D, MaxPool2D, Dense, Flatten
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD
from keras.utils.np_utils import to_categorical
from image_dataset_loader import load
import numpy as np
import argparse
from sklearn.model_selection import train_test_split
import os
import tensorflow as tf
# Construct the argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-m", "--model", required=True, help="path to output model")
ap.add_argument("-o", "--output", required=True, help="path to output directory (logs, plots, etc.)")
args = vars(ap.parse_args())
# Loading FERET Dataset
print("[INFO] loading FERET Dataset...")
# Josh Plan 1 Pathway: LeNet (LFW/CARE)
mainPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_LFW-CARE/Plan1-DL-LFW-CARE/Images_LeNet-LFW-A"
trainPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_LFW-CARE/Plan1-DL-LFW-CARE/Images_LeNet-LFW-A/Josh_Training_SEN_8I_100S_227"
testPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_LFW-CARE/Plan1-DL-LFW-CARE/Images_LeNet-LFW-A/Josh_Testing_SEN_100S_227"
(xtrain, ytrain), (xtest, ytest) = load(mainPath, [trainPath, testPath])
# Josh Plan 1 Pathway: LeNet (FERET)
#mainPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_FERET/Plan1-DL/Images_LeNet"
#trainPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_FERET/Plan1-DL/Images_LeNet/Josh_Training_CHD_8I_24S_227"
#testPath = "C:/Users/JoshG/PycharmProjects/Local-Binary-Patterns/Images_FERET/Plan1-DL/Images_LeNet/Josh_Testing_CHD_24S_227"
#(xtrain, ytrain), (xtest, ytest) = load(mainPath, [trainPath, testPath])
# Plot first few images
for i in range(9):
# define subplot
pyplot.subplot(330 + 1 + i)
pyplot.imshow(xtest[i])
# Show the figure
pyplot.show()
LeNet = Sequential()
LeNet.add(Conv2D(6, (5, 5), strides=(1, 1), padding='valid', activation='relu',
input_shape=(227, 227, 3)))
LeNet.add(MaxPool2D((2, 2), (2, 2)))
LeNet.add(Conv2D(16, (5, 5), (1, 1), padding='valid', activation='relu'))
LeNet.add(MaxPool2D((2, 2), (2, 2)))
LeNet.add(Conv2D(120, (5, 5), (1, 1), padding='valid', activation='relu'))
LeNet.add(Flatten())
LeNet.add(Dense(84, activation='relu'))
LeNet.add(Dense(2, activation='softmax'))
# model.compile(optimizer='adam', loss=keras.losses.binary_crossentropy, metrics=['accuracy'])
LeNet.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001), loss='categorical_crossentropy',
metrics=["accuracy"])
LeNet.summary()
# Construct the argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-m", "--model", required=True, help="path to output model")
ap.add_argument("-o", "--output", required=True, help="path to output directory (logs, plots, etc.)")
args = vars(ap.parse_args())
# Dimension of the FERET dataset
print('----- Dimension of FERET Dataset -----')
print("The shape of the training image:", xtrain.shape)
print("The shape of testing image: ", xtest.shape)
print("Shape of a single image: ", xtest[0].shape)
print('-----------------------------------------')
# Number of epoches & learning rate I use
num_EPOCHS = 75
learn_rate = 5e-3
def decay_poly(epoch):
epochMax = num_EPOCHS
learn_rateBase = learn_rate
power = 1.0
# compute the new learning rate based on polynomial decay
alpha = learn_rateBase * (1 - (epoch / float(epochMax))) ** power
# return the new learning rate
return alpha
# Print out first test to make sure we can see images
xtrain = xtrain.astype("float32")
xtest = xtest.astype("float32")
# apply mean subtraction to the data
mean = np.mean(xtrain, axis=0)
xtrain -= mean
xtest -= mean
# Train-validation-test split
xtrain, x_val, ytrain, y_val = train_test_split(xtrain, ytrain, test_size=.12)
print('')
# Dimension of the FERET dataset
print('Dimension of FERET Dataset')
print((xtrain.shape, ytrain.shape))
print((x_val.shape, y_val.shape))
print((xtest.shape, ytest.shape))
# Since we have 10 classes we should expect the shape[1] of ytrain,y_val and ytest to change from 1 to 10
ytrain = to_categorical(ytrain)
y_val = to_categorical(y_val)
ytest = to_categorical(ytest)
print('')
print('Verifying the dimension after one hot encoding')
print((xtrain.shape, ytrain.shape))
print((x_val.shape, y_val.shape))
print((xtest.shape, ytest.shape))
# Image Data Augmentation
train_generator = ImageDataGenerator(rotation_range=2, horizontal_flip=True, zoom_range=.1)
val_generator = ImageDataGenerator(rotation_range=2, horizontal_flip=True, zoom_range=.1)
test_generator = ImageDataGenerator(rotation_range=2, horizontal_flip=True, zoom_range=.1)
# Fitting the augmentation defined above to the data
train_generator.fit(xtrain.reshape(96, 227, 227, 1))
val_generator.fit(x_val.reshape(96, 227, 227, 1))
test_generator.fit(xtest.reshape(96, 227, 227, 1))
# Construct the image generator for data augmentation
aug = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1,
horizontal_flip=True, fill_mode="nearest")
# Construct the set of callbacks
figPath = os.path.sep.join([args["output"], "{}.png".format(os.getpid())])
jsonPath = os.path.sep.join([args["output"], "{}.json".format(os.getpid())])
callbacks = [train(figPath, jsonPath=jsonPath),
LearningRateScheduler(decay_poly)]
print('')
# Initialize the optimizer and model
print("[INFO] compiling model...")
opt = SGD(lr=learn_rate, momentum=0.9)
LeNet = mini_lenet.build(width=227, height=227, depth=3, classes=100) # CHANGE WHEN RUNNING EITHER LFW OR FERET (FERET: 24 or LFW: 100)
LeNet.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"])
LeNet.evaluate(xtest, ytest)
# Train the network
print("[INFO] training network...")
LeNet.fit(aug.flow(xtrain, ytrain, batch_size=25),
validation_data=(xtest, ytest), steps_per_epoch=len(xtrain) // 64,
epochs=num_EPOCHS, callbacks=callbacks, verbose=1)
# Plotting the training and validation loss
f, ax = plt.subplots(1, 1) # Creates 2 subplots under 1 column
# Assigning the first subplot to graph training loss and validation loss
ax.plot(LeNet.history.history['loss'], color='b', label='Training Loss')
ax.plot(LeNet.history.history['val_loss'], color='r', label='Validation Loss')
ax.set_title('Training & Validation Loss Graph')
plt.legend()
plt.show()
# Plotting the training and validation accuracy
f, ax = plt.subplots(1, 1) # Creates 2 subplots under 1 column
# Plotting the training accuracy and validation accuracy
ax.plot(LeNet.history.history['accuracy'], color='b', label='Training Accuracy')
ax.plot(LeNet.history.history['val_accuracy'], color='r', label='Validation Accuracy')
ax.set_title('Training & Validation Accuracy Graph')
plt.legend()
plt.show()
# Defining function for confusion matrix plot
def plot_confusion_matrix(y_true, y_pred, classes,
normalize=False,
title=None,
cmap=plt.cm.Blues):
if not title:
if normalize:
title = 'Normalized confusion matrix'
else:
title = 'Confusion matrix, without normalization'
# Compute confusion matrix
cm = confusion_matrix(y_true, y_pred)
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
# Print Confusion matrix
fig, ax = plt.subplots(figsize=(5, 5))
im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
ax.figure.colorbar(im, ax=ax)
# We want to show all ticks...
ax.set(xticks=np.arange(cm.shape[1]),
yticks=np.arange(cm.shape[0]),
xticklabels=classes, yticklabels=classes,
title=title,
ylabel='True label',
xlabel='Predicted label')
# Rotate the tick labels and set their alignment.
plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
rotation_mode="anchor")
# Loop over data dimensions and create text annotations.
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i in range(cm.shape[0]):
for j in range(cm.shape[1]):
ax.text(j, i, format(cm[i, j], fmt),
ha="center", color="white"
if cm[i, j] > thresh else "black")
plt.tight_layout()
return ax
# Making prediction
xtest_arg = np.argmax(ytest, axis=1)
y_pred = np.argmax(LeNet.predict(xtest), axis=1)
y_true = np.argmax(ytest, axis=1)
print(y_pred)
print(y_pred.shape)
# Plotting the confusion matrix
confusion_mtx = confusion_matrix(y_true, y_pred)
CLASS_NAMES = [f.name for f in os.scandir(trainPath) if f.is_dir()]
class_names = CLASS_NAMES
print(class_names)
print("ypred\n", y_pred)
print("ytrue", y_true)
# Print missed labels
print('')
print('Print Missed Labels:')
for i in range(len(y_pred)):
if y_pred[i] != y_true[i]:
print(class_names[y_true[i]], "has been missed classified as", class_names[y_pred[i]])
print('')
# Plotting non-normalized confusion matrix
plot_confusion_matrix(y_true, y_pred, classes=class_names, title='Confusion matrix, without normalization')
plt.show()
# Plotting normalized confusion matrix
# plot_confusion_matrix(y_true, y_pred, classes=class_names, normalize=True, title='Normalized confusion matrix')
# plt.show()
# Classification Metrics
acc_score = accuracy_score(y_true, y_pred)
print('\n\n\t\t Accuracy Score: ', str(round((100 * acc_score), 2)), '%')
prec_score = precision_score(y_true, y_pred, average='macro')
print(' Precision Score Macro: ', str(round((100 * prec_score), 2)), '%')
prec_score = precision_score(y_true, y_pred, average='micro')
print(' Precision Score Micro: ', str(round((100 * prec_score), 2)), '%')
prec_score = precision_score(y_true, y_pred, average='weighted')
print('Precision Score Weighted: ', str(round((100 * prec_score), 2)), '%')
rec_score = recall_score(y_true, y_pred, average='macro')
print('\t\t\tRecall Macro: ', str(round((100 * rec_score), 2)), '%')
rec_score = recall_score(y_true, y_pred, average='micro')
print('\t\t\tRecall Micro: ', str(round((100 * rec_score), 2)), '%')
rec_score = recall_score(y_true, y_pred, average='weighted')
print('\t\t Recall Weighted: ', str(round((100 * rec_score), 2)), '%')
f_score = f1_score(y_true, y_pred, average='macro')
print('\t\t F1 Score Macro: ', str(round((100 * f_score), 2)), '%')
f_score = f1_score(y_true, y_pred, average='micro')
print('\t\t F1 Score Micro: ', str(round((100 * f_score), 2)), '%')
f_score = f1_score(y_true, y_pred, average='weighted')
print('\t F1 Score Weighted: ', str(round((100 * f_score), 2)), '%')
I then added the following
# Fitting the augmentation defined above to the data
train_generator.fit(xtrain.reshape(96, 227, 227, 1))
val_generator.fit(x_val.reshape(96, 227, 227, 1))
test_generator.fit(xtest.reshape(96, 227, 227, 1))
but then got this:
Traceback (most recent call last):
File "C:\Users\JoshG\PycharmProjects\LeNet\LeNet.py", line 134, in <module>
train_generator.fit(xtrain.reshape(96, 227, 227, 1))
ValueError: cannot reshape array of size 36276416 into shape (96,227,227,1)
I have added the full code for more help on how to combat this issue.

AttributeError: 'TensorShape' object has no attribute 'assert_has_rank' (when trying to predict on an image classification keras model)

I took over a simple convolutional network from the internet for a simple binary image classification task, it's my first try on image classification. It trains well on the sets of images I collected. This is the code for training:
import matplotlib.pyplot as plt
import seaborn as sns
import keras
from keras.models import Sequential
from keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import Adam
from sklearn.metrics import classification_report
import tensorflow as tf
import cv2
import os
import numpy as np
from pathlib import Path
cwd = os.getcwd()
my_file = Path(cwd + "/ddornotbiclassifier/keras_model.h5")
labels = ['dd', 'notadd']
img_size = 224
def get_data(data_dir):
data = []
for label in labels:
path = os.path.join(data_dir, label)
class_num = labels.index(label)
for img in os.listdir(path):
try:
img_arr = cv2.imread(os.path.join(path, img))[...,::-1] #convert BGR to RGB format
resized_arr = cv2.resize(img_arr, (img_size, img_size)) # Reshaping images to preferred size
data.append([resized_arr, class_num])
except Exception as e:
print(e)
return np.array(data)
train = get_data(cwd + "/images/dd_notadd/train")
val = get_data(cwd + "/images/dd_notadd/valid")
l = []
for i in train:
if(i[1] == 0):
l.append("dd")
else:
l.append("notdd")
sns.set_style('darkgrid')
sns.countplot(l)
x_train = []
y_train = []
x_val = []
y_val = []
for feature, label in train:
x_train.append(feature)
y_train.append(label)
for feature, label in val:
x_val.append(feature)
y_val.append(label)
# Normalize the data
x_train = np.array(x_train) / 255
x_val = np.array(x_val) / 255
x_train.reshape(-1, img_size, img_size, 1)
y_train = np.array(y_train)
x_val.reshape(-1, img_size, img_size, 1)
y_val = np.array(y_val)
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range = 30, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range = 0.2, # Randomly zoom image
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip = True, # randomly flip images
vertical_flip=False) # randomly flip images
datagen.fit(x_train)
if not my_file.is_file():
print("There was no saved model so model will be trained from scratch")
model = Sequential()
model.add(Conv2D(32,3,padding="same", activation="relu", input_shape=(224,224,3)))
model.add(MaxPool2D())
model.add(Conv2D(32, 3, padding="same", activation="relu"))
model.add(MaxPool2D())
model.add(Conv2D(64, 3, padding="same", activation="relu"))
model.add(MaxPool2D())
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(128,activation="relu"))
model.add(Dense(2, activation="softmax"))
model.summary()
opt = Adam(lr=0.000001)
model.compile(optimizer = opt , loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) , metrics = ['accuracy'])
else:
print("There was a saved model so that model will be loaded and continued to train")
pathtomodel = cwd + '/ddornotbiclassifier/keras_model.h5'
model = keras.models.load_model(pathtomodel)
opt = Adam(lr=0.000001)
model.compile(optimizer = opt , loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) , metrics = ['accuracy'])
history = model.fit(x_train,y_train,epochs = 3085, validation_data = (x_val, y_val))
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs_range = range(3085)
plt.figure(figsize=(15, 15))
plt.subplot(2, 2, 1)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')
plt.subplot(2, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()
predictions = model.predict_classes(x_val)
predictions = predictions.reshape(1,-1)[0]
print(classification_report(y_val, predictions, target_names = ['dd (Class 0)','notadd (Class 1)']))
model.save("./ddornotbiclassifier/keras_model.h5")
Then I created another python file to predict on the model. This is what I have so far:
import os
import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout
from keras.optimizers import Adam
import tensorflow as tf
import cv2
cwd = os.getcwd()
model = Sequential()
model.add(Conv2D(32,3,padding="same", activation="relu", input_shape=(224,224,3)))
model.add(MaxPool2D())
model.add(Conv2D(32, 3, padding="same", activation="relu"))
model.add(MaxPool2D())
model.add(Conv2D(64, 3, padding="same", activation="relu"))
model.add(MaxPool2D())
model.add(Dropout(0.4))
model.add(Flatten())
model.add(Dense(128,activation="relu"))
model.add(Dense(2, activation="softmax"))
pathtomodel = cwd + '/ddornotbiclassifier/keras_model.h5'
model.load_weights(pathtomodel)
opt = Adam(lr=0.000001)
model.compile(optimizer = opt , loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) , metrics = ['accuracy'])
path = cwd + "/images/dd_notadd/testimage/testimage.jpeg"
img = cv2.imread(path)
img = cv2.resize(img,(224,224))
img = np.reshape(img,[1,224,224,3])
model.predict(img)
Then this is the output:
WARNING:tensorflow:9 out of the last 11 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fe6e5799430> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating #tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your #tf.function outside of the loop. For (2), #tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.
Traceback (most recent call last):
File "/Users/lennartkonst/Documents/Chatbots/Pythonbots/DreamDestinationsBot/ML/ddornotinference2.py", line 44, in <module>
model.predict(img)
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 130, in _method_wrapper
return method(self, *args, **kwargs)
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 1614, in predict
all_outputs = nest.map_structure_up_to(batch_outputs, concat, outputs)
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/nest.py", line 1135, in map_structure_up_to
return map_structure_with_tuple_paths_up_to(
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/nest.py", line 1234, in map_structure_with_tuple_paths_up_to
results = [func(*args, **kwargs) for args in zip(flat_path_list,
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/nest.py", line 1234, in <listcomp>
results = [func(*args, **kwargs) for args in zip(flat_path_list,
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/nest.py", line 1137, in <lambda>
lambda _, *values: func(*values), # Discards the path arg.
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py", line 2673, in concat
return array_ops.concat(tensors, axis=axis)
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
return target(*args, **kwargs)
File "/opt/anaconda3/lib/python3.8/site-packages/tensorflow/python/ops/array_ops.py", line 1650, in concat
ops.convert_to_tensor(
AttributeError: 'TensorShape' object has no attribute 'assert_has_rank'
I'm stuck and it gives this error on many type of tries. A numpy array of shape 1,224,224,3 should do it right?
What should I do?
Thank you.

How to get confusion matrix from the below model?

The X represents features and Y represents labels for image classification. I am using CNN for binary image classification purpose like that of cats and dogs.
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
import pickle
import numpy as np
from sklearn.metrics import confusion_matrix
X = np.array(pickle.load(open("X.pickle","rb")))
Y = np.array(pickle.load(open("Y.pickle","rb")))
x_test = np.array(pickle.load(open("x_test.pickle","rb")))
y_test = np.array(pickle.load(open("y_test.pickle","rb")))
# X = np.array(pickle.load(open("x_train.pickle","rb")))
# Y = np.array(pickle.load(open("y_train.pickle","rb")))
#scaling our image data
X = X/255.0
model = Sequential()
#model.add(Conv2D(64 ,(3,3), input_shape = X.shape[1:]))
model.add(Conv2D(64 ,(3,3), input_shape = X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Conv2D(128 ,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Conv2D(256 ,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Conv2D(512 ,(3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Flatten())
model.add(Dense(2048))
model.add(Activation("relu"))
model.add(Dropout(0.5))
np.argmax(model.add(Dense(2)))
model.add(Activation('softmax'))
model.compile(loss="binary_crossentropy",
optimizer = "adam",
metrics = ['accuracy'])
predicted = model.predict(x_test)
print(predicted.shape)
print(y_test.shape)
print(confusion_matrix(y_test,predicted))
The output of predicted and y_test shapes are (90, 2) and
(90,) and when I used confusion matrix it flushes:-
ValueError: Classification metrics can't handle a mix of binary and continuous-multioutput targets.

You can use scikit-learn:
from sklearn.metrics import confusion_matrix
predicted = model.predict(x_test)
print(confusion_matrix(y_test,predicted.round()))
Here's scikit learn documentation for confusion matrix:
https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html
Edit:
Advice:
Prefer using Softmax activation on output layer and whether it is binary or multi label classification. Use softmax with number of nodes in output layer = no of classes.

Here's one example how we can get the confusion matrix using the PyCM library:
you need to install the pycm library in anaconda or by using pip3
conda install -c sepandhaghighi pycm
from pycm import *
def plot_confusion_matrix(cm, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):
"""
This function modified to plots the ConfusionMatrix object.
Normalization can be applied by setting 'normalize=True'.
Code Reference :
http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html
"""
plt_cm = []
for i in cm.classes :
row=[]
for j in cm.classes:
row.append(cm.table[i][j])
plt_cm.append(row)
plt_cm = np.array(plt_cm)
if normalize:
plt_cm = plt_cm.astype('float') / plt_cm.sum(axis=1)[:, np.newaxis]
plt.imshow(plt_cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(cm.classes))
plt.xticks(tick_marks, cm.classes, rotation=45)
plt.yticks(tick_marks, cm.classes)
fmt = '.2f' if normalize else 'd'
thresh = plt_cm.max() / 2.
for i, j in itertools.product(range(plt_cm.shape[0]), range(plt_cm.shape[1])):
plt.text(j, i, format(plt_cm[i, j], fmt),
horizontalalignment="center",
color="white" if plt_cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('Actual')
plt.xlabel('Prediction')
.....
svm.fit(x_train, y_train)
y_predicted = svm.predict(x_test)
#Get the confusion Matrix:
cm = ConfusionMatrix(actual_vector=y_test, predict_vector=y_predicted)
#Print classes
print("[INFO] Clases")
print(cm.classes)
#Print the table of cmatrix
print(cm.table)
#indicators of the confusion matrix
print(cm)
greetings!

Why does my Deep Convolution Neural Network architecture works better with small dataset

I developed a Deep convolution neural network for multiclass image classification with Keras and Tensorflow as the background. I discovered that my model works better when I used 1000 dataset compared to when I used 5000-25,000 dataset. I know deep learning works better with more dataset but reverse is the case for me. Any clue or assistance will be appreciated. Below is my code:
from keras.utils import to_categorical
from keras.preprocessing import image
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.model_selection import train_test_split
# Importing Keras packages
from keras.models import Sequential
from keras.layers import Convolution2D, Conv2D
from keras.layers import MaxPooling2D
from keras.layers import AveragePooling2D
from keras.layers import Flatten
from keras.layers import Dense, Dropout, BatchNormalization
from matplotlib import pyplot as plt
# demonstration of calculating metrics for a neural network model using sklearn
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import precision_score
from sklearn.metrics import multilabel_confusion_matrix
from sklearn.metrics import roc_auc_score, average_precision_score,f1_score,recall_score
from sklearn.metrics import confusion_matrix, balanced_accuracy_score
train_csv= pd.read_csv('dataset/ISIC_2019_Training_GroundTruth1k.csv') # reading the csv file
train_image = []
for i in tqdm(range(train_csv.shape[0])):
img = image.load_img('dataset/data/'+train_csv['Original'][i]+'.jpg',target_size=(32,32,3))
img = image.img_to_array(img)
img = img/255
train_image.append(img)
X = np.array(train_image)
X.shape
y = np.array(train_csv.drop(['Original', 'Filtered', 'Segmented'], axis=1))
y.shape
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2)
#initializing the CNN
classifier = Sequential()
#Adding the Convolution Layer
classifier.add(Convolution2D(32, 3, 3, input_shape = (32, 32, 3), activation = "relu"))
classifier.add(Convolution2D(32, 3, 3, activation = "relu"))
classifier.add(Dropout(0.25))
classifier.add(MaxPooling2D(pool_size=(2, 2)))
classifier.add(Convolution2D(32, 3, 3, activation = "relu"))
classifier.add(Convolution2D(32, 3, 3, activation = "relu"))
classifier.add(Dropout(0.25))
classifier.add(AveragePooling2D(pool_size=(2, 2)))
#Flattening the layer
classifier.add(Flatten())
classifier.add(Dense(128, activation = "relu"))
classifier.add(Dropout(0.5))
classifier.add(Dense(9, activation ="softmax"))
classifier.summary()
#Compiling the CNN
classifier.compile(optimizer = 'adam', loss='categorical_crossentropy', metrics=['accuracy'])
history = classifier.fit(X_train, y_train, epochs=60, validation_data=(X_test, y_test))
# evaluate the model
_, train_acc = classifier.evaluate(X_train, y_train, verbose=0)
_, test_acc = classifier.evaluate(X_test, y_test, verbose=0)
print('Train Loss: %.3f, Test Loss: %.3f' % (np.amin(history.history['loss']), np.amin(history.history['val_loss'])))
print('Train Accuracy: %.3f, Test Accuracy: %.3f' % (np.amax(history.history['acc']), np.amax(history.history['val_acc'])))
# Convert to 1D
yhat_probs = classifier.predict(X_test, verbose=0)
yhat_probs[1]
yhat_classes = classifier.predict_classes(X_test, verbose=0)
yhat_classes[1]
# Convert to 1D
rounded_labels=np.argmax(y_test, axis=1)
rounded_labels[1]
balanced_accuracy = balanced_accuracy_score(rounded_labels, yhat_classes)
print('Balanced Accuracy: %f' % balanced_accuracy)
matthews = matthews_corrcoef(rounded_labels, yhat_classes)
print('Matthews: %f' % matthews)
# confusion matrix
matrix = confusion_matrix(rounded_labels, yhat_classes)
print("Confusion Matrix: %s" % matrix)
multilabel_matrix = multilabel_confusion_matrix(rounded_labels, yhat_classes)
print("Multilabel Matrix: %s" % multilabel_matrix)
precision = precision_score(rounded_labels, yhat_classes)
print("Precision %f" % precision)
f1 = f1_score(rounded_labels, yhat_classes)
print("F1: %f" % f1)
recall = recall_score(rounded_labels, yhat_classes)
print("Recall: %f" % recall)
average_precision= average_precision_score(rounded_labels, yhat_classes)
print("Average Precision: %f" % average_precision)
# ROC AUC
roc_auc = roc_auc_score(rounded_labels, yhat_classes)
print('ROC AUC: %f' % roc_auc)
# plot loss during training
plt.subplot(211)
plt.title('Loss')
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='test')
plt.legend()
# plot accuracy during training
plt.subplot(212)
plt.title('Accuracy')
plt.plot(history.history['acc'], label='train')
plt.plot(history.history['val_acc'], label='test')
plt.legend()
plt.show()

It would be great printing out the evolution of training and validation loss during the 60 epochs. You can do it like this:
import matplotlib.pyplot as plt
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'val_loss'], loc='upper right')
Maybe with samples, you should use more epochs, you could realize this looking at the evolution of the training loss.
You should also verify the distribution of the data you use. If the 5000 samples you use have a different distribution w.r.t the first 1000, that may be the cause of a different result. Try using more data, and including both the sets.

while using the large dataset you should try these techniques
Try to use transfer learning on pretrained models https://keras.io/applications/
Try to use data augmentation https://keras.io/preprocessing/image/
You can also try K-fold cross validation for test train split https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.StratifiedKFold.html

ROC curve looks too sharp and not smooth

I am learning to write CNNs in Keras on Kaggle using one of the datasets I found there.
The link to my notebook is
https://www.kaggle.com/vj6978/brain-tumor-vimal?scriptVersionId=16814133
The code, the dataset and the ROC curve are available at the link. The ROC curve I am plotting seems to look sharp and not smooth.
The code is as follows:
import os
import cv2
import random
import numpy as np
from numpy.lib.stride_tricks import as_strided
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import matplotlib.pyplot as plt
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, Conv2D, MaxPooling2D, AveragePooling2D, Dropout, Flatten
from PIL import Image
data_set = []
data_label = []
training_data = []
input_path = "../input/brain_tumor_dataset"
CATEGORIES = ["no", "yes"]
"""
The show function simply takes in a numpy array and displays it as an image.
"""
def show(img_input):
plt.imshow(img_input)
plt.show()
def create_training_data():
for category in CATEGORIES:
path = os.path.join(input_path, category)
category_index = CATEGORIES.index(category)
for image in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path, image), cv2.IMREAD_GRAYSCALE)
img_array = img_array.astype(np.float32)
img_array = cv2.resize(img_array, (128, 128))
training_data.append([img_array, category_index])
except Exception as e:
print(e)
create_training_data()
random.shuffle(training_data)
for feature, label in training_data:
data_set.append(feature)
data_label.append(label)
x_train, x_test, y_train, y_test = train_test_split(data_set, data_label, test_size = 0.1,
random_state = 45)
data_set = np.array(x_train).reshape(-1, 128, 128, 1)
x_test = np.array(x_test).reshape(-1, 128, 128, 1)
data_set = data_set/255.0
model = Sequential()
model.add(Conv2D(128, (3,3), input_shape = data_set.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Conv2D(128, (3,3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Dense(1))
model.add(Activation("sigmoid"))
model.summary()
model.compile(optimizer = "adam", loss = "binary_crossentropy", metrics = ['accuracy'])
model.fit(data_set, y_train, batch_size = 32, epochs = 15, validation_split = 0.1)
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
y_pred_keras = model.predict(x_test).ravel()
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_test)
auc_keras = auc(fpr_keras, tpr_keras)
plt.figure(1)
plt.plot([0, 1], [0, 1], 'k--')
plt.plot(fpr_keras, tpr_keras, label='Keras (area = {:.3f})'.format(auc_keras))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')
plt.show()
The curve looks like this
Any help would be appreciated.
Thanks Vimal James

fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_test) this line is where your error is. I think you meant :
fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, y_pred_keras)