Develop Reference

Accuracy and val_accuracy don't change while training

I tried to train my convolutional neural network using tensorflow and keras libraries. But the values of accuracy and val_accuracy didn't change the whole time. There is my neural network code:
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
import pickle
X = pickle.load(open("X.pickle", "rb"))
y = pickle.load(open("y.pickle", "rb"))
X = X/255.0
model = Sequential()
model.add(Conv2D(64, (3, 3), input_shape=X.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64, activation="relu"))
model.add(Dense(1, activation="sigmoid"))
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.fit(X, y, batch_size=10, epochs=10, validation_split=0.1)
There is the creation of traning data, features and labels (X - features, y - labels)
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR, category)
class_num = CATEGORIES.index(category)
for img in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path, img), cv2.IMREAD_GRAYSCALE)
new_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
training_data.append([new_array, class_num])
except Exception as e:
pass
create_training_data()
random.shuffle(training_data)
X = []
y = []
for features, label in training_data:
X.append(features)
y.append(label)
X = np.array(X).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y = np.array(y)
And this is the log of training:
2023-01-15 00:36:42.368335: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
Epoch 1/10
70/70 [==============================] - 45s 619ms/step - loss: 0.3039 - accuracy: 0.9627 - val_loss: 0.1211 - val_accuracy: 0.9744
Epoch 2/10
70/70 [==============================] - 42s 600ms/step - loss: 0.1524 - accuracy: 0.9670 - val_loss: 0.1189 - val_accuracy: 0.9744
Epoch 3/10
70/70 [==============================] - 42s 600ms/step - loss: 0.1537 - accuracy: 0.9670 - val_loss: 0.1622 - val_accuracy: 0.9744
Epoch 4/10
70/70 [==============================] - 44s 627ms/step - loss: 0.1563 - accuracy: 0.9670 - val_loss: 0.1464 - val_accuracy: 0.9744
Epoch 5/10
70/70 [==============================] - 42s 604ms/step - loss: 0.1591 - accuracy: 0.9670 - val_loss: 0.1185 - val_accuracy: 0.9744
Epoch 6/10
70/70 [==============================] - 42s 605ms/step - loss: 0.1511 - accuracy: 0.9670 - val_loss: 0.1338 - val_accuracy: 0.9744
Epoch 7/10
70/70 [==============================] - 49s 698ms/step - loss: 0.1623 - accuracy: 0.9670 - val_loss: 0.1188 - val_accuracy: 0.9744
Epoch 8/10
70/70 [==============================] - 50s 709ms/step - loss: 0.1480 - accuracy: 0.9670 - val_loss: 0.1397 - val_accuracy: 0.9744
Epoch 9/10
70/70 [==============================] - 45s 637ms/step - loss: 0.1508 - accuracy: 0.9670 - val_loss: 0.1203 - val_accuracy: 0.9744
Epoch 10/10
70/70 [==============================] - 47s 665ms/step - loss: 0.1716 - accuracy: 0.9670 - val_loss: 0.1238 - val_accuracy: 0.9744
Process finished with exit code 0
What should I do to fix this problem?

There are a couple potential reasons as to why you are facing this:
Your dataset is far too small. If your validation set is tiny, there is a high probability that your model will get the same % of predictions correct/incorrect
There is a great imbalance in your dataset. If one class heavily outweighs another, your model will favor the majority class, and predict it no matter what, as that is what brings the optimal accuracy for the model.
From what I see, there is nothing wrong with your code, rather modifications that need to be made to the dataset itself.

Hmm accuracy and validation accuracy are high even on the first epoch. Try using a lower learning rate in the Adam optimizer say .0002, On the first epoch pay attention to the loss and validation loss as the batches are process. It should start low and gradually increase during the epoch.

Tensorflow model validation accuracy not increasing

I have built a tensorflow model and am getting no change in my validation accuracy in different epochs, which makes me believe there is something wrong in my setup. Below is my code.
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import regularizers
import tensorflow as tf
model = Sequential()
model.add(Conv2D(16, (3, 3), input_shape=(299, 299,3),padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Conv2D(32, (3, 3),padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Conv2D(64, (3, 3),padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
model.add(Conv2D(64, (3, 3),padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2),padding='same'))
# this converts our 3D feature maps to 1D feature vectors
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
batch_size=32
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
rescale=1./255,
# shear_range=0.2,
# zoom_range=0.2,
horizontal_flip=True)
# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255)
# this is a generator that will read pictures found in
# subfolers of 'data/train', and indefinitely generate
# batches of augmented image data
train_generator = train_datagen.flow_from_directory(
'Documents/Training', # this is the target directory
target_size=(299, 299), #all images will be resized to 299
batch_size=batch_size,
class_mode='binary') # since we use binary_crossentropy loss, we need binary labels
# this is a similar generator, for validation data
validation_generator = test_datagen.flow_from_directory(
'Documents/Dev',
target_size=(299, 299),
batch_size=batch_size,
class_mode='binary')
#w1 = tf.Variable(tf.truncated_normal([784, 30], stddev=0.1))
model.fit_generator(
train_generator,
steps_per_epoch=50 // batch_size,
verbose = 1,
epochs=10,
validation_data=validation_generator,
validation_steps=8 // batch_size)
Which when I run produces the following output. Anything I'm missing here as far as my architecture is concerned or data generation steps? I have referenced Tensorflow model accuracy not increasing and accuracy not increasing in tensorflow model to no avail yet.
Epoch 1/10
3/3 [==============================] - 2s 593ms/step - loss: 0.6719 - accuracy: 0.6250 - val_loss: 0.8198 - val_accuracy: 0.5000
Epoch 2/10
3/3 [==============================] - 2s 607ms/step - loss: 0.6521 - accuracy: 0.6667 - val_loss: 0.8518 - val_accuracy: 0.5000
Epoch 3/10
3/3 [==============================] - 2s 609ms/step - loss: 0.6752 - accuracy: 0.6250 - val_loss: 0.7129 - val_accuracy: 0.5000
Epoch 4/10
3/3 [==============================] - 2s 611ms/step - loss: 0.6841 - accuracy: 0.6250 - val_loss: 0.7010 - val_accuracy: 0.5000
Epoch 5/10
3/3 [==============================] - 2s 608ms/step - loss: 0.6977 - accuracy: 0.5417 - val_loss: 0.6551 - val_accuracy: 0.5000
Epoch 6/10
3/3 [==============================] - 2s 607ms/step - loss: 0.6508 - accuracy: 0.7083 - val_loss: 0.5752 - val_accuracy: 0.5000
Epoch 7/10
3/3 [==============================] - 2s 615ms/step - loss: 0.6596 - accuracy: 0.6875 - val_loss: 0.9326 - val_accuracy: 0.5000
Epoch 8/10
3/3 [==============================] - 2s 604ms/step - loss: 0.7022 - accuracy: 0.6458 - val_loss: 0.6976 - val_accuracy: 0.5000
Epoch 9/10
3/3 [==============================] - 2s 591ms/step - loss: 0.6331 - accuracy: 0.7292 - val_loss: 0.9571 - val_accuracy: 0.5000
Epoch 10/10
3/3 [==============================] - 2s 595ms/step - loss: 0.6085 - accuracy: 0.7292 - val_loss: 0.6029 - val_accuracy: 0.5000
Out[24]: <keras.callbacks.callbacks.History at 0x1ee4e3a8f08>

You are setting the training steps per epoch =50//32=1. So do you only have 50 training images? Similarly for validation you have steps = 8//32=0. Do you have only 8 validation images? When you execute the program how many images do the training and validation generators print out they have found? You will need more images than that. Try setting your batch size =1

Loss is not decreasing while training Keras Sequential Model

I'm creating a very simple 2 layer feed forward network but am finding that the loss is not updating at all. I have some ideas but I wanted to get additional feedback/guidance.
Details about the data:
X_train:
(336876, 158)
X_dev:
(42109, 158)
Y_train counts:
0 285793
1 51083
Name: default, dtype: int64
Y_dev counts:
0 35724
1 6385
Name: default, dtype: int64
And here is my model architecture:
# define the architecture of the network
model = Sequential()
model.add(Dense(50, input_dim=X_train.shape[1], init="uniform", activation="relu"))
model.add(Dense(3print("[INFO] compiling model...")
adam = Adam(lr=0.01)
model.compile(loss="binary_crossentropy", optimizer=adam,
metrics=['accuracy'])
model.fit(np.array(X_train), np.array(Y_train), epochs=12, batch_size=128, verbose=1)Dense(1, activation = 'sigmoid'))
Now, with this, my loss after the first few epochs are as follows:
Epoch 1/12
336876/336876 [==============================] - 8s - loss: 2.4441 - acc: 0.8484
Epoch 2/12
336876/336876 [==============================] - 7s - loss: 2.4441 - acc: 0.8484
Epoch 3/12
336876/336876 [==============================] - 6s - loss: 2.4441 - acc: 0.8484
Epoch 4/12
336876/336876 [==============================] - 7s - loss: 2.4441 - acc: 0.8484
Epoch 5/12
336876/336876 [==============================] - 7s - loss: 2.4441 - acc: 0.8484
Epoch 6/12
336876/336876 [==============================] - 7s - loss: 2.4441 - acc: 0.8484
Epoch 7/12
336876/336876 [==============================] - 7s - loss: 2.4441 - acc: 0.8484
Epoch 8/12
336876/336876 [==============================] - 6s - loss: 2.4441 - acc: 0.8484
Epoch 9/12
336876/336876 [==============================] - 6s - loss: 2.4441 - acc: 0.8484
And when I test the model after, my f1_score is 0. My main thought was that I may need more data but I'd still expect it to perform better than it is now on the test set. Could it be that it is overfitting? I added Dropout but no luck there either.
Any help would be much appreciated.

at first glance, I believe that your learning rate is too high. Also, please consider normalizing your data especially if different features have different ranges of values (look at Scaling). Also, please consider changing your layer activations depending on whether your labels are multi-class or not. Assuming your code is of this form (you seem to have some typos in problem description):
# define the architecture of the network
model = Sequential()
#also what is the init="uniform" argument? I did not find this in keras documentation, consider removing this.
model.add(Dense(50, input_dim=X_train.shape[1], init="uniform",
activation="relu"))
model.add(Dense(1, activation = 'sigmoid')))
#a slightly more conservative learning rate, play around with this.
adam = Adam(lr=0.0001)
model.compile(loss="binary_crossentropy", optimizer=adam,
metrics=['accuracy'])
model.fit(np.array(X_train), np.array(Y_train), epochs=12, batch_size=128,
verbose=1)
This should lead the loss to converge. If not, please consider deepening your neural net (think about how many parameters you may need).

Consider adding the classification layer before compiling your model.
model.add(Dense(1, activation = 'sigmoid'))
adam = Adam(lr=0.01)
model.compile(loss="binary_crossentropy", optimizer=adam,
metrics=['accuracy'])
model.fit(np.array(X_train), np.array(Y_train), epochs=12, batch_size=128, verbose=1)

Keras network fit: loss is 'nan', accuracy doesn't change

I try to fit keras network, but in each epoch loss is 'nan' and accuracy doesn't change... I tried to change epoch, layers count, neurons count, learning rate, optimizers, I checked nan data in datasets, normalize data by different ways, but problem was not solved. Thanks for your help.
np.random.seed(1337)
# example of input vector: [-1.459746, 0.2694708, ... 0.90043]
# example of output vector: [1, 0] or [0, 1]
model = Sequential()
model.add(Dense(1000, activation='tanh', init='normal', input_dim=503))
model.add(Dense(2, init='normal', activation='softmax'))
opt = optimizers.sgd(lr=0.01)
model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=['accuracy'])
print(model.summary())
model.fit(x_train, y_train, batch_size=1000, nb_epoch=100, verbose=1)
99804/99804 [==============================] - 5s 52us/step - loss: nan - acc: 0.4938
Epoch 1/100
99804/99804 [==============================] - 5s 49us/step - loss: nan - acc: 0.4938
Epoch 2/100
99804/99804 [==============================] - 5s 51us/step - loss: nan - acc: 0.4938
Epoch 3/100
99804/99804 [==============================] - 5s 52us/step - loss: nan - acc: 0.4938
Epoch 4/100
99804/99804 [==============================] - 5s 52us/step - loss: nan - acc: 0.4938
Epoch 5/100
99804/99804 [==============================] - 5s 51us/step - loss: nan - acc: 0.4938
...

Oh, problem has been found! After normalization, one nan neuron appeared in the input vector

First convert your output to categorical, as described in Keras documentation:
Note: when using the categorical_crossentropy loss, your targets should be in categorical format. In order to convert integer targets into categorical targets, you can use the Keras utility to_categorical:
from keras.utils import to_categorical
categorical_labels = to_categorical(int_labels, num_classes=None)

Issue with Combining LSTM and CNN? (Python, Keras)

I want to predict 8-characters license plates, so I wrote the below model in Keras:
x = Input(shape=(HEIGHT, WIDTH, CHANNELS))
base_model = InceptionV3(include_top=False, weights='imagenet', input_shape=(HEIGHT, WIDTH, CHANNELS))
base_model.trainable = False
y = base_model(x)
y = Reshape((8, 9 * 256))(y)
y = LSTM(units=20, return_sequences='true')(y)
y = Dropout(0.5)(y)
y = TimeDistributed(Dense(TOTAL_CHARS, activation="softmax", activity_regularizer=regularizers.l2(REGUL_PARAM)))(y)
y = Dropout(0.25)(y)
model = Model(input=x, output=y)
model.compile(loss="categorical_crossentropy", optimizer='rmsprop', metrics=['accuracy'])
I have about 6000 data for training which I augment them with ImageGenerator. My problem is that the loss and accuracy are approximately constant during time:
************************************************************
Epoch: 1
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6856/6869 [============================>.] - ETA: 0s - loss: 5.4525 - acc: 0.1924Epoch 00001: val_loss improved from 2.17175 to 2.15020, saving model to ./trained_model_V10.hdf5
6869/6869 [==============================] - 25s 4ms/step - loss: 5.4535 - acc: 0.1924 - val_loss: 2.1502 - val_acc: 0.2232
************************************************************
Epoch: 2
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6848/6869 [============================>.] - ETA: 0s - loss: 5.4543 - acc: 0.1959Epoch 00001: val_loss improved from 2.15020 to 2.11809, saving model to ./trained_model_V10.hdf5
6869/6869 [==============================] - 26s 4ms/step - loss: 5.4537 - acc: 0.1958 - val_loss: 2.1181 - val_acc: 0.2281
************************************************************
Epoch: 3
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6856/6869 [============================>.] - ETA: 0s - loss: 5.4284 - acc: 0.1977Epoch 00001: val_loss improved from 2.11809 to 2.09679, saving model to ./trained_model_V10.hdf5
6869/6869 [==============================] - 25s 4ms/step - loss: 5.4282 - acc: 0.1978 - val_loss: 2.0968 - val_acc: 0.2304
************************************************************
Epoch: 4
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6856/6869 [============================>.] - ETA: 0s - loss: 5.4500 - acc: 0.2004Epoch 00001: val_loss did not improve
6869/6869 [==============================] - 25s 4ms/step - loss: 5.4490 - acc: 0.2004 - val_loss: 2.1146 - val_acc: 0.2355
************************************************************
Epoch: 5
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6848/6869 [============================>.] - ETA: 0s - loss: 5.4399 - acc: 0.2006Epoch 00001: val_loss did not improve
6869/6869 [==============================] - 25s 4ms/step - loss: 5.4374 - acc: 0.2009 - val_loss: 2.1102 - val_acc: 0.2324
************************************************************
Epoch: 6
************************************************************
Train on 6869 samples, validate on 1718 samples
Epoch 1/1
6856/6869 [============================>.] - ETA: 0s - loss: 5.4636 - acc: 0.1977Epoch 00001: val_loss improved from 2.09679 to 2.09076, saving model to ./trained_model_V10.hdf5
6869/6869 [==============================] - 25s 4ms/step - loss: 5.4629 - acc: 0.1978 - val_loss: 2.0908 - val_acc: 0.2341
************************************************************
Now, I am not sure exactly the correctness of my model and I think the problem is my model. Is this the correct way to combine CNN and LSTM?
I also have tried the below mode:
REGUL_PARAM = 0
image = Input(shape=(HEIGHT, WIDTH, CHANNELS))
x = Reshape((8, HEIGHT, int(WIDTH/8), CHANNELS))(image)
y = TimeDistributed(Conv2D(16, (3, 3), activation='relu', padding='same', activity_regularizer=regularizers.l2(REGUL_PARAM)))(x)
y = TimeDistributed(MaxPooling2D((2, 2)))(y)
y = TimeDistributed(Conv2D(32, (3, 3), activation='relu', padding='same', activity_regularizer=regularizers.l2(REGUL_PARAM)))(y)
y = TimeDistributed(MaxPooling2D((2, 2)))(y)
y = TimeDistributed(Conv2D(64, (3, 3), activation='relu', padding='same', activity_regularizer=regularizers.l2(REGUL_PARAM)))(y)
y = Reshape((int(y.shape[1]), int(y.shape[4]*y.shape[3]*y.shape[2])))(y)
y = Bidirectional(LSTM(units=50, return_sequences='true'))(y)
y = TimeDistributed(Dense(64, activity_regularizer=regularizers.l2(REGUL_PARAM), activation='relu'))(y)
y = Dropout(0.25)(y)
y = TimeDistributed(Dense(TOTAL_CHARS, activity_regularizer=regularizers.l2(REGUL_PARAM), activation='softmax'))(y)
y = Dropout(0.25)(y)
model = Model(inputs=image, outputs=y)
the accuracy for this is about 70%, but the point is that I cannot overfit even on a small potion of my data.

Apparently, your model doesn't work well.
You may take a look at this code.
'''Train a recurrent convolutional network on the IMDB sentiment
classification task.
Gets to 0.8498 test accuracy after 2 epochs. 41s/epoch on K520 GPU.
'''
from __future__ import print_function
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.layers import LSTM
from keras.layers import Conv1D, MaxPooling1D
from keras.datasets import imdb
# Embedding
max_features = 20000
maxlen = 100
embedding_size = 128
# Convolution
kernel_size = 5
filters = 64
pool_size = 4
# LSTM
lstm_output_size = 70
# Training
batch_size = 30
epochs = 2
'''
Note:
batch_size is highly sensitive.
Only 2 epochs are needed as the dataset is very small.
'''
print('Loading data...')
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1))
model.add(MaxPooling1D(pool_size=pool_size))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)