I have built a simple CNN using tensorflow-gpu 2.4.0 for cifar10
# Set constent
num_classes = len(label_names) # The number of classes
input_shape = (32, 32, 3) # The input shape 32 * 32 pixels, 3 RGB brightness for each pixel
# Construct a sequential model
model = Sequential()
# Starting add layers to the model
# First VGG blockes
model.add(Conv2D(16, (3, 3), padding='same', activation='relu', input_shape=input_shape))
model.add(BatchNormalization()) # BatchNormalization after convolutional layer
model.add(Conv2D(16, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization()) # BatchNormalization after convolutional layer
model.add(MaxPooling2D((2, 2))) # MaxPooling2D to keep important features as well as shirnk info size
model.add(Dropout(0.1)) # Drop some portion of nodes to increase generality of model
# Second VGG blockes
model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(32, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.2))
# Third VGG blockes
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.3))
# Fourth VGG blockes
model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(128, (3, 3), padding='same', activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D((2, 2)))
model.add(Dropout(0.4))
# Flat and dense layer
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
# Final softmax layer
model.add(Dense(num_classes, activation='softmax'))
# The optimizer gives the learning rate lr and some other parameters.
opt = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
# Model complie
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
And while I am training this model:
#Train model
epochs = 100
batch_size = 128
# create data generator
datagen = ImageDataGenerator(horizontal_flip=True,
width_shift_range=5,
height_shift_range=5,
rotation_range=20)
history = model.fit(datagen.flow(x_training, y_training, batch_size=batch_size),
epochs=epochs,
validation_data=(x_validate, y_validate),
verbose=1,
shuffle=True)
The RAM of my computer keeps grow until the training process crashes. Adding python garbage collector for at end of each epoch did not work. Disabling eager detection did not work.
I using
tensorflow-gpu 2.4.0
CUDA 11.0
cuDNN 8.0
Related
when fitting the CNN model,
training accuracy = 68%
validation accuracy = 63%
but when I try to evaluate the same CNN model with the same validation data and training data the results are,
Training Loss 1.86870 Training Accuracy = 36.16%
Val Loss 1.89060 val Accuracy = 36.54%
Testing Loss 1.86273 Testing Accuracy = 36.36%
There is a big gap between these accuracies.
What can be the reason behind this?
The CNN model:
import tensorflow as tf
from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Activation ,Dropout, Normalization, AveragePooling2D, BatchNormalization
from tensorflow.keras.optimizers import Adam, Adamax
from tensorflow.python.keras import regularizers
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(height, width, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Conv2D(32, (3, 3), input_shape=(height, width, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Conv2D(32, (3, 3), input_shape=(height, width, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Conv2D(32, (3, 3), input_shape=(height, width, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Conv2D(64, (3, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Dropout(.2))
model.add(Conv2D(64, (3, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Dropout(.2))
model.add(Conv2D(64, (3, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Dropout(.2))
model.add(Conv2D(64, (3, 3), padding='same', activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), padding='same', strides=(2)))
model.add(Flatten()) # this converts our 3D feature maps to 1D feature vectors
model.add(Dense(256, activation="relu"))
model.add(BatchNormalization())
model.add(Dropout(.5))
model.add(Dense(126, activation="relu"))
model.add(BatchNormalization())
model.add(Dropout(.5))
model.add(Dense(4,activation='softmax'))
Adam(learning_rate=0.001, name='Adam')
model.compile(optimizer = 'Adam',loss = 'categorical_crossentropy',metrics = ['accuracy'])
epochs = 50
from tensorflow.keras import callbacks
import time
import keras
from keras.callbacks import EarlyStopping
es_callback = keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
datagen = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
featurewise_std_normalization=True,
samplewise_std_normalization=True
)
checkpoint = callbacks.ModelCheckpoint(
filepath='/content/drive/MyDrive/model.{epoch:02d}-{accuracy:.2f}-{val_accuracy:.2f}.h5',
monitor='val_accuracy',
verbose=1,
save_best_only=True,
mode='auto'
)
datagen.fit(X_train)
history5 = model.fit(datagen.flow(X_train,y_train, batch_size=batch_size),
epochs = epochs, validation_data = datagen.flow(X_val,y_val, batch_size=batch_size),
callbacks=[ checkpoint]
)
stop = time.time()
Here, in the fit function, I have augmented training data as well as validation data, which is not correct.
Data augmentation is used to expand the training set and generate more diverse images. It should apply only to training data. Test data and validation data must not be touched.
I'm working on a binary classification problem. I was getting 69% accuracy at first, but kept running out of memory so I shrunk certain parameters, now it's coming up 0. Any idea whats going on?
model = Sequential()
from keras.layers import Dropout
model.add(Conv2D(96, kernel_size=11, padding="same", input_shape=(300, 300, 1), activation = 'relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))
model.add(Conv2D(128, kernel_size=3, padding="same", activation = 'relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(128, kernel_size=3, padding="same", activation = 'relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
from keras.layers.core import Activation
model.add(Flatten())
# model.add(Dense(units=1000, activation='relu' ))
model.add(Dense(units= 300, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
model.add(Activation("softmax"))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
from keras.preprocessing.image import ImageDataGenerator
datagen = ImageDataGenerator(
featurewise_center=True,
rotation_range=90,
fill_mode='nearest',
validation_split = 0.2
)
datagen.fit(train)
train_generator = datagen.flow(train, train_labels, batch_size=8)
# # fits the model on batches with real-time data augmentation:
history = model.fit_generator(generator=train_generator,
use_multiprocessing=True,
steps_per_epoch = len(train_generator) / 8,
epochs = 5,
workers=20)
Softmax should only be used if you have a multiclass classification problem. You have a single output from your Dense layer, so you should use sigmoid.
I am trying to make an emotion classifier using face expressions with FER2013 dataset. It contains of 35887 samples with 2304 features each and an integer label 0-6 for emotions. When I was using Conv1D with shape (2304,1) then it achieved training accuracy of ~86% but wasn't doing well on any unseen test image. So I thought of reshaping it to (48,48,1) for each sample and using Conv2D on it. But now it just gets stuck on 0.2505 while training after the 2nd epoch and never increases. What's happening?
import pandas as pd
import numpy as np
from PIL import Image
import matplotlib.image as mpimg
from skimage import transform
import random
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
emotion = {0 : 'Angry', 1 : 'Disgust',2 : 'Fear',3 : 'Happy',
4 : 'Sad',5 : 'Surprise',6 : 'Neutral'}
df=pd.read_csv('fer.csv')
faces=df.values[:,1]
faces=faces.tolist()
emos=df.values[:,0]
for i in range(len(faces)):
faces[i]=[int(x) for x in faces[i].split()]
emos[i]=int(emos[i])
faces=np.array(faces)
faces=transform.resize(faces, (35887,48,48))
faces=np.expand_dims(faces, axis=3)
model = Sequential()
model.add(Conv2D(48, (3,3), padding='same', input_shape=(48,48,1), activation='relu'))
model.add(Conv2D(48, (3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(96, (3,3), padding='same', activation='relu'))
model.add(Conv2D(96, (3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(192, (3,3), padding='same', activation='relu'))
model.add(Conv2D(192, (3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(384, (3,3), padding='same', activation='relu'))
model.add(Conv2D(384, (3,3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(384, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(192, activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(96, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(faces,emos,epochs=100,batch_size=48)
model.save_weights('model.h5')
Model Accuracy Curves
Model Loss Curves
Normalizing the output batch after each layer fixes the issue.
Just add
model.add(BatchNormalization())
after every layer.
EDIT :
Thought that I should add more information here.
So, this was the final model I ended up making.
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(200,200,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.25))
model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(256, (3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.3))
model.add(Dense(256, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.4))
model.add(Dense(128, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(len(classes), activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='nadam',
metrics=['accuracy'])
And these were the results I got with it.
I increased the nodes and changed the optimizer too but it was the batch normalisation which gave a dramatic increase in accuracy. Using more nodes and nadam optimizer further helped a bit.
I want to predict numerical Y variable using time-series image data.
My dataset has these dimensions:
X: (7000, 3, 128, 128)
Y: (7000, 1)
7000 is the number of samples, each sample is 128 x 128 image and 3 is the number of channels (RGB). The range of the Y variable is 0-1.
Therefore, I want to apply an LSTM model after a CNN. But if I feed the CNN features to LSTM, so many errors occurs. Please help me.
This is my code:
size = 128
model = Sequential()
model.add(Convolution2D(32, (3, 3), input_shape=(3, size, size)))
model.add(Activation('relu'))
model.add(Convolution2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(Convolution2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
#### LSTM #####
model.add(LSTM(50)) ???????????????????????????
model.add(Flatten())
model.add(Dense(32))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='mse', optimizer= 'adam')
return model
I've used Floyd hub to to train the following model and saved it
# Create the model
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(3, 32, 32), activation='relu', padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(64, (3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
model.add(Dropout(0.2))
model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(1024, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
epochs = 50
adammax = keras.optimizers.Adamax(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(loss='categorical_crossentropy', optimizer=adammax, metrics=['accuracy'])
print(model.summary())
When I try to load it on my PC, it's working fine. But when I load it on the Raspberry Pi I get the following error. I tried also to save just the weights and load them, but it didn't work and I got the same error. I am using the same version of Tensorflow as Floyd hub on the Raspberry Pi.
As mentioned above, you're passing T=DT_INT64, while that is not one of the supported kernels for this op. You could see if the int64 version is just not shipped in the .so file, write the op kernel yourself, or try casting to tf.int32 right before this op in the python code. The last one worked well for me.