callback causes ValueError - python

The codes were working fine for the past months but somehow went wrong after something I have done but I cannot restore it.
def bi_LSTM_model(X_train, y_train, X_test, y_test, num_classes, loss,batch_size=68, units=128, learning_rate=0.005,epochs=20, dropout=0.2, recurrent_dropout=0.2):
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if (logs.get('acc') > 0.90):
print("\nReached 90% accuracy so cancelling training!")
self.model.stop_training = True
callbacks = myCallback()
model = tf.keras.models.Sequential()
model.add(Masking(mask_value=0.0, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout)))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=loss,
optimizer=adamopt,
metrics=['accuracy'])
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=1,
callbacks=[callbacks])
score, acc = model.evaluate(X_test, y_test,
batch_size=batch_size)
yhat = model.predict(X_test)
return history, yhat
def duo_bi_LSTM_model(X_train, y_train, X_test, y_test, num_classes, loss,batch_size=68, units=128, learning_rate=0.005,epochs=20, dropout=0.2, recurrent_dropout=0.2):
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if (logs.get('acc') > 0.90):
print("\nReached 90% accuracy so cancelling training!")
self.model.stop_training = True
callbacks = myCallback()
model = tf.keras.models.Sequential()
model.add(Masking(mask_value=0.0, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Bidirectional(
LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout)))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=loss,
optimizer=adamopt,
metrics=['accuracy'])
history = model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=1,
callbacks=[callbacks])
score, acc = model.evaluate(X_test, y_test,
batch_size=batch_size)
yhat = model.predict(X_test)
return history, yhat
Basically, I have defined two models and whenever the second one runs, the error comes up.
BTW, I use tf.keras.backend.clear_session() between the models.
ValueError: Tensor("Adam/bidirectional/forward_lstm/kernel/m:0", shape=(), dtype=resource) must be from the same graph as Tensor("bidirectional/forward_lstm/kernel:0", shape=(), dtype=resource).
The only modifications I ever done to the codes was that I tried to bring the callback class out of the two models, and put it before them, reducing the redundancy of the code.

The problem is not the callback function. The error shows up because you pass the same optimizer to two different models, which is not possible since they are two different computational graphs.
Try to define the optimizer inside the function where you define the model before the model.compile() call.

Related

Keras CNN, High training while low testing, each time my training increase, the validation decrease

am doing a text classification, my dataset size is 16000 KB, my problem is I have 95% of training and 90% in testing, I have some values are zero in my dataset, is that effect ?.. can I increase testing ? and how?
here is my training and validation curves
here is my code
model = Sequential()
model.add(Conv1D( filters=256,kernel_size=5, activation = 'relu',input_shape=(7,1)))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(Dense(11, activation='softmax'))
model.summary()
model.compile(Adam(lr=0.001),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(X_train, y_train,
epochs=200,
verbose=True,
validation_data=(X_test, y_test),
batch_size=128)
loss, accuracy = model.evaluate(X_train, y_train, verbose=True)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy: {:.4f}".format(accuracy))

Any difference?

I have 2 pieces of code written using tensorflow.
One is this:
import tensorflow as tf
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>0.99):
print("\nReached 99% accuracy so cancelling training!")
self.model.stop_training = True
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
callbacks = myCallback()
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=10, callbacks=[callbacks])
The other one is this:
import tensorflow as tf
def train_mnist():
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>99):
print("\n Se incheie antrenamentul")
self.model.stop_training = True
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
callbacks = myCallback()
model = tf.keras.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# model fitting
history = model.fit(x_train, y_train, epochs = 10, callbacks=[callbacks])
# model fitting
return history.epoch, history.history['acc'][-1]
train_mnist()
The first one gives an accuracy of 0.99 after 3 or 4 epochs. The second one gives an accuracy of 0.91 after 10 epochs. Why? They both look the same to me. Any ideas?
They both are almost identical. I have just checked both accuracy. The only reason why your accuracy is showing different is because in 2nd code you have returned
history.history['acc'][-1]
instead of
history.history['accuracy'][-1]
Also you need to save the history of 1st code for comparision like this:
history = model.fit(x_train, y_train, epochs = 10, callbacks=[callbacks])
Also figured out that you have stopped model training outside of if condition in 1st code.
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>0.99):
print("\nReached 99% accuracy so cancelling training!")
self.model.stop_training = True
It should be like this:
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>0.99):
print("\nReached 99% accuracy so cancelling training!")
self.model.stop_training = True
Both code must be giving around 0.99% accuracy.
Since your 2nd code is not showing the exact accuracy. I am posting the whole modified code for your 2nd code
import tensorflow as tf
from os import path, getcwd, chdir
path = f"{getcwd()}/../tmp2/mnist.npz"
def train_mnist():
class myCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')>99):
print("\n Se incheie antrenamentul")
self.model.stop_training = True
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
callbacks = myCallback()
model = tf.keras.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# model fitting
history = model.fit(x_train, y_train, epochs = 10, callbacks=[callbacks])
# model fitting
return history.epoch, history.history['accuracy'][-1]
train_mnist()

How can I restart model.fit after x epochs if loss remains high?

I am having trouble sometimes getting a fit on my data, and when I restart fit (with shuffle=true) then I sometimes get a good fit.
See my previous question:
https://datascience.stackexchange.com/questions/62516/why-does-my-model-sometimes-not-learn-well-from-same-data
As a work around, I want to automatically restart the fitting process, if loss is high after x epochs. How can I achieve this?
I assume I would need to use a custom version of EarlyStopping callback? How could I differentiate between ES because of finding a low loss ( < 0.5) so training is finished, or because loss > 0.5 after x epochs so need to restart training?
Here is a simplified structure:
def train_till_good():
while not_finished:
train()
def train():
load_data()
model = VerySimpleNet2();
checkpoint = keras.callbacks.ModelCheckpoint(filepath=images_root + dataset_name + '\\CheckPoint.hdf5')
myOpt = keras.optimizers.Adam(lr=0.001,decay=0.01)
model.compile(optimizer=myOpt, loss='categorical_crossentropy', metrics=['accuracy'])
LRS = CyclicLR(base_lr=0.000005, max_lr=0.0003, step_size=200.)
tensorboard = keras.callbacks.TensorBoard(log_dir='C:\\Tensorflow', histogram_freq=0,write_graph=True, write_images=False)
ES = keras.callbacks.EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
model.fit(train_images, train_labels, shuffle=True, epochs=num_epochs,
callbacks=[checkpoint,
tensorboard,
ES,
LRS],
validation_data = (test_images, test_labels)
)
def VerySimpleNet2():
model = keras.Sequential([
keras.layers.Dense(112, activation=tf.nn.relu, input_shape=(224, 224, 3)),
keras.layers.Dropout(0.4),
keras.layers.Flatten(),
keras.layers.Dense(3, activation=tf.nn.softmax)
])
return model

Why is the accuracy and f1 score of this model the same no matter what?

After doing some data wrangling, I have some variables that are either categorical ones labeled 0 or 1, or numerical variables that have been normalized to have values ranging from 0.0 to 1.0. Then here is where I define and run the model:
def baseline_model():
# create model
model = Sequential()
model.add(Dense(8, input_dim=6, activation='relu'))
model.add(Dense(2, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
# Fit the model
estimator = KerasClassifier(build_fn=baseline_model, nb_epoch=200, batch_size=10, verbose=0)
kfold = KFold(n_splits=5, shuffle=True, random_state=seed)
results = cross_val_score(estimator, X, Y, cv=kfold, scoring="f1_macro")
print("\nF1 score: %.2f%% (%.2f%%)" % (results.mean()*100, results.std()*100))
However, no matter which features I put into X, or how much of them, the F1 Score will always be 49.79%
EDIT: I've also tested with different optimizers, but the result is still the same

Keras earlystopping: print selected epoch

Simple question. I am using Keras earlystopping in the following form:
Earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
How can I get Keras to print the selected epoch once the model has been fit? I think you have to use logs but don't quite know how.
Thanks.
Edit:
The full code is very long! Let me add a bit more than I gave. Hopefully it will help.
# Define model
def design_flexiNN(m_type, neurons, shape_timestep, shape_feature, activation, kernel_ini):
model = Sequential()
model.add(Dense(neurons, input_dim=shape_feature, activation = activation, use_bias=True, kernel_initializer=kernel_ini))
model.add(Dense(1, use_bias=True))
model.compile(loss='mae', optimizer='Adam')
return model
# fit model
def fit_flexiNN(m_type, train_X, train_y, epochs, batch_size, test_X, test_y):
history = model.fit(train_X, train_y, epochs=epochs, batch_size=batch_size, callbacks=callbacks_list, validation_data=(test_X, test_y), verbose=0, shuffle=False)
Earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')
callbacks_list = [Earlystop]
model = design_flexiNN(m_type, neurons, neurons_step, train_X_feature_shape, activation, kernel_ini);
history = fit_flexiNN(m_type, train_X, train_y, ini_epochs, batch_size, test_X, test_y)
I've been able to infer the selected epoch by doing len(history.history['val_loss']) minus 1, but that doesn't work if you have a patience above zero.
Been trying to solve this myself and realised that the len(history.history['val_loss']) method is almost correct. All you need to add is:
len(history.history['val_loss']) - patience
which should give you the epoch number for the selected model (assuming that the model didnt run for the full number of epochs).
A slightly more thorough method would be:
model_loss = history.history["val_loss"]
epoch_chosen = model_loss.index(min(model_loss)) +1
print(epoch_chosen)
Hope this helps!

Categories

Resources