I have trained a keras CNN monitoring the metrics as follow:
METRICS = [
TruePositives(name='tp'),
FalsePositives(name='fp'),
TrueNegatives(name='tn'),
FalseNegatives(name='fn'),
BinaryAccuracy(name='accuracy'),
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc'),
]
and then the model.compile:
model.compile(optimizer='nadam', loss='binary_crossentropy',
metrics=METRICS)
it works perfectly and I saved my h5 model (model.h5).
Now I have downloaded the model and I would like to use it in other script importing the model with:
from keras.models import load_model
model = load_model('model.h5')
model.predict(....)
but during the running the compiler returns:
ValueError: Unknown metric function: {'class_name': 'TruePositives', 'config': {'name': 'tp', 'dtype': 'float32', 'thresholds': None}}
How I should manage this issue?
Thank you in advance
When you have custom metrics you need to follow slightly different approach.
Create model, train and save the model
Load the model with custom_objects and compile = False
Finally compile the model with the custom_objects
I am showing the approach here
import tensorflow as tf
from tensorflow import keras
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
# Custom Loss1 (for example)
##tf.function()
def customLoss1(yTrue,yPred):
return tf.reduce_mean(yTrue-yPred)
# Custom Loss2 (for example)
##tf.function()
def customLoss2(yTrue, yPred):
return tf.reduce_mean(tf.square(tf.subtract(yTrue,yPred)))
def create_model():
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy', customLoss1, customLoss2])
return model
# Create a basic model instance
model=create_model()
# Fit and evaluate model
model.fit(x_train, y_train, epochs=5)
loss, acc,loss1, loss2 = model.evaluate(x_test, y_test,verbose=1)
print("Original model, accuracy: {:5.2f}%".format(100*acc)) # Original model, accuracy: 98.11%
# saving the model
model.save('./Mymodel',save_format='tf')
# load the model
loaded_model = tf.keras.models.load_model('./Mymodel',custom_objects={'customLoss1':customLoss1,'customLoss2':customLoss2},compile=False)
# compile the model
loaded_model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy', customLoss1, customLoss2])
# loaded model also has same accuracy, metrics and loss
loss, acc,loss1, loss2 = loaded_model.evaluate(x_test, y_test,verbose=1)
print("Loaded model, accuracy: {:5.2f}%".format(100*acc)) #Loaded model, accuracy: 98.11%
custom_objects['METRICS'] = METRICS
model = load_model('model.h5', custom_objects=custom_objects)
It looks like you are playing with a tensorflow tutorial. I also used these exact metrics and had the same problem. What worked for me was to load the model with compile = False and then compile it with the custom metrics. Then you should be able to use model.predict(....) as expected.
import keras
model = keras.models.load_model('model.h5', compile = False)
METRICS = [
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc'),
]
model.compile(optimizer = keras.optimizers.Adam(learning_rate=1e-4),
loss = 'binary_crossentropy',
metrics = METRICS
)
Related
Once the training is finished, what I need is to save and convert the model to later export it, but I get the following error:
converter = tf.lite.TFLiteConverter.from_keras_model_file('models/modelo.h5')
AttributeError: type object 'TFLiteConverterV2' has no attribute 'from_keras_model_file'
to be honest I found a problem similar to this on the web but it doesn't suit my problem. Also who gives the answer is not very explicit.
here my code:
import tensorflow as tf
from tensorflow import keras
#dataset
mnist = keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
#class myCallback(tf.keras.callbacks.Callback):
# def on_epoch_end(self, epoch, logs={}):
# If you are using Tensorflow 1.x, replace 'accuracy' for 'acc' in the next line
# if(logs.get('accuracy')>0.99):
# print("\nReached 99.0% accuracy so cancelling training!")
# self.model.stop_training = True
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation='softmax')
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# Train the model
model.fit(x_train,
y_train,
epochs=25,)
# callbacks=[myCallback()])
# Evaluate the model
model.evaluate(x_test, y_test)
# Save the model
model.save('models/modelo.h5')
# Convert the model.
converter = tf.lite.TFLiteConverter.from_keras_model_file('models/modelo.h5')
tflite_model = converter.convert()
open("models/converted_mnist_model.tflite", "wb").write(tflite_model)
I'm new in the world of Tensorflow and I'm working on the simple example of mnist dataset classification. I would like to know how can I obtain other metrics (e.g precision, recall etc) in addition to accuracy and loss (and possibly to show them). Here's my code:
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.callbacks import TensorBoard
import os
#load mnist dataset
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
#create and compile the model
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation='softmax')
])
model.summary()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
#model checkpoint (only if there is an improvement)
checkpoint_path = "logs/weights-improvement-{epoch:02d}-{accuracy:.2f}.hdf5"
cp_callback = ModelCheckpoint(checkpoint_path, monitor='accuracy',save_best_only=True,verbose=1, mode='max')
#Tensorboard
NAME = "tensorboard_{}".format(int(time.time())) #name of the model with timestamp
tensorboard = TensorBoard(log_dir="logs/{}".format(NAME))
#train the model
model.fit(x_train, y_train, callbacks = [cp_callback, tensorboard], epochs=5)
#evaluate the model
model.evaluate(x_test, y_test, verbose=2)
Since I get only accuracy and loss, how can i get other metrics?
Thank you in advance, I'm sorry if it is a simple question or If was already answered somewhere.
I am adding another answer because this is the cleanest way in order to compute these metrics correctly on your test set (as of 22nd of March 2020).
The first thing you need to do is to create a custom callback, in which you send your test data:
import tensorflow as tf
from tensorflow.keras.callbacks import Callback
from sklearn.metrics import classification_report
class MetricsCallback(Callback):
def __init__(self, test_data, y_true):
# Should be the label encoding of your classes
self.y_true = y_true
self.test_data = test_data
def on_epoch_end(self, epoch, logs=None):
# Here we get the probabilities
y_pred = self.model.predict(self.test_data))
# Here we get the actual classes
y_pred = tf.argmax(y_pred,axis=1)
# Actual dictionary
report_dictionary = classification_report(self.y_true, y_pred, output_dict = True)
# Only printing the report
print(classification_report(self.y_true,y_pred,output_dict=False)
In your main, where you load your dataset and add the callbacks:
metrics_callback = MetricsCallback(test_data = my_test_data, y_true = my_y_true)
...
...
#train the model
model.fit(x_train, y_train, callbacks = [cp_callback, metrics_callback,tensorboard], epochs=5)
Starting from TensorFlow 2.X, precision and recall are both available as built-in metrics.
Therefore, you do not need to implement them by hand. In addition to this, they were removed before in Keras 2.X versions because they were misleading --- as they were being computed in a batch-wise manner, the global(true) values of precision and recall would be actually different.
You can have a look here : https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Recall
Now they have a built-in accumulator, which ensures the correct calculation of those metrics.
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy',tf.keras.metrics.Precision(),tf.keras.metrics.Recall()])
There is a list of available metrics in the Keras documentation. It includes recall, precision, etc.
For instance, recall:
model.compile('adam', loss='binary_crossentropy',
metrics=[tf.keras.metrics.Recall()])
I could not get Timbus' answer to work and I found a very interesting explanation here.
It says:
The meaning of 'accuracy' depends on the loss function. The one that corresponds to sparse_categorical_crossentropy is tf.keras.metrics.SparseCategoricalAccuracy(), not tf.metrics.Accuracy().
Which makes a lot of sense.
So what metrics you can use depend on the loss you chose. E.g. using the metric 'TruePositives' won't work in the case of SparseCategoricalAccuracy, because that loss means you're working with more than 1 class, which in turn means True Positives cannot be defined because it is only used in binary classification problems.
A loss like tf.keras.metrics.CategoricalCrossentropy() will work because it is designed with multiple classes in mind! Example:
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.callbacks import TensorBoard
import time
import os
#load mnist dataset
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
#create and compile the model
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation='softmax')
])
model.summary()
# This will work because it makes sense
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=[tf.keras.metrics.SparseCategoricalAccuracy(),
tf.keras.metrics.CategoricalCrossentropy()])
# This will not work because it isn't designed for the multiclass classification problem
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=[tf.keras.metrics.SparseCategoricalAccuracy(),
tf.keras.metrics.TruePositives()])
#model checkpoint (only if there is an improvement)
checkpoint_path = "logs/weights-improvement-{epoch:02d}-{accuracy:.2f}.hdf5"
cp_callback = ModelCheckpoint(checkpoint_path,
monitor='accuracy',
save_best_only=True,
verbose=1,
mode='max')
#Tensorboard
NAME = "tensorboard_{}".format(int(time.time())) # name of the model with timestamp
tensorboard = TensorBoard(log_dir="logs/{}".format(NAME))
#train the model
model.fit(x_train, y_train, epochs=5)
#evaluate the model
model.evaluate(x_test, y_test, verbose=2)
In my case the other 2 answers gave me shape mismatches.
For a list of supported metrics, see:
tf.keras Metrics
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy',tf.keras.metrics.Precision(),tf.keras.metrics.Recall()])
I am using this code to train FCN, I have succssfully run this code. However, I want to test new images on this trained model, can anyone help me?
#Training
from keras import optimizers
sgd = optimizers.SGD(lr=1E-2, decay=5**(-4), momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
hist1 = model.fit(X_train,y_train,
validation_data=(X_test,y_test),
batch_size=2,epochs=20,verbose=1)
for key in ['loss', 'acc', 'val_loss', 'val_acc']:
plt.plot(hist1.history[key],label=key)
plt.legend()
plt.show()
y_pred = model.predict(X_test)
y_predi = np.argmax(y_pred, axis=3)
y_testi = np.argmax(y_test, axis=3)
print(y_testi.shape,y_predi.shape)
As stated in the keras FAQs (https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model):
To save your model to your disk you can do:
model.save("my_model_file.h5")
And to load it again later or in another file for using it:
from keras.models import load_model
model = load_model("my_model_file.h5")
y_pred = model.predict(X_test)
I encountered some problems when I continued training my model and visualized the progress on tensorboard.
My question is how do I resume training from the same step without specifying any epoch manually? If possible, simply by loading the saved model, it somehow could read the global_step from the optimizer saved and continue training from there.
I have provided some codes below to reproduce similar errors.
import tensorflow as tf
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.models import load_model
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=10, callbacks=[Tensorboard()])
model.save('./final_model.h5', include_optimizer=True)
del model
model = load_model('./final_model.h5')
model.fit(x_train, y_train, epochs=10, callbacks=[Tensorboard()])
You can run the tensorboard by using the command:
tensorboard --logdir ./logs
You can set the parameter initial_epoch in the function model.fit() to the number of the epoch you want your training to start from. Take into account that the model trains until the epoch of index epochs is reached (and not a number of iterations given by epochs).
In your example, if you want to train for 10 epochs more, it should be:
model.fit(x_train, y_train, initial_epoch=9, epochs=19, callbacks=[Tensorboard()])
It will allow you to visualise your plots on Tensorboard in a correct manner.
More extensive information about these parameters can be found in the docs.
Here is sample code in case someone needs it. It implements the idea proposed by Abhinav Anand:
mca = ModelCheckpoint(join(dir, 'model_{epoch:03d}.h5'),
monitor = 'loss',
save_best_only = False)
tb = TensorBoard(log_dir = join(dir, 'logs'),
write_graph = True,
write_images = True)
files = sorted(glob(join(fold_dir, 'model_???.h5')))
if files:
model_file = files[-1]
initial_epoch = int(model_file[-6:-3])
print('Resuming using saved model %s.' % model_file)
model = load_model(model_file)
else:
model = nn.model()
initial_epoch = 0
model.fit(x_train,
y_train,
epochs = 100,
initial_epoch = initial_epoch,
callbacks = [mca, tb])
Replace nn.model() with your own function for defining the model.
It's very simple. Create checkpoints while training the model and then use those checkpoints to resume training from where you left of.
import tensorflow as tf
from tensorflow.keras.callbacks import TensorBoard
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.models import load_model
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.fit(x_train, y_train, epochs=10, callbacks=[Tensorboard()])
model.save('./final_model.h5', include_optimizer=True)
model = load_model('./final_model.h5')
callbacks = list()
tensorboard = Tensorboard()
callbacks.append(tensorboard)
file_path = "model-{epoch:02d}-{loss:.4f}.hdf5"
# now here you can create checkpoints and save according to your need
# here period is the no of epochs after which to save the model every time during training
# another option is save_weights_only, for your case it should be false
checkpoints = ModelCheckpoint(file_path, monitor='loss', verbose=1, period=1, save_weights_only=False)
callbacks.append(checkpoints)
model.fit(x_train, y_train, epochs=10, callbacks=callbacks)
After this just load the checkpoint from where you want to resume training again
model = load_model(checkpoint_of_choice)
model.fit(x_train, y_train, epochs=10, callbacks=callbacks)
And you are done.
Let me know if you have more questions about this.
I trained a model hand position classifier with Keras and I ended up saving the model with the code (model.save('model.h5') )
now i'm traying to predict an image using this model is it doable? if yes could you give me some examples please ?
PS:my data is provided as a CSV file
First of all, you have to import the saved model using load_model function.
from keras.models import load_model
model = load_model('model.h5')
Before you will predict the result for a new given input you have to invoke compile method.
classifier.compile(loss='your_loss', optimizer='your_optimizer', metrics=['your_metrics'])
After compiling, you're done to deal with new images.
from keras.preprocessing import image
test_image= image.load_img(picturePath, target_size = (img_width, img_height))
test_image = image.img_to_array(test_image)
test_image = numpy.expand_dims(test_image, axis = 0)
test_image = test_image.reshape(img_width, img_height)
result = model.predict(test_image)
when i run the code posting by #Mihai Alexandru-Ionut
# dimensions of our images
img_width, img_height = 313, 220
# load the model we saved
model = load_model('hmodel.h5')
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy','mse'])
test_image= image.load_img('/Images/1.jpg',target_size = (img_width, img_height))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
result = model.predict(test_image)
i get the following error: line 113, in _standardize_input_data
'with shape ' + str(data_shape))
ValueError: Error when checking : expected dense_1_input to have 2 dimensions, but got array with shape (1, 313, 220, 3)
could somoeone help me to fix this error
Here I am providing an example of saving a tensorflow.keras model to model_path folder under current directory. This works well with most recent tensorflow (TF2.0.0rc2). I will update this description if there is any change in near future. Follow the example code below and change data loading, shape etc.
Saving and loading entire model
import tensorflow as tf
from tensorflow import keras
mnist = tf.keras.datasets.mnist
#import data
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
# create a model
def create_model():
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])
# compile the model
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return model
# Create a basic model instance
model=create_model()
model.fit(x_train, y_train, epochs=1)
loss, acc = model.evaluate(x_test, y_test,verbose=1)
print("Original model, accuracy: {:5.2f}%".format(100*acc))
# Save entire model to a HDF5 file
model.save('./model_path/my_model.h5')
# Recreate the exact same model, including weights and optimizer.
new_model = keras.models.load_model('./model_path/my_model.h5')
loss, acc = new_model.evaluate(x_test, y_test)
print("Restored model, accuracy: {:5.2f}%".format(100*acc))
model.save approach saves everything:
The weight values
The model's architecture
The optimizer configuration
Training config (what passed to compile)
As model.save saves training config, we don't need to compile the model after restoring using keras.models.load_model