Develop Reference

How do you declare a file path in a callback method?

Currently, I am looking for ways to optimize my model (image classification for simple triangles and squares) and I've been stuck on understanding what these file paths are supposed to be referencing. Is it a path on your computer, or is it something else?
checkpoint_filepath = 'C:\tempfile'
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_accuracy',
mode='max',
save_best_only=True)
model_fit = model.fit(train_dataset,
steps_per_epoch = 5,
epochs = 30,
validation_data= validation_dataset,
callbacks=[reduce_lr, model_checkpoint_callback]
)
I've had the same issues also with Tensorboard. The code runs just fine when with the reduce_lr callback, but has issues when I add the modelCheckpoint callback.
Here is the rest of my code for reference:
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.preprocessing import image
from tensorflow.keras.optimizers import RMSprop
import tensorflow as tf
import matplotlib.pyplot as plt
import cv2
import os
import numpy as np
from keras.callbacks import ReduceLROnPlateau
from tensorflow.keras import layers
from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import ModelCheckpoint
img = image.load_img('C:/Input_DataTS/Triangles/triangle.jpg')
plt.imshow(img)
cv2.imread('C:\Input_DataTS')
train = ImageDataGenerator(rescale= 1/255)
validation = ImageDataGenerator(rescale= 1/255)
train_dataset = train.flow_from_directory('C:\Input_DataTS',
target_size= (200,200),
batch_size= 3,
class_mode = 'binary')
validation_dataset = train.flow_from_directory('C:\Validiation_DataTS',
target_size= (200,200),
batch_size= 3,
class_mode = 'binary')
model = tf.keras.models.Sequential([ tf.keras.layers.Conv2D(16,(3,3),activation = 'relu', input_shape =(200, 200, 3)),
tf.keras.layers.MaxPool2D(2,2),
#
tf.keras.layers.Conv2D(32,(3,3),activation = 'relu'),
tf.keras.layers.MaxPool2D(2,2),
#
tf.keras.layers.Conv2D(64,(3,3),activation = 'relu'),
tf.keras.layers.MaxPool2D(2,2),
#
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(32,activation = 'relu'),
tf.keras.layers.Dense(1,activation= 'sigmoid')
])
model.compile(loss= 'binary_crossentropy',
optimizer = RMSprop(lr=0.001),
metrics =['accuracy'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
patience=5, min_lr=0.001)
checkpoint_filepath = 'C:\tempfile'
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_accuracy',
mode='max',
save_best_only=True)
model_fit = model.fit(train_dataset,
steps_per_epoch = 5,
epochs = 30,
validation_data= validation_dataset,
callbacks=[reduce_lr, model_checkpoint_callback]
)
dir_path = 'C:/Testing_DataTS'
for i in os.listdir(dir_path ):
img = image.load_img(dir_path + '//' + i, target_size=(200,200))
plt.imshow(img)
plt.show()
X = image.img_to_array(img)
X = np.expand_dims(X,axis =0)
images = np.vstack([X])
val = model.predict(images)
if val == 0:
print("square")
else:
print("triangle")
Although this isn't needed for this model, I would like to learn how to do it properly for future cases. If anyone can help me with this issue, I'd greatly appreciate it. Thank you for your time!

Keep getting NaNs value for scoring when tuning on KerasRegressor

I am trying to tune hyperparameter on the KerasRegressor
However, i only get the result of NaN's which is shown below, may i know what cause the issue?
everything works fine when i try to compile my model... but the scoring for the best parameters it always show NaNs, metrics that i used is RMSE
code snippet at below:
def create_model(optimizer,activation,lstm_unit_1,lstm_unit_2,lstm_unit_3, init='glorot_uniform'):
model = Sequential()
model.add(Conv1D(lstm_unit_1, kernel_size=1, activation=activation, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(GRU(lstm_unit_2, activation = activation, return_sequences=True, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(GRU(lstm_unit_3, activation = activation, return_sequences=True, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(Dense(units = 1))
model.add(Flatten())
model.compile(optimizer = optimizer, loss = 'mse', metrics = ['mean_squared_error'])
return model
model = tf.keras.wrappers.scikit_learn.KerasRegressor(build_fn = create_model,
epochs = 150,
verbose=False)
batch_size = [16,32,64,128]
lstm_unit_1 = [128,256,512]
lstm_unit_2 = lstm_unit_1.copy()
lstm_unit_3 = lstm_unit_1.copy()
optimizer = ['SGD','Adam','Adamax','RMSprop']
activation = ['relu','linear','sigmoid',]
param_grid = dict(lstm_unit_1=lstm_unit_1,
lstm_unit_2=lstm_unit_2,
lstm_unit_3=lstm_unit_3,
optimizer=optimizer,
activation=activation,
batch_size = batch_size)
warnings.filterwarnings("ignore")
random = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_jobs=-1, scoring='neg_mean_squared_error')
random_result = random.fit(trainX,trainY)
print(random_result.best_score_)
print(random_result.best_params_)

Keras model.fit() showing loss as nan

I am trying to train my model for Instrument Detection. The output is displaying as loss: nan from the first epoch. I tried to change the loss function, activation function, and add some regularisation like Dropout, but it didn't affect the result.
Here is the code:
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, Dropout
from keras.optimizers import Adam
import pickle
import os
import numpy as np
from sklearn.model_selection import train_test_split
def one_hot_encoding(target):
Instruments = ['vio', 'pia', 'tru', 'flu']
enc_tar = np.zeros([len(target), 4])
for i in range(len(target)):
enc_tar[i][Instruments.index(target[i])] = 1
return enc_tar
def create_model_cnn(inp_shape):
classifier = Sequential()
classifier.add(Conv2D(25, kernel_size = 3, activation = 'relu', input_shape = inp_shape))
classifier.add(Conv2D(10, kernel_size = 3, activation = 'relu'))
classifier.add(Flatten())
classifier.add(Dense(4, activation = 'softmax'))
adam = Adam(0.001)
classifier.compile(optimizer = adam, loss = 'categorical_crossentropy', metrics = ['accuracy'])
return classifier
def create_model_mlp(inp_shape):
classifier = Sequential()
classifier.add(Dense(22, activation = 'softsign', input_shape = (42,)))
classifier.add(Dropout(0.25))
classifier.add(Dense(10, activation = 'softsign'))
classifier.add(Dropout(0.25))
classifier.add(Dense(4, activation = 'softmax'))
adam = Adam(0.0001)
classifier.compile(optimizer = adam, loss = 'categorical_crossentropy', metrics = ['accuracy'])
return classifier
def get_weights(classifier):
return classifier.get_weights()
def set_weights(classifier, weights):
classifier.set_weights(weights)
return classifier
def train_model(classifier, data, target, epoch = 40):
classifier.fit(data, target, epochs = epoch, validation_split=0.4, batch_size = 32, verbose = 1)
return classifier
def predict(classifier, data):
return classifier.predict(data)
if __name__ == '__main__':
#Get the data and the target
[data, target] = pickle.load(open('../input/music-features/feat_targ.pickle', 'rb'))
#if 'model.pickle' not in os.listdir():
#Generate the classifiers
cnn_classifier = create_model_cnn((6, 7, 1))
mlp_classifier = create_model_mlp((42))
# else:
# #Load the existing model (from a pickle dump)
# classifier = pickle.load(open('model.pickle', 'rb'))
tr_data, tst_data, tr_target, tst_target = train_test_split(data, target)
tr_data_lin = np.array(tr_data)
tr_data = tr_data_lin.reshape((tr_data_lin.shape[0], 6, 7, 1))
tst_data_lin = np.array(tst_data)
tst_data = tst_data_lin.reshape((tst_data_lin.shape[0], 6, 7, 1))
enc_target = one_hot_encoding(tr_target)
#print(tr_data, enc_target)
# train_model(cnn_classifier, tr_data, enc_target)
train_model(mlp_classifier, tr_data_lin, enc_target)
# pickle.dump([cnn_classifier, mlp_classifier], open('model.pickle', 'wb'))
The training data and the test data are from the pickle file where the shape is (15000, 42).

How to merge multiple sequential models in Keras Python?

I'm building a model with multiple sequential models that I need to merge before training the dataset. It seems keras.engine.topology.Merge isn't supported on Keras 2.0 anymore. I tried keras.layers.Add and keras.layers.Concatenate and it doesn't work as well.
Here's my code:
model = Sequential()
model1 = Sequential()
model1.add(Embedding(len(word_index) + 1, 300, weights = [embedding_matrix], input_length = 40, trainable = False))
model1.add(TimeDistributed(Dense(300, activation = 'relu')))
model1.add(Lambda(lambda x: K.sum(x, axis = 1), output_shape = (300, )))
model2 = Sequential()
###Same as model1###
model3 = Sequential()
model3.add(Embedding(len(word_index) + 1, 300, weights = [embedding_matrix], input_length = 40, trainable = False))
model3.add(Convolution1D(nb_filter = nb_filter, filter_length = filter_length, border_mode = 'valid', activation = 'relu', subsample_length = 1))
model3.add(GlobalMaxPooling1D())
model3.add(Dropout(0.2))
model3.add(Dense(300))
model3.add(Dropout(0.2))
model3.add(BatchNormalization())
model4 = Sequential()
###Same as model3###
model5 = Sequential()
model5.add(Embedding(len(word_index) + 1, 300, input_length = 40, dropout = 0.2))
model5.add(LSTM(300, dropout_W = 0.2, dropout_U = 0.2))
model6 = Sequential()
###Same as model5###
merged_model = Sequential()
merged_model.add(Merge([model1, model2, model3, model4, model5, model6], mode = 'concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(Dense(1))
merged_model.add(BatchNormalization())
merged_model.add(Activation('sigmoid'))
merged_model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
checkpoint = ModelCheckpoint('weights.h5', monitor = 'val_acc', save_best_only = True, verbose = 2)
merged_model.fit([x1, x2, x1, x2, x1, x2], y = y, batch_size = 384, nb_epoch = 200, verbose = 1, validation_split = 0.1, shuffle = True, callbacks = [checkpoint])
Error:
name 'Merge' is not defined
Using keras.layers.Add and keras.layers.Concatenate says cannot do it with sequential models.
What's the workaround for it?

If I were you, I would use Keras functional API in this case, at least for making the final model (i.e. merged_model). It gives you much more flexibility and let you easily define complex models:
from keras.models import Model
from keras.layers import concatenate
merged_layers = concatenate([model1.output, model2.output, model3.output,
model4.output, model5.output, model6.output])
x = BatchNormalization()(merged_layers)
x = Dense(300)(x)
x = PReLU()(x)
x = Dropout(0.2)(x)
x = Dense(1)(x)
x = BatchNormalization()(x)
out = Activation('sigmoid')(x)
merged_model = Model([model1.input, model2.input, model3.input,
model4.input, model5.input, model6.input], [out])
merged_model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
You can also do the same thing for other models you have defined. As I mentioned, functional API gives you more control over the structure of the model, so it is recommended to be used in case of creating complex models like this.

how use grid search with fit generator in keras

i want to grid search the parameter of the model with fit_generator as input in keras
i find below code in stack overflow and change it
1- but i don't understand how give the fit_generator or flow_from_directory to fit function(last line in the code)
2- how can add early stop?
thanks
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.wrappers.scikit_learn import KerasClassifier
from keras import backend as K
from sklearn.grid_search import GridSearchCV
from tqdm import tqdm # a nice pretty percentage bar for tasks. Thanks to viewer Daniel Bühler for this suggestion
import os # dealing with directories
import numpy as np # dealing with arrays
from random import shuffle # mixing up or currently ordered data that might lead our network astray in training.
num_classes = 10
# input image dimensions
img_rows, img_cols = 28, 28
input_shape = (img_rows, img_cols, 1)
def make_model(dense_layer_sizes, filters, kernel_size, pool_size):
'''Creates model comprised of 2 convolutional layers followed by dense layers
dense_layer_sizes: List of layer sizes.
This list has one number for each layer
filters: Number of convolutional filters in each convolutional layer
kernel_size: Convolutional kernel size
pool_size: Size of pooling area for max pooling
'''
model = Sequential()
model.add(Conv2D(filters, kernel_size,
padding='valid',
input_shape=input_shape))
model.add(Activation('relu'))
model.add(Conv2D(filters, kernel_size))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
model.add(Dropout(0.25))
model.add(Flatten())
for layer_size in dense_layer_sizes:
model.add(Dense(layer_size))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
return model
class KerasClassifier(KerasClassifier):
""" adds sparse matrix handling using batch generator
"""
def fit(self, x, y, **kwargs):
""" adds sparse matrix handling """
if not issparse(x):
return super().fit(x, y, **kwargs)
############ adapted from KerasClassifier.fit ######################
if self.build_fn is None:
self.model = self.__call__(**self.filter_sk_params(self.__call__))
elif not isinstance(self.build_fn, types.FunctionType):
self.model = self.build_fn(
**self.filter_sk_params(self.build_fn.__call__))
else:
self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
### fit => fit_generator
fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit_generator))
fit_args.update(kwargs)
############################################################
self.model.fit_generator(
self.get_batch(x, y, self.sk_params["batch_size"]),
samples_per_epoch=x.shape[0],
**fit_args)
return self
def get_batch(self, x, y=None, batch_size=32):
""" batch generator to enable sparse input """
index = np.arange(x.shape[0])
start = 0
while True:
if start == 0 and y is not None:
np.random.shuffle(index)
batch = index[start:start+batch_size]
if y is not None:
yield x[batch].toarray(), y[batch]
else:
yield x[batch].toarray()
start += batch_size
if start >= x.shape[0]:
start = 0
def predict_proba(self, x):
""" adds sparse matrix handling """
if not issparse(x):
return super().predict_proba(x)
preds = self.model.predict_generator(
self.get_batch(x, None, self.sk_params["batch_size"]),
val_samples=x.shape[0])
return preds
dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)
validator = GridSearchCV(my_classifier,
param_grid={'dense_layer_sizes': dense_size_candidates,
# epochs is avail for tuning even when not
# an argument to model building function
'epochs': [3, 6],
'filters': [8],
'kernel_size': [3],
'pool_size': [2]},
scoring='neg_log_loss',
n_jobs=1)
batch_size = 20
validation_datagen = ImageDataGenerator(rescale=1./255)
train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
'd:/train', # this is the target directory
target_size=(width, height), # all images will be resized to 150x150
batch_size=batch_size,
color_mode= "grayscale",
class_mode='binary',
shuffle=True
# ,save_to_dir='preview', save_prefix='cat', save_format='png'
) # since we use binary_crossentropy loss, we need binary labels
# this is a similar generator, for validation data
validation_generator = validation_datagen.flow_from_directory(
'd:/validation',
target_size=(width, height),
batch_size=batch_size,
color_mode= "grayscale",
class_mode='binary')
test_generator = test_datagen.flow_from_directory(
'd:/test',
target_size=(width, height),
batch_size=batch_size,
color_mode= "grayscale",
class_mode='binary')
validator.fit(??????

I 'm using this implementation, I hope it could help you.
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger
from keras.wrappers.scikit_learn import KerasClassifier
import types
class KerasBatchClassifier(KerasClassifier):
def fit(self, X, y, **kwargs):
# taken from keras.wrappers.scikit_learn.KerasClassifier.fit ###################################################
if self.build_fn is None:
self.model = self.__call__(**self.filter_sk_params(self.__call__))
elif not isinstance(self.build_fn, types.FunctionType) and not isinstance(self.build_fn, types.MethodType):
self.model = self.build_fn(**self.filter_sk_params(self.build_fn.__call__))
else:
self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
################################################################################################################
datagen = ImageDataGenerator(
rotation_range=45,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest'
)
if 'X_val' in kwargs and 'y_val' in kwargs:
X_val = kwargs['X_val']
y_val = kwargs['y_val']
val_gen = ImageDataGenerator(
horizontal_flip=True
)
val_flow = val_gen.flow(X_val, y_val, batch_size=32)
val_steps = len(X_val) / 32
early_stopping = EarlyStopping( patience=5, verbose=5, mode="auto")
model_checkpoint = ModelCheckpoint("results/best_weights.{epoch:02d}-{loss:.5f}.hdf5", verbose=5, save_best_only=True, mode="auto")
else:
val_flow = None
val_steps = None
early_stopping = EarlyStopping(monitor="acc", patience=3, verbose=5, mode="auto")
model_checkpoint = ModelCheckpoint("results/best_weights.{epoch:02d}-{loss:.5f}.hdf5", monitor="acc", verbose=5, save_best_only=True, mode="auto")
callbacks = [early_stopping, model_checkpoint]
epochs = self.sk_params['epochs'] if 'epochs' in self.sk_params else 100
self.__history = self.model.fit_generator(
datagen.flow(X, y, batch_size=32),
steps_per_epoch=len(X) / 32,
validation_data=val_flow,
validation_steps=val_steps,
epochs=epochs,
callbacks=callbacks
)
return self.__history
def score(self, X, y, **kwargs):
kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)
loss_name = self.model.loss
if hasattr(loss_name, '__name__'):
loss_name = loss_name.__name__
if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
y = to_categorical(y)
outputs = self.model.evaluate(X, y, **kwargs)
if type(outputs) is not list:
outputs = [outputs]
for name, output in zip(self.model.metrics_names, outputs):
if name == 'acc':
return output
raise Exception('The model is not configured to compute accuracy. '
'You should pass `metrics=["accuracy"]` to '
'the `model.compile()` method.')
#property
def history(self):
return self.__history
As you can see, it's specific to images, but you can adapt it to your specific needs.
I'm using it like this:
from sklearn.model_selection import GridSearchCV
model = KerasBatchClassifier(build_fn=create_model, epochs=epochs)
learn_rate = [0.001, 0.01, 0.1]
epsilon = [None, 1e-2, 1e-3]
dropout_rate = [0.25, 0.5]
param_grid = dict(learn_rate=learn_rate, epsilon=epsilon, dropout_rate=dropout_rate)
grid = GridSearchCV(estimator=model, param_grid=param_grid)
grid_result = grid.fit(X_train, Y_train, X_val = X_test, y_val = Y_test)

There is a class called ParameterGrid, which in GridSearchCV() makes all combinations of the parameters given for the grid search. You can store them in a list. For example:
from sklearn.model_selection import ParameterGrid
parameters = {'epochs': [32, 64, 128],
'batch_size':[24, 32, 48, 64],
}
list(ParameterGrid(parameters))
prints out
[{'batch_size': 24, 'epochs': 32},
{'batch_size': 24, 'epochs': 64},
{'batch_size': 24, 'epochs': 128},
{'batch_size': 32, 'epochs': 32},
{'batch_size': 32, 'epochs': 64},
{'batch_size': 32, 'epochs': 128},
{'batch_size': 48, 'epochs': 32},
{'batch_size': 48, 'epochs': 64},
{'batch_size': 48, 'epochs': 128},
{'batch_size': 64, 'epochs': 32},
{'batch_size': 64, 'epochs': 64},
{'batch_size': 64, 'epochs': 128}]
In a loop for this list, you can train your model with these specific combinations. At the end of every loop you can check for the val_acc and val_loss with other functions.

def create_model(learn_rate=0.01, momentum=0):
image_size = 224
input_shape = (image_size, image_size, 3)
pre_trained_model = VGG16(input_shape=input_shape, include_top=False, weights="imagenet")
last_layer = pre_trained_model.get_layer('block5_pool')
last_output = last_layer.output
# Flatten the output layer to 1 dimension
x = GlobalMaxPooling2D()(last_output)
# Add a fully connected layer with 512 hidden units and ReLU activation
x = Dense(512, activation='relu')(x)
# Add a dropout rate of 0.5
x = Dropout(0.5)(x)
# Add a final sigmoid layer for classification
x = layers.Dense(1, activation='sigmoid')(x)
model = Model(pre_trained_model.input, x)
model.compile(loss='binary_crossentropy',
optimizer=optimizers.SGD(lr=learn_rate, momentum=momentum),
metrics=['accuracy'])
return model
learn_rate = [1e-9, 1e-3]
momentum = [0.6, 0.9]
def try_fit(learn_rate,momentum):
history_page=[]
for lr in learn_rate:
for moment in momentum:
model = create_model(lr,moment)
history = model.fit_generator(
train_generator,
epochs=epochs,
validation_data=validation_generator,
validation_steps=total_validate//batch_size,
steps_per_epoch=total_train//batch_size)
history_page.append(history)
return history_page
history_page = try_fit(learn_rate,momentum)
history_page[0].history['accuracy']
I think you can try this way