I am having difficulty figuring out how to pass multiple arguments through keras tuner function. I looked all over all available documentation and questions related to this and I could not find anything for this particular problem.
I just want to be able to pass additional arguments through this function:
def build_model(hp, some_val_1, some_val_2)
Overall Code (Simplified):
import kerastuner as kt
def build_model(hp, some_val_1, some_val_2):
print(some_val_1)
print(some_val_2)
conv1d_val_1 = hp.Int("1-input_units", min_value=32, max_value=1028, step=64)
conv1d_filt_1 = hp.Int("1b-filter_units", min_value=2, max_value=10, step=1)
model.add(Conv1D(conv1d_val_1, conv1d_filt_1, activation='relu', input_shape=input_shape, padding='SAME'))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
return model
model = kt.Hyperband(build_model, objective="val_loss", max_epochs = 10, factor = 3, directory=os.path.normpath(path_save_dir))
model.search(x=x_train, y=y_train, epochs=10, batch_size=500, validation_data=(x_test, y_test), shuffle=True)
Attempt #1 (I tried many variations) - Does not work:
model = kt.Hyperband(build_model(kt.HyperParameters(), some_val_1, some_val_2), objective="val_loss", max_epochs = 10, factor = 3, directory=os.path.normpath(path_save_dir))
Attempt #2 (I tried many variations) - Does not work:
model = kt.Hyperband(build_model, some_val_1='1', some_val_2='2',objective="val_loss", max_epochs = 10, factor = 3, directory=os.path.normpath(path_save_dir))
Attempt #3 (I tried many variations) - Does not work:
model = kt.Hyperband(build_model, args=(some_val_1, some_val_2,),objective="val_loss", max_epochs = 10, factor = 3, directory=os.path.normpath(path_save_dir))
Please send help
You can create your own HyperModel subclass to do achieve this, check this link.
Example implementation, which will do what you are trying to do :-
import kerastuner as kt
class MyHyperModel(kt.HyperModel):
def __init__(self, some_val_1, some_val_2):
self.some_val_1 = some_val_1
self.some_val_2 = some_val_2
def build(self, hp):
## You can use self.some_val_1 and self.some_val_2 here
conv1d_val_1 = hp.Int("1-input_units", min_value=32, max_value=1028, step=64)
conv1d_filt_1 = hp.Int("1b-filter_units", min_value=2, max_value=10, step=1)
model.add(Conv1D(conv1d_val_1, conv1d_filt_1, activation='relu', input_shape=input_shape, padding='SAME'))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
return model
some_val_1 = 10
some_val_2 = 20
my_hyper_model = MyHyperModel(some_val_1 = some_val_1, some_val_2 = some_val_2)
model = kt.Hyperband(my_hyper_model, objective="val_loss", max_epochs = 10,
factor = 3, directory=os.path.normpath(path_save_dir))
Adding a complete example with the HyperModel tuned (I use input_shape and output_shape for some_val_1 and some_val_2).
## The hypermodel
class MyHyperModel(keras_tuner.HyperModel):
def __init__(self, input_shape, output_shape):
self.input_shape = input_shape
self.output_shape = output_shape
def build(self, hp):
model = keras.Sequential()
model.add(keras.Input(shape=(self.input_shape,)))
model.add(
layers.Dense(
units=hp.Int("units", min_value=32, max_value=64, step=32),
activation="relu"
)
) # tuning number of layers
model.add(layers.Dense(self.output_shape, activation='softmax'))
model.compile(loss='categorical_crossentropy', metrics=['accuracy'])
return model
## The tuner
tuner = keras_tuner.RandomSearch(
hypermodel=CustomHyperModel(input_shape, output_shape),
objective='val_accuracy',
max_trials=3,
overwrite=True
)
tuner.search(X_train, y_train, epochs=3, validation_data=(X_val, y_val))
## The final model
model = tuner.get_best_models()[0]
model.summary()
Related
I want to predict a time series using cnn-lstm model.This is my model:
def generate_model():
model = keras.models.Sequential([
Conv1D(64, 3, padding='causal', activation='relu', input_shape=(24, 20)),
BatchNormalization(),
Conv1D(64, 3, padding='causal', activation='relu'),
BatchNormalization(),
Conv1D(32, 3, padding='causal', activation='relu'),
MaxPool1D(3),
LSTM(100, dropout=0.2, return_sequences=True),
LSTM(50, dropout=0.3),
Dense(1, activation='relu')
])
model.compile(optimizer=tf.keras.optimizers.Adam(),
loss='mean_squared_error',
metrics=[tf.keras.metrics.MeanAbsoluteError(), tf.keras.metrics.RootMeanSquaredError(), RSquare()])
return model
Then I use this line of code to train my model:
history1 = model1.fit(X1_train, y1_train, epochs=200, batch_size=32, validation_data=(X1_test, y1_test), verbose=2, callbacks=callbacks)
But values of loss and metrics stays the same and does not change. This is how they look.
These are my callbacks, just in case:
from keras.callbacks import LearningRateScheduler
def decay_schedule(epoch, lr):
lr = lr - 0.0001
return lr
lr_scheduler = LearningRateScheduler(decay_schedule)
callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='max', min_delta=1e-3, patience=50)
callbacks=[lr_scheduler, callback]
Thank you in advance.
I am trying to tune hyperparameter on the KerasRegressor
However, i only get the result of NaN's which is shown below, may i know what cause the issue?
everything works fine when i try to compile my model... but the scoring for the best parameters it always show NaNs, metrics that i used is RMSE
code snippet at below:
def create_model(optimizer,activation,lstm_unit_1,lstm_unit_2,lstm_unit_3, init='glorot_uniform'):
model = Sequential()
model.add(Conv1D(lstm_unit_1, kernel_size=1, activation=activation, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(GRU(lstm_unit_2, activation = activation, return_sequences=True, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(GRU(lstm_unit_3, activation = activation, return_sequences=True, input_shape = (trainX.shape[1], trainX.shape[2])))
model.add(Dense(units = 1))
model.add(Flatten())
model.compile(optimizer = optimizer, loss = 'mse', metrics = ['mean_squared_error'])
return model
model = tf.keras.wrappers.scikit_learn.KerasRegressor(build_fn = create_model,
epochs = 150,
verbose=False)
batch_size = [16,32,64,128]
lstm_unit_1 = [128,256,512]
lstm_unit_2 = lstm_unit_1.copy()
lstm_unit_3 = lstm_unit_1.copy()
optimizer = ['SGD','Adam','Adamax','RMSprop']
activation = ['relu','linear','sigmoid',]
param_grid = dict(lstm_unit_1=lstm_unit_1,
lstm_unit_2=lstm_unit_2,
lstm_unit_3=lstm_unit_3,
optimizer=optimizer,
activation=activation,
batch_size = batch_size)
warnings.filterwarnings("ignore")
random = RandomizedSearchCV(estimator=model, param_distributions=param_grid, n_jobs=-1, scoring='neg_mean_squared_error')
random_result = random.fit(trainX,trainY)
print(random_result.best_score_)
print(random_result.best_params_)
def rnn_model(self,activation="relu"):
in_out_neurons = 50
n_hidden = 512
model = Sequential()
model.add(LSTM(n_hidden, batch_input_shape=(None, self.seq_len, in_out_neurons), return_sequences=True))
model.add(Dense(in_out_neurons, activation=activation))
optimizer = Adam(learning_rate=0.001)
model.compile(loss="mean_squared_error", optimizer=optimizer)
model.summary()
return model
# then try to fit the model
final_x = np.zeros((319083, 2, 50))
final_y = np.zeros((319083, 1, 50))
# this works.
model = self.rnn_model()
model.fit(
final_x,final_y,
batch_size=400,
epochs=10,
validation_split=0.1
)
#However, when I trid to use hyperparameter sarch, this shows the error `ValueError: Invalid shape for y: (319083, 1, 50)`
activation = ["relu","sigmoid"]
model = KerasClassifier(build_fn=self.rnn_model,verbose=0)
param_grid = dict(activation=activation)
grid = GridSearchCV(estimator=model,param_grid=param_grid)
grid_result= grid.fit(final_x,final_y)
How dimension changes when using GridSearchCV
You should be using a KerasRegressor, since your model is not a classifier in that sense:
import tensorflow as tf
import numpy as np
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasRegressor
def rnn_model(activation="relu"):
in_out_neurons = 50
n_hidden = 512
model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(n_hidden, batch_input_shape=(None, 2, in_out_neurons), return_sequences=True))
model.add(tf.keras.layers.Dense(in_out_neurons, activation=activation))
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
model.compile(loss="mean_squared_error", optimizer=optimizer)
model.summary()
return model
final_x = np.zeros((319083, 2, 50))
final_y = np.zeros((319083, 2, 50))
model = rnn_model()
activation = ["relu","sigmoid"]
model = KerasRegressor(build_fn=rnn_model,verbose=0)
param_grid = dict(activation=activation)
grid = GridSearchCV(estimator=model, param_grid=param_grid)
grid_result= grid.fit(final_x,final_y)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_)) # run with a way smaller dataset
Best: 0.000000 using {'activation': 'relu'}
I made this function that incorporates a resnet into a model. It works well, and I can save it.
My problem is that I can't load it because it needs a call function. I am not exactly sure of how to turn this into a class. The attempt is at the bottom. some pointers would be helpful.
def build_network():
inp = Input(shape=(256,256,3))
resnet = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000
)
# classifier_activation='softmax'
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation='softmax')(x)
model = tf.keras.Model(inputs=inp,outputs = x)
opt = tf.keras.optimizers.SGD(momentum=0.9)
# optimizer = 'adam',
model.compile(loss='categorical_crossentropy',
optimizer = opt,
metrics=['accuracy'])
model.summary()
return model
class Resnet(tf.keras.Model):
def __init__(self, num_classes=9):
super(Resnet, self).__init__()
self.block_1 = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000)
self.global_pool = layers.GlobalAveragePooling2D()
self.dropout = Dropout(0.3)
self.classifier = Dense(num_classes, activation = 'softmax')
def call(self, inputs):
x = self.block_1(inputs)
x = self.global_pool(x)
x = self.dropout(x)
x = self.classifier(x)
return tf.keras.Model(inputs = inputs, outputs = x)
Using the subclassing API will actually make your model unserializable (see the "Limitations section in the "What are Symbolic and Imperative APIs in TensorFlow 2.0? " blogpost):
Imperative models are also more difficult to inspect, copy, or clone.
For example, model.save(), model.get_config(), and clone_model do not work for subclassed models. Likewise, model.summary() only gives you a list of layers (and doesn’t provide information on how they’re connected, since that’s not accessible).
Edit: From Tensorflow 2.4, it is possible to pass a save_traces argument to model.save to serialize models built using the subclassing API. See https://www.tensorflow.org/guide/keras/save_and_serialize#how_savedmodel_handles_custom_objects.
Here's a simple example of how you can do this:
import tensorflow as tf
from tensorflow.keras.layers import (Dense, Dropout, GlobalAveragePooling2D,
Input)
def build_network():
inp = Input(shape=(256, 256, 3))
resnet = tf.keras.applications.ResNet152V2(include_top=False,
weights="imagenet",
input_tensor=None,
input_shape=(256, 256, 3),
pooling=None,
classes=1000)
# classifier_activation="softmax"
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation="softmax")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
# optimizer = "adam",
opt = tf.keras.optimizers.SGD(momentum=0.9)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
model.summary()
return model
if __name__ == "__main__":
model = build_network()
model.summary()
# Save
model.save("my_model.h5")
# Load
loaded_model = tf.keras.models.load_model("my_model.h5")
loaded_model.summary()
To load your saved model from your build_network function use tf.keras.models.load_model.
I have a simple Keras model:
model_2 = Sequential()
model_2.add(Dense(32, input_shape=(500,)))
model_2.add(Dense(4))
#answer = concatenate([response, question_encoded])
model_1 = Sequential()
model_1.add(LSTM(32, dropout_U = 0.2, dropout_W = 0.2, return_sequences=True, input_shape=(None, 2048)))
model_1.add(LSTM(16, dropout_U = 0.2, dropout_W = 0.2, return_sequences=False))
#model.add(LSTM(16, return_sequences=False))
merged = Merge([model_1, model_2])
model = Sequential()
model.add(merged)
model.add(Dense(8, activation='softmax'))
#model.build()
#print(model.summary(90))
print("Compiled")
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
The code fails with the error when calling fit():
raise RuntimeError('You must compile your model before using it.')
RuntimeError: You must compile your model before using it.
Clearly, I have called compile. How could I resolve the error?
It looks like the problem is that you are creating 3 instances of the Sequential model but only compile the 3rd one (the merged).
It might be easier to use a different structure for a multi-modal network:
input_2 = Input(shape=(500,))
model_2 = Dense(32)(input_2 )
model_2 = Dense(4)(model_2)
input_1 = Input(shape=(None, 2048))
model_1 = LSTM(32, dropout_U = 0.2, dropout_W = 0.2, return_sequences=True)(input_1 )
model_1 = LSTM(16, dropout_U = 0.2, dropout_W = 0.2, return_sequences=False)(model_1)
merged = concatenate([model_2, model_1])
merged = Dense(8, activation='softmax')(merged)
model = Model(inputs=[input_2 , input_1], outputs=merged)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
Hope this helps!