I'd like to do a hyperparameter-tuning on a Keras model with Keras tuner.
import tensorflow as tf
from tensorflow import keras
import keras_tuner as kt
def model_builder(hp):
model = keras.Sequential()
model.add(keras.layers.Flatten(input_shape=(28, 28)))
hp_units = hp.Int('units', min_value=32, max_value=512, step=32)
model.add(keras.layers.Dense(units=hp_units, activation='relu'))
model.add(keras.layers.Dense(10))
hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
return model
tuner = kt.Hyperband(model_builder,
objective='val_accuracy',
max_epochs=10,
factor=3)
tuner.search(train_X, train_y, epochs=50)
So far, so good. However, I additionally want to define some model parameters (like input image dimensions) as input parameters for model_builder, I'm clueless, how to done:
def model_builder(hp, img_dim1, img_dim2):
model = keras.Sequential()
model.add(keras.layers.Flatten(input_shape=(img_dim1, img_dim2)))
...
and
tuner = kt.Hyperband(model_builder(img_dim1, img_dim2),
objective='val_accuracy',
max_epochs=10,
factor=3)
seemingly doesn't work. How to feed img_dim1, img_dim2 to the model beyond hp?
A simple solution is using "partial function" in python like the following:
from functools import partial
#...
model_builder_ready = partial(model_builder, img_dim1 = value1, img_dim2 = value2)
tuner = kt.Hyperband(model_builder_ready,
objective='val_accuracy',
max_epochs=10,
factor=3)
The solution I came up with was to create a function that return a function (probably what partial does), so this should look like this :
def model_builder(img_dim1, img_dim2):
def func(hp):
"""
Your original builder but here img_dim1 and img_dim2 exist in the scope so you can use them as parameter
"""
return func
tuner = kt.Hyperband(model_builder(img_dim1, img_dim2),
objective='val_accuracy',
max_epochs=10,
factor=3)
Related
I'm trying to run multiple Keras model from functions but it gives OOM error my code looks like this :
def mainModel():
trainX, testX, trainY, testY = train_test_split()
evScoreVGG, imHolderVGG = runKerasVGGModel( train and test )
evScoreResNet, imHolderResnet = runResnetKerasModel(train and test values)
def createKerasVGG19Model():
base_model = VGG19( weights='imagenet',
include_top=False,
input_shape=(width, height,3),
)
base_model.trainable = False
model = Sequential([
base_model,
Flatten(),
Dense(164, activation="softmax"),
])
return model
#The ResNet is the same with ResNet50V2 base model
def runKerasVGGModel(trainX,testX,trainY,testY,mode):
modelKeras = createKerasVGG19Model()
modelKeras.compile( foo )
modelKeras.fit( foo )
evScore = modelKeras.evaluate(testX, testY)
predictedValue = modelKeras.predict(testX, mode)
del(modelKeras)
K.clear_session()
return evScore
Code for running and creating Resnet is the same as the VGG model without any change. I didn't add for readability.
I tried writing del(model) at the end of the runKeras() as del(model)
I tried writing K.clear_session() at the end of the runKeras()
I tried writing K.clear_session() between the runKeras1() and runKeras2() inside the mainModel
I have a set of pre-processing stages in sklearn Pipeline and an estimator which is a KerasClassifier (from tensorflow.keras.wrappers.scikit_learn import KerasClassifier).
My overall goal is to tune and log the whole sklearn pipeline in mlflow (in databricks evn). I get a confusing type error which I can't figure out how to reslove:
TypeError: can't pickle _thread.RLock objects
I have the following code (without tuning stage) which returns the above error:
conda_env = _mlflow_conda_env(
additional_conda_deps=None,
additional_pip_deps=[
"cloudpickle=={}".format(cloudpickle.__version__),
"scikit-learn=={}".format(sklearn.__version__),
"numpy=={}".format(np.__version__),
"tensorflow=={}".format(tf.__version__),
],
additional_conda_channels=None,
)
search_space = {
"estimator__dense_l1": 20,
"estimator__dense_l2": 20,
"estimator__learning_rate": 0.1,
"estimator__optimizer": "Adam",
}
def create_model(n):
model = Sequential()
model.add(Dense(int(n["estimator__dense_l1"]), activation="relu"))
model.add(Dense(int(n["estimator__dense_l2"]), activation="relu"))
model.add(Dense(1, activation="sigmoid"))
model.compile(
loss="binary_crossentropy",
optimizer=n["estimator__optimizer"],
metrics=["accuracy"],
)
return model
mlflow.sklearn.autolog()
with mlflow.start_run(nested=True) as run:
classfier = KerasClassifier(build_fn=create_model, n=search_space)
# fit the pipeline
clf = Pipeline(steps=[("preprocessor", preprocessor),
("estimator", classfier)])
h = clf.fit(
X_train,
y_train.values,
estimator__validation_split=0.2,
estimator__epochs=10,
estimator__verbose=2,
)
# log scores
acc_score = clf.score(X=X_test, y=y_test)
mlflow.log_metric("accuracy", acc_score)
signature = infer_signature(X_test, clf.predict(X_test))
# Log the model with a signature that defines the schema of the model's inputs and outputs.
mlflow.sklearn.log_model(
sk_model=clf, artifact_path="model",
signature=signature,
conda_env=conda_env
)
I also get this warning before the error:
WARNING mlflow.sklearn.utils: Truncated the value of the key `steps`. Truncated value: `[('preprocessor', ColumnTransformer(n_jobs=None, remainder='drop', sparse_threshold=0.3,
transformer_weights=None,
transformers=[('num',
Pipeline(memory=None,
note the the whole pipeline runs outside mlflow.
can someone help?
I think I find sort of a workaround/solution for this for now, but I think this issue needs to be addressed in MLFloow anyways.
What I did is not the best way probably.
I used a python package called scikeras that does this wrapping and then could log the model
The code:
import scikeras
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input, Dense, Dropout, LSTM, Flatten, Activation
from scikeras.wrappers import KerasClassifier
class ModelWrapper(mlflow.pyfunc.PythonModel):
def __init__(self, model):
self.model = model
def predict(self, context, model_input):
return self.model.predict(model_input)
conda_env = _mlflow_conda_env(
additional_conda_deps=None,
additional_pip_deps=[
"cloudpickle=={}".format(cloudpickle.__version__),
"scikit-learn=={}".format(sklearn.__version__),
"numpy=={}".format(np.__version__),
"tensorflow=={}".format(tf.__version__),
"scikeras=={}".format(scikeras.__version__),
],
additional_conda_channels=None,
)
param = {
"dense_l1": 20,
"dense_l2": 20,
"optimizer__learning_rate": 0.1,
"optimizer": "Adam",
"loss":"binary_crossentropy",
}
def create_model(dense_l1, dense_l2, meta):
n_features_in_ = meta["n_features_in_"]
X_shape_ = meta["X_shape_"]
n_classes_ = meta["n_classes_"]
model = Sequential()
model.add(Dense(n_features_in_, input_shape=X_shape_[1:], activation="relu"))
model.add(Dense(dense_l1, activation="relu"))
model.add(Dense(dense_l2, activation="relu"))
model.add(Dense(1, activation="sigmoid"))
return model
mlflow.sklearn.autolog()
with mlflow.start_run(run_name="sample_run"):
classfier = KerasClassifier(
create_model,
loss=param["loss"],
dense_l1=param["dense_l1"],
dense_l2=param["dense_l2"],
optimizer__learning_rate = param["optimizer__learning_rate"],
optimizer= param["optimizer"],
)
# fit the pipeline
clf = Pipeline(steps=[('preprocessor', preprocessor),
('estimator', classfier)])
h = clf.fit(X_train, y_train.values)
# log scores
acc_score = clf.score(X=X_test, y=y_test)
mlflow.log_metric("accuracy", acc_score)
signature = infer_signature(X_test, clf.predict(X_test))
model_nn = ModelWrapper(clf,)
mlflow.pyfunc.log_model(
python_model= model_nn,
artifact_path = "model",
signature = signature,
conda_env = conda_env
)
I made this function that incorporates a resnet into a model. It works well, and I can save it.
My problem is that I can't load it because it needs a call function. I am not exactly sure of how to turn this into a class. The attempt is at the bottom. some pointers would be helpful.
def build_network():
inp = Input(shape=(256,256,3))
resnet = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000
)
# classifier_activation='softmax'
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation='softmax')(x)
model = tf.keras.Model(inputs=inp,outputs = x)
opt = tf.keras.optimizers.SGD(momentum=0.9)
# optimizer = 'adam',
model.compile(loss='categorical_crossentropy',
optimizer = opt,
metrics=['accuracy'])
model.summary()
return model
class Resnet(tf.keras.Model):
def __init__(self, num_classes=9):
super(Resnet, self).__init__()
self.block_1 = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000)
self.global_pool = layers.GlobalAveragePooling2D()
self.dropout = Dropout(0.3)
self.classifier = Dense(num_classes, activation = 'softmax')
def call(self, inputs):
x = self.block_1(inputs)
x = self.global_pool(x)
x = self.dropout(x)
x = self.classifier(x)
return tf.keras.Model(inputs = inputs, outputs = x)
Using the subclassing API will actually make your model unserializable (see the "Limitations section in the "What are Symbolic and Imperative APIs in TensorFlow 2.0? " blogpost):
Imperative models are also more difficult to inspect, copy, or clone.
For example, model.save(), model.get_config(), and clone_model do not work for subclassed models. Likewise, model.summary() only gives you a list of layers (and doesn’t provide information on how they’re connected, since that’s not accessible).
Edit: From Tensorflow 2.4, it is possible to pass a save_traces argument to model.save to serialize models built using the subclassing API. See https://www.tensorflow.org/guide/keras/save_and_serialize#how_savedmodel_handles_custom_objects.
Here's a simple example of how you can do this:
import tensorflow as tf
from tensorflow.keras.layers import (Dense, Dropout, GlobalAveragePooling2D,
Input)
def build_network():
inp = Input(shape=(256, 256, 3))
resnet = tf.keras.applications.ResNet152V2(include_top=False,
weights="imagenet",
input_tensor=None,
input_shape=(256, 256, 3),
pooling=None,
classes=1000)
# classifier_activation="softmax"
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation="softmax")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
# optimizer = "adam",
opt = tf.keras.optimizers.SGD(momentum=0.9)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
model.summary()
return model
if __name__ == "__main__":
model = build_network()
model.summary()
# Save
model.save("my_model.h5")
# Load
loaded_model = tf.keras.models.load_model("my_model.h5")
loaded_model.summary()
To load your saved model from your build_network function use tf.keras.models.load_model.
I am trying to utilize Model.fit_generator of Keras in tensorflow 1.10.
Simplified reproducible code:
import tensorflow as tf
import numpy as np
class TestNet(tf.keras.Model):
def __init__(self, class_count, name='TestNet', **kwargs):
super(TestNet, self).__init__(name=name, **kwargs)
self.convolution = tf.keras.layers.Conv1D(class_count, kernel_size=1, input_shape=(None, 3))
def call(self, points):
return self.convolution(points)
def segmentation_loss(labels, logits):
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
return tf.reduce_mean(cross_entropy)
def generate():
while True:
yield (np.zeros(shape=(100,3)), np.zeros(shape=(100)))
if __name__ == "__main__":
test_net = TestNet(class_count=5)
optimizer = tf.keras.optimizers.Adam()
test_net.compile(optimizer, loss=segmentation_loss)
history = test_net.fit_generator(generate, steps_per_epoch=1000, epochs=10)
While this works in tensorflow 1.14, executing this in 1.10 yields the NotImplementedError in the title:
NotImplementedError: fit_generator is not yet enabled for unbuilt Model subclasses
Anybody knows how to work around this?
I see a mistake in fit_generator, update to this:
history = test_net.fit_generator(generate(), steps_per_epoch=1000, epochs=10)
Now, I don't know how Tensorflow 1.10 works, but this kind of modeling is rather new, usually a Keras model is built like this:
#model's inputs
inputs = Input((None,3))
#model's layers
convolution = Conv1D(class_count, kernel_size=1)
#model's call
outputs = convolution(inputs)
#model finish
test_net = Model(inputs, outputs)
A workaround for you:
inputs = Input(shape)
test_net = TestNet(...)
outputs = test_net.call(inputs)
test_net_model = Model(inputs, outputs)
I am quite new to machine learning and I am trying to implement my custom layer in keras. I found a couple of tutorials and it seems comparatively straight forward. What I do not understand, though, is how to implement my new custom layer in Sequential(). See for example this classification problem that I took from the tensorflow website(https://www.tensorflow.org/tutorials/keras/basic_text_classification), posted here for your convenience:
from __future__ import absolute_import, division, print_function
import tensorflow as tf
from tensorflow import keras
import numpy as np
imdb = keras.datasets.imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)
# A dictionary mapping words to an integer index
word_index = imdb.get_word_index()
# The first indices are reserved
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2 # unknown
word_index["<UNUSED>"] = 3
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
return ' '.join([reverse_word_index.get(i, '?') for i in text])
train_data = keras.preprocessing.sequence.pad_sequences(train_data,
value=word_index["<PAD>"],
padding='post',
maxlen=256)
test_data = keras.preprocessing.sequence.pad_sequences(test_data,
value=word_index["<PAD>"],
padding='post',
maxlen=256)
# input shape is the vocabulary count used for the movie reviews (10,000 words)
vocab_size = 10000
model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
x_val = train_data[:10000]
partial_x_train = train_data[10000:]
y_val = train_labels[:10000]
partial_y_train = train_labels[10000:]
history = model.fit(partial_x_train,
partial_y_train,
epochs=40,
batch_size=512,
validation_data=(x_val, y_val),
verbose=1)
results = model.evaluate(test_data, test_labels)
print(results)
Do I have to change the source code for keras.Sequential() or is there an easy way?
Furthermore, looking at the source code for the class Sequential() made me wonder: I can't figure out how functions like 'summary()','compile()', 'fit()' and 'evaluate()' can be called if those are not even provided in the source code in this class. Here is the source code for Sequential():
https://github.com/keras-team/keras/blob/a1397169ddf8595736c01fcea084c8e34e1a3884/keras/engine/sequential.py
Sequential is a Model, and not a layer.
The functions you mentioned (summary, compile, fit, evaluate) are implemented in the Model class linked here, as Sequential is a subclass of Model.
If you're writing a custom layer, you should be subclassing Layer instead, and not Model or Sequential.
You would need to implement build, call, and compute_output_shape to create your own layer.
There's a few examples on the Keras documentation:
from keras import backend as K
from keras.layers import Layer
class MyLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[1], self.output_dim),
initializer='uniform',
trainable=True)
super(MyLayer, self).build(input_shape) # Be sure to call this at the end
def call(self, x):
return K.dot(x, self.kernel)
def compute_output_shape(self, input_shape):
return (input_shape[0], self.output_dim)
To use it, import the MyLayer class from whichever file you put it in, and then add it like the default Keras layers:
from custom.layers import MyLayer
model = keras.Sequential()
model.add(MyLayer())