I made this function that incorporates a resnet into a model. It works well, and I can save it.
My problem is that I can't load it because it needs a call function. I am not exactly sure of how to turn this into a class. The attempt is at the bottom. some pointers would be helpful.
def build_network():
inp = Input(shape=(256,256,3))
resnet = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000
)
# classifier_activation='softmax'
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation='softmax')(x)
model = tf.keras.Model(inputs=inp,outputs = x)
opt = tf.keras.optimizers.SGD(momentum=0.9)
# optimizer = 'adam',
model.compile(loss='categorical_crossentropy',
optimizer = opt,
metrics=['accuracy'])
model.summary()
return model
class Resnet(tf.keras.Model):
def __init__(self, num_classes=9):
super(Resnet, self).__init__()
self.block_1 = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000)
self.global_pool = layers.GlobalAveragePooling2D()
self.dropout = Dropout(0.3)
self.classifier = Dense(num_classes, activation = 'softmax')
def call(self, inputs):
x = self.block_1(inputs)
x = self.global_pool(x)
x = self.dropout(x)
x = self.classifier(x)
return tf.keras.Model(inputs = inputs, outputs = x)
Using the subclassing API will actually make your model unserializable (see the "Limitations section in the "What are Symbolic and Imperative APIs in TensorFlow 2.0? " blogpost):
Imperative models are also more difficult to inspect, copy, or clone.
For example, model.save(), model.get_config(), and clone_model do not work for subclassed models. Likewise, model.summary() only gives you a list of layers (and doesn’t provide information on how they’re connected, since that’s not accessible).
Edit: From Tensorflow 2.4, it is possible to pass a save_traces argument to model.save to serialize models built using the subclassing API. See https://www.tensorflow.org/guide/keras/save_and_serialize#how_savedmodel_handles_custom_objects.
Here's a simple example of how you can do this:
import tensorflow as tf
from tensorflow.keras.layers import (Dense, Dropout, GlobalAveragePooling2D,
Input)
def build_network():
inp = Input(shape=(256, 256, 3))
resnet = tf.keras.applications.ResNet152V2(include_top=False,
weights="imagenet",
input_tensor=None,
input_shape=(256, 256, 3),
pooling=None,
classes=1000)
# classifier_activation="softmax"
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation="softmax")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
# optimizer = "adam",
opt = tf.keras.optimizers.SGD(momentum=0.9)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
model.summary()
return model
if __name__ == "__main__":
model = build_network()
model.summary()
# Save
model.save("my_model.h5")
# Load
loaded_model = tf.keras.models.load_model("my_model.h5")
loaded_model.summary()
To load your saved model from your build_network function use tf.keras.models.load_model.
Related
I was able to finally get the below code, up to the keras_tuner, I mention this as I'm not exactly sure I'm even working out the class structure correctly. I'm doing all of this so as to try put in my own values for "input_size" and "output_size" for the input and output size of the deep layers.
but then I get the error:
raise TypeError('Inputs to a layer should be tensors. Got: %s' % (x,))
TypeError: Inputs to a layer should be tensors. Got: <keras_tuner.engine.hyperparameters.HyperParameters object at 0x000001650C5DCE50>
I don't know how to fix this and why it would even be an issue as everything before it doesn't throw an error.
from tensorflow import keras
from tensorflow.keras import layers, Sequential
from tensorflow.keras.layers import Dense, Dropout
import keras_tuner
from kerastuner import HyperModel
class CNNHyperModel(HyperModel):
def call_existing_code(self,units, activation, dropout, lr, layers, optimizer, loss, input_size, output_size):
model = Sequential()
model.add(Dense(units=units, input_dim=input_size, activation=activation))
for i in range(layers):
model.add(Dense(units=units, activation=activation))
if dropout:
model.add(Dropout(rate=0.25))
model.add(Dense(output_size, activation=activation))
# model.add(Dense(10, activation="softmax"))
model.compile(
optimizer=optimizer,
loss=loss,
metrics=["accuracy"],
)
return model
def build_model(test, input_size, output_size):
hp = keras_tuner.HyperParameters()
units = hp.Int("units", min_value=32, max_value=512, step=32)
activation = hp.Choice("activation", ["relu", "tanh"])
dropout = hp.Boolean("dropout")
layers = hp.Int('layers', 2, 6)
lr = hp.Float("lr", min_value=1e-4, max_value=1e-2, sampling="log")
loss = hp.Choice("loss", ['sparse_categorical_crossentropy', 'categorical_crossentropy'])
optimizer = hp.Choice("optimizer", ["adam", "RMSProp"])
# call existing model-building code with the hyperparameter values.
model = test.call_existing_code(
units=units, activation=activation, dropout=dropout, lr=lr, layers=layers, optimizer=optimizer, loss=loss, \
input_size=input_size, output_size=output_size)
return model
input_size = 11
output_size = 8
Testclass = CNNHyperModel(keras_tuner.HyperParameters())
built_model = Testclass.build_model(input_size, output_size)
tuner = keras_tuner.RandomSearch(
hypermodel=built_model,
objective="val_accuracy",
max_trials=3,
executions_per_trial=2,
overwrite=True,
directory="my_dir",
)
tuner.search_space_summary()
I want to add attention layer before transfer learning, something like below, however, in the function create_model_tl I do not suppose to pass x yet, how to write a function to create a model so later on I can call the function to create model myModel = create_model_tl(input_shape, mdl_type, weights) and pass for compile and fit?
def attention(v, k, q, d_model, num_heads, mask):
......
return output
def create_model_tl(input_shape, mdl_type, weights):
print("start creating model - transfer learning ...")
conv_layer = Conv2D(16, kernel_size=(1, 1), activation='relu', input_shape=input_shape)
x = conv_layer(x)
x = attention(x, x, x, 16, 4, None) # k, q, v, d_model, num_heads, mask
base_model = vgg16.VGG16(include_top=False, input_shape=(x.shape[1], x.shape[2], x.shape[3], weights=weights)
flat1 = Flatten()(base_model.layers[-1].output)
class1 = Dense(1024, activation='relu')(flat1)
dropout1 = Dropout(0.2)(class1)
class2 = Dense(512, activation='relu')(dropout1)
dropout2 = Dropout(0.2)(class2)
output = Dense(2, activation='softmax')(dropout2)
#output = Dense(1, activation='sigmoid')(dropout2)
model = Model(inputs=base_model.inputs, outputs=output)
return model
I'm trying to use VGG16 with some modification on it. I followed this blog post from keras.io
Here the code I'm using to create the model:
def create_model():
vgg16_model = vgg16.VGG16(weights='imagenet', include_top=False)
print('[INFO] Model loaded.')
x = vgg16_model.output
x = Flatten()(x)
x = Dense(256, activation="relu")(x)
x = Dropout(0.5)(x)
x = Dense(1, activation='linear')(x)
model = Model(inputs=vgg16_model.inputs, outputs=x)
return model
Calling the model = create_model() gives an error:
ValueError: The last dimension of the inputs to Dense should be defined. Found None.
What could be the problem?
try to pass an input_shape when you use the vgg16.VGG16
def create_model():
vgg16_model = vgg16.VGG16(input_shape=(224,224,3), weights='imagenet', include_top=False)
print('[INFO] Model loaded.')
x = vgg16_model.output
x = Flatten()(x)
x = Dense(256, activation="relu")(x)
x = Dropout(0.5)(x)
x = Dense(1, activation='linear')(x)
model = Model(inputs=vgg16_model.inputs, outputs=x)
return model
I am building a small network using some custom network boxes for each use case, It looks like this :
def top_block(dropout = None, training = None):
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
if dropout:
layer_one = tf.keras.layers.Dropout(rate = dropout)(input_1, training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout)(input_2, training = training)
return [layer_one,layer_two]
return [input_1,input_2]
def bottom_layer(input_layers):
data = tf.reduce_mean(input_layers,0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model([input_layers[0], input_layers[1]], cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
If I am trying to access this network without dropout, it's working fine :
top_ = top_block()
model = bottom_layer(top_ )
But if I am accessing with dropout, it's giving error:
top_ = top_block(dropout = 0.2, training = True)
model = bottom_layer(top_ )
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("input_72:0", shape=(None, 1, 15), dtype=float32) at layer "input_72". The following previous layers were accessed without issue: []
How to access the model with dropout layer?
How to disable training = False during evaluate? Do I need to load full model and old model weights?
Thank You!
I just realized my input is coming from intermediate layer (dropout layer), It should come directly from Input layer :
def top_block():
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
return [input_1, input_2]
def apply_dropout(layers_data, dropout_val, training):
layer_one = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[0], training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[1], training = training)
return [layer_one, layer_two]
def bottom_layer(input_layers, data):
data = tf.reduce_mean(data, 0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model(input_layers, cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
It's working now
top_ = top_block()
dropout_ = apply_dropout(top_, 0.2, True)
model = bottom_layer(top_ , dropout_)
I am trying to convert code from Keras to tensorflow, I don't have much idea about Keras api, I am a Tensorflow user, Here is Keras code :
rawmeta = layers.Input(shape=(1,), dtype="string")
emb = elmolayer()(rawmeta)
d1 = layers.Dense(256, activation='relu')(emb)
yhat = layers.Dense(31, activation='softmax', name = "output_node")(d1)
model = Model(inputs=[rawmeta], outputs=yhat)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
Where elmolayer defined as follows :
class elmolayer(Layer):
def __init__(self, **kwargs):
self.dimensions = 1024
self.trainable=True
super(elmolayer, self).__init__(**kwargs)
def build(self, input_shape):
self.elmo = hub.Module('https://tfhub.dev/google/elmo/2', trainable=self.trainable,
name="{}_module".format(self.name))
self.trainable_weights += K.tf.trainable_variables(scope="^{}_module/.*".format(self.name))
super(elmolayer, self).build(input_shape)
def call(self, x, mask=None):
result = self.elmo(K.squeeze(K.cast(x, tf.string), axis=1),
as_dict=True,
signature='default',
)['default']
return result
def compute_mask(self, inputs, mask=None):
return K.not_equal(inputs, '--PAD--')
def compute_output_shape(self, input_shape):
return (input_shape[0], self.dimensions)
My Tensorflow implementation of this code is :
class Base_model(object):
def __init__(self, elmo_embedding_matrix):
tf.reset_default_graph()
# define placeholders
sentences = tf.placeholder(tf.int32, [None, None], name='sentences')
y_true = tf.placeholder(tf.int32, [None, None], name='labels' )
self.elmo = tf.get_variable(name="relation_embedding", shape=[elmo_embedding_matrix.shape[0],elmo_embedding_matrix.shape[1]],
initializer=tf.constant_initializer(np.array(elmo_embedding_matrix)),
trainable=True,dtype=tf.float32)
embedding_lookup = tf.nn.embedding_lookup(self.elmo,sentences)
d1 = tf.layers.dense(embedding_lookup, 256, tf.nn.relu)
y_pred = tf.layers.dense(d1, 31, tf.nn.softmax)
matches = tf.equal(tf.argmax(y_pred,1),tf.argmax(y_true,1))
acc = tf.reduce_mean(tf.cast(matches,tf.float32))
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true,logits=y_pred))
train = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cross_entropy)
My confusion is the last dense layer in keras model is :
yhat = layers.Dense(31, activation='softmax', name = "output_node")(d1)
While in tensorflow code if i am using tf.nn.softmax_cross_entropy_with_logits_v2 then should i pass second dense layer to softmax eg.,
y_pred = tf.layers.dense(d1, 31, tf.nn.softmax)
Because if i am using softmax here then tf.nn.softmax_cross_entropy_with_logits_v2 will use softmax again on logits.
How to convert that Keras code to Tensorflow?
Specifying the comment here (Answer Section) even though it is present in Comments Section, for the benefit of the Community.
The Tensorflow equivalent Code for the Keras Code to represent Output Layer,
yhat = layers.Dense(31, activation='softmax', name = "output_node")(d1)
is
y_logits = tf.layers.dense(d1, 31, tf.nn.softmax)
y_pred = tf.nn.softmax(y_logits)
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true,logits=y_logits))
Hope this helps. Happy Learning!