I have a Keras functional model defined as:
# Construct DNN
spec_input = keras.layers.Input(shape=(1, ctx, fft), name='spec')
x = keras.layers.Flatten(data_format)(spec_input)
for layer in range(len(args.dnn_struct)):
x = Dense(args.dnn_struct[layer])(x)
x = BatchNormalization()(x)
x = keras.layers.ReLU()(x)
out = Dense(fft, activation="sigmoid", name=f'spp')(x)
model = Model(inputs=spec_input, outputs=out)
I would like to get the output of each layer in the model for a given input and the answers given in Keras, How to get the output of each layer? do not work for a functional model. I am currently using Tensorflow 1.14
When I try using
from keras import backend as K
inp = model.input # input placeholder
outputs = [layer.output for layer in model.layers] # all layer outputs
functor = K.function([inp, K.learning_phase()], outputs ) # evaluation function
# Testing
test = np.random.random(input_shape)[np.newaxis,...]
layer_outs = functor([test, 1.])
print(layer_outs)
I get the error
Traceback (most recent call last):
File "/home/xyz/anaconda3/envs/tf/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1446, in __init__
session._session, options_ptr)
tensorflow.python.framework.errors_impl.InvalidArgumentError: spec:0 is both fed and fetched.
UPDATE: You cannot fetch outputs of all layers, because "all layers" includes Input - and the error message is self-explanatory. Use:
outputs = get_all_outputs(model, input_data, 1)
Below should work for any model, Model or Sequential:
def get_all_outputs(model, input_data, learning_phase=1):
outputs = [layer.output for layer in model.layers[1:]] # exclude Input
layers_fn = K.function([model.input, K.learning_phase()], outputs)
return layers_fn([input_data, learning_phase])
Layer-level solutions:
def get_layer_outputs(model, layer_name, input_data, learning_phase=1):
outputs = [layer.output for layer in model.layers if layer_name in layer.name]
layers_fn = K.function([model.input, K.learning_phase()], outputs)
return layers_fn([input_data, learning_phase])
# or, for passing in a layer directly
def get_layer_outputs(model, layer, input_data, learning_phase=1):
layer_fn = K.function([model.input, K.learning_phase()], layer.output)
return layer_fn([input_data, learning_phase])
Related
I'm having troubles making the Preprocessing layers and the Keras Tuner cooperate.
I am referring to this tutorial Load CSV data | TensorFlow Core for the Preprocessing part, and to the Getting started with KerasTuner documentation for the Keras Tuner part.
Briefly, here's the code.
He loads the data:
titanic = pd.read_csv("https://storage.googleapis.com/tf-datasets/titanic/train.csv")
titanic_features = titanic.copy()
titanic_labels = titanic_features.pop('survived')
He creates the symbolic tensors of the features in a dictionary
inputs = {}
for name, column in titanic_features.items():
dtype = column.dtype
if dtype == object:
dtype = tf.string
else:
dtype = tf.float32
inputs[name] = tf.keras.Input(shape=(1,), name=name, dtype=dtype)
inputs
Then he applies normalization on the numerical features:
numeric_inputs = {name:input for name,input in inputs.items()
if input.dtype==tf.float32}
x = layers.Concatenate()(list(numeric_inputs.values()))
norm = layers.Normalization()
norm.adapt(np.array(titanic[numeric_inputs.keys()]))
all_numeric_inputs = norm(x)
all_numeric_inputs
He creates a list
preprocessed_inputs = [all_numeric_inputs]
He one hot encodes the categorical features:
for name, input in inputs.items():
if input.dtype == tf.float32:
continue
lookup = layers.StringLookup(vocabulary=np.unique(titanic_features[name]))
one_hot = layers.CategoryEncoding(num_tokens=lookup.vocabulary_size())
x = lookup(input)
x = one_hot(x)
preprocessed_inputs.append(x)
and then He concatenates it:
preprocessed_inputs_cat = layers.Concatenate()(preprocessed_inputs)
titanic_preprocessing = tf.keras.Model(inputs, preprocessed_inputs_cat)
Now what I want to do is insert this preprocessing part in KerasTuner. I tried this:
def titanic_model(units, activation):
model_inputs = tf.keras.Input(shape=28)
dense_1 = layers.Dense(units=units, activation=activation)(model_inputs)
dense_output = layers.Dense(1)(dense_1)
body = tf.keras.Model(inputs = model_inputs, outputs = dense_output)
return body
def build_model(hp,preprocessing_head, inputs):
units = hp.Int("units", min_value=32, max_value=512, step=32)
activation = hp.Choice("activation", ["relu", "tanh"])
preprocessed_inputs = preprocessing_head(inputs)
result = titanic_model(units,
activation)(preprocessed_inputs)
model = tf.keras.Model(inputs, result)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(),
metrics = ['accuracy'])
return model
titanic_model = build_model(keras_tuner.HyperParameters(),titanic_preprocessing, inputs)
but it gives me the following error:
Inputs to a layer should be tensors. Got: <keras_tuner.engine.hyperparameters.HyperParameters
object at 0x7ff52844da30>
I cannot understand if I am close to the solution, or this is not the right way to proceed.
However, the workaround i found was to insert directly in the build_model function the preprocessing layer (titanic_preprocessing) and the inputs dictionary, without passing it as an argument of the function.
Hence:
def titanic_model(units, activation):
model_inputs = tf.keras.Input(shape=28)
dense_1 = layers.Dense(units=units, activation=activation)(model_inputs)
dense_output = layers.Dense(1)(dense_1)
body = tf.keras.Model(inputs = model_inputs, outputs = dense_output)
return body
def build_model(hp):
units = hp.Int("units", min_value=32, max_value=512, step=32)
activation = hp.Choice("activation", ["relu", "tanh"])
preprocessed_inputs = titanic_preprocessing(inputs)
result = titanic_model(units,
activation)(preprocessed_inputs)
model = tf.keras.Model(inputs, result)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(),
metrics = ['accuracy'])
return model
titanic_model = build_model()
In this case it seems to work, and by setting the tuner
tuner = keras_tuner.RandomSearch(
hypermodel = build_model,
objective=keras_tuner.Objective("accuracy", direction="max"),
max_trials = 1,
overwrite = True,
directory = "tuner_dir",
project_name = "regression_tuner")
and searching it works:
tuner.search(x=titanic_features_dict, y=titanic_labels, epochs=10)
However, I have doubts about this solution and would appreciate feedback on this.
Thank you!
Just starting on tensorflow
Working on imdb dataset. Process: Text encoding using textvectorization layer and passing it to embedded layer:
# Create a custom standardization function to strip HTML break tags '<br />'.
def custom_standardization(input_data):
lowercase = tf.strings.lower(input_data)
stripped_html = tf.strings.regex_replace(lowercase, '<br />', ' ')
return tf.strings.regex_replace(stripped_html,
'[%s]' % re.escape(string.punctuation), '')
# Vocabulary size and number of words in a sequence.
vocab_size = 10000
sequence_length = 100
# Use the text vectorization layer to normalize, split, and map strings to
# integers. Note that the layer uses the custom standardization defined above.
# Set maximum_sequence length as all samples are not of the same length.
vectorize_layer = TextVectorization(
standardize=custom_standardization,
max_tokens=vocab_size,
output_mode='int',
output_sequence_length=sequence_length)
# Make a text-only dataset (no labels) and call adapt to build the vocabulary.
text_ds = train_ds.map(lambda x, y: x)
vectorize_layer.adapt(text_ds)
I then try to build a functional API:
embedding_dim=16
text_model_catprocess2 = vectorize_layer
text_model_embedd = tf.keras.layers.Embedding(vocab_size, embedding_dim, name = 'embedding')(text_model_catprocess2)
text_model_embed_proc = tf.keras.layers.Lambda(embedding_mean_standard)(text_model_embedd)
text_model_dense1 = tf.keras.layers.Dense(2, activation = 'relu')(text_model_embed_proc)
text_model_dense2 = tf.keras.layers.Dense(2, activation = 'relu')(text_model_dense1)
text_model_output = tf.keras.layers.Dense(1, activation = 'sigmoid')(text_model_dense2)
However, this is giving the following error:
~\anaconda3\lib\site-packages\keras\backend.py in dtype(x)
1496
1497 """
-> 1498 return x.dtype.base_dtype.name
1499
1500
AttributeError: Exception encountered when calling layer "embedding" (type Embedding).
'str' object has no attribute 'base_dtype'
Call arguments received:
• inputs=<keras.layers.preprocessing.text_vectorization.TextVectorization object at 0x0000029B483AADC0>
Upon making a sequential API like this, it is working fine:
embedding_dim=16
modelcheck = tf.keras.Sequential([
vectorize_layer,
tf.keras.layers.Embedding(vocab_size, embedding_dim, name="embedding"),
tf.keras.layers.GlobalAveragePooling1D(),
tf.keras.layers.Dense(16, activation='relu'),
tf.keras.layers.Dense(1)
])
I am not sure why this is happening. Is it necessary for the functional API to have an input? Please help!
You have two options. Either you use a Sequential model and it will work as you have confirmed because you do not have to define an Input layer, or you use the functional API where you have to define an Input layer:
embedding_dim = 16
text_model_input = tf.keras.layers.Input(dtype=tf.string, shape=(1,))
text_model_catprocess2 = vectorize_layer(text_model_input)
text_model_embedd = tf.keras.layers.Embedding(vocab_size, embedding_dim, name = 'embedding')(text_model_catprocess2)
text_model_embed_proc = tf.keras.layers.Lambda(embedding_mean_standard)(text_model_embedd)
text_model_dense1 = tf.keras.layers.Dense(2, activation = 'relu')(text_model_embed_proc)
text_model_dense2 = tf.keras.layers.Dense(2, activation = 'relu')(text_model_dense1)
text_model_output = tf.keras.layers.Dense(1, activation = 'sigmoid')(text_model_dense2)
model = tf.keras.Model(text_model_input, text_model_output)
I made this function that incorporates a resnet into a model. It works well, and I can save it.
My problem is that I can't load it because it needs a call function. I am not exactly sure of how to turn this into a class. The attempt is at the bottom. some pointers would be helpful.
def build_network():
inp = Input(shape=(256,256,3))
resnet = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000
)
# classifier_activation='softmax'
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation='softmax')(x)
model = tf.keras.Model(inputs=inp,outputs = x)
opt = tf.keras.optimizers.SGD(momentum=0.9)
# optimizer = 'adam',
model.compile(loss='categorical_crossentropy',
optimizer = opt,
metrics=['accuracy'])
model.summary()
return model
class Resnet(tf.keras.Model):
def __init__(self, num_classes=9):
super(Resnet, self).__init__()
self.block_1 = tf.keras.applications.ResNet152V2(
include_top=False, weights='imagenet', input_tensor=None,
input_shape=(256,256,3), pooling=None, classes=1000)
self.global_pool = layers.GlobalAveragePooling2D()
self.dropout = Dropout(0.3)
self.classifier = Dense(num_classes, activation = 'softmax')
def call(self, inputs):
x = self.block_1(inputs)
x = self.global_pool(x)
x = self.dropout(x)
x = self.classifier(x)
return tf.keras.Model(inputs = inputs, outputs = x)
Using the subclassing API will actually make your model unserializable (see the "Limitations section in the "What are Symbolic and Imperative APIs in TensorFlow 2.0? " blogpost):
Imperative models are also more difficult to inspect, copy, or clone.
For example, model.save(), model.get_config(), and clone_model do not work for subclassed models. Likewise, model.summary() only gives you a list of layers (and doesn’t provide information on how they’re connected, since that’s not accessible).
Edit: From Tensorflow 2.4, it is possible to pass a save_traces argument to model.save to serialize models built using the subclassing API. See https://www.tensorflow.org/guide/keras/save_and_serialize#how_savedmodel_handles_custom_objects.
Here's a simple example of how you can do this:
import tensorflow as tf
from tensorflow.keras.layers import (Dense, Dropout, GlobalAveragePooling2D,
Input)
def build_network():
inp = Input(shape=(256, 256, 3))
resnet = tf.keras.applications.ResNet152V2(include_top=False,
weights="imagenet",
input_tensor=None,
input_shape=(256, 256, 3),
pooling=None,
classes=1000)
# classifier_activation="softmax"
x = resnet(inp)
x = GlobalAveragePooling2D()(x)
x = Dropout(0.3)(x)
x = Dense(9, activation="softmax")(x)
model = tf.keras.Model(inputs=inp, outputs=x)
# optimizer = "adam",
opt = tf.keras.optimizers.SGD(momentum=0.9)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
model.summary()
return model
if __name__ == "__main__":
model = build_network()
model.summary()
# Save
model.save("my_model.h5")
# Load
loaded_model = tf.keras.models.load_model("my_model.h5")
loaded_model.summary()
To load your saved model from your build_network function use tf.keras.models.load_model.
I am building a small network using some custom network boxes for each use case, It looks like this :
def top_block(dropout = None, training = None):
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
if dropout:
layer_one = tf.keras.layers.Dropout(rate = dropout)(input_1, training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout)(input_2, training = training)
return [layer_one,layer_two]
return [input_1,input_2]
def bottom_layer(input_layers):
data = tf.reduce_mean(input_layers,0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model([input_layers[0], input_layers[1]], cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
If I am trying to access this network without dropout, it's working fine :
top_ = top_block()
model = bottom_layer(top_ )
But if I am accessing with dropout, it's giving error:
top_ = top_block(dropout = 0.2, training = True)
model = bottom_layer(top_ )
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("input_72:0", shape=(None, 1, 15), dtype=float32) at layer "input_72". The following previous layers were accessed without issue: []
How to access the model with dropout layer?
How to disable training = False during evaluate? Do I need to load full model and old model weights?
Thank You!
I just realized my input is coming from intermediate layer (dropout layer), It should come directly from Input layer :
def top_block():
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
return [input_1, input_2]
def apply_dropout(layers_data, dropout_val, training):
layer_one = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[0], training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[1], training = training)
return [layer_one, layer_two]
def bottom_layer(input_layers, data):
data = tf.reduce_mean(data, 0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model(input_layers, cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
It's working now
top_ = top_block()
dropout_ = apply_dropout(top_, 0.2, True)
model = bottom_layer(top_ , dropout_)
I'm trying to use a Lambda layer to wrap a function ('get_reconstruction_loss') that combines two layers in a way so that it calculates the MSE of the results of both. Unfortunately, I can't instantiate the model due to the error shown below.
I'm thankful for any hints!
The code is based on the work of https://github.com/rajatkb/Deep-Super-Resolution-Research
Code:
import cv2
from keras import Model
from keras import backend as K
from keras.applications.vgg16 import VGG16
from keras.layers import Conv2D, Input, Lambda
import numpy as np
class MyClass:
# Source: https://github.com/rajatkb/Deep-Super-Resolution-Research
def __init__(self, img_size, channels, is_train):
# Var definition
self.lambda_content = 1
loss_layer = 'block2_conv2'
##############
### define Model here ###
model_inp = Input(shape = (img_size , img_size , channels) , name='input_layer')
model_output = Conv2D(filters = 64, kernel_size = (9,9),padding ='same', activation ='relu', kernel_initializer= 'RandomNormal' )(model_inp)
model_output = Conv2D(filters = 32, kernel_size = (1,1),padding ='same', activation ='relu', kernel_initializer= 'RandomNormal' )(model_output)
model_output = Conv2D(filters = 3, kernel_size = (5,5),padding ='same', activation ='linear', kernel_initializer= 'RandomNormal', name = 'model_output')(model_output)
self.inference_model = Model(inputs=model_inp, outputs=model_output)
##############
if is_train:
vgg_inp = Input(shape =(img_size, img_size, channels), name='vgg_net_input')
vgg = VGG16(input_tensor =vgg_inp, input_shape =(img_size,img_size,channels) , weights='imagenet' , include_top=False)
for l in vgg.layers: l.trainable =False
# Layer Output
loss_layer_output = [vgg.get_layer(loss_layer).output]
# Define a Model that calculates the feature representation
vgg_reconstruction_model = Model(inputs =vgg_inp, outputs =loss_layer_output)
vgg_reconstruction_model.summary()
# Feature represenation of hr image and prediction image
hr_vgg = vgg_reconstruction_model(vgg_inp)
pred_vgg = vgg_reconstruction_model(model_output)
reconstruction_loss = Lambda(self.get_reconstruction_loss,output_shape=(1,), name='reconstruction_loss')([pred_vgg[0], hr_vgg[0]])
self.loss_model = Model(inputs=[model_inp, vgg_inp] , outputs = [model_output, reconstruction_loss], name='loss_model')
def get_reconstruction_loss(self,args):
new_activation, content_activation = args[0], args[1]
return K.constant(self.lambda_content) * K.mean(K.square(new_activation - content_activation))
if __name__ == "__main__":
net = MyClass(500,3,True)
Error:
Exception has occurred: AttributeError
'NoneType' object has no attribute '_inbound_nodes'
File "/home/robousb2/gD_tools/playground/percep_loss_question.py", line 44, in __init__
self.loss_model = Model(inputs=[model_inp, vgg_inp] , outputs = [model_output, reconstruction_loss], name='loss_model')
File "/home/robousb2/gD_tools/playground/percep_loss_question.py", line 54, in <module>
net = MyClass(500,3,True)
The problem is here:
reconstruction_loss = Lambda(self.get_reconstruction_loss,output_shape=(1,), name='reconstruction_loss')([pred_vgg[0], hr_vgg[0]])
I am not sure why you are indexing with [0], if you remove these, it works:
reconstruction_loss = Lambda(self.get_reconstruction_loss,output_shape=(1,), name='reconstruction_loss')([pred_vgg, hr_vgg])