I'm trying to save my TensorFlow model using model.save(), however - I am getting this error.
The model summary is provided here:
Model Summary
The code for the transformer model:
def transformer(vocab_size, num_layers, units, d_model, num_heads, dropout, name="transformer"):
inputs = tf.keras.Input(shape=(None,), name="inputs")
dec_inputs = tf.keras.Input(shape=(None,), name="dec_inputs")
enc_padding_mask = tf.keras.layers.Lambda(
create_padding_mask, output_shape=(1, 1, None),
name='enc_padding_mask')(inputs)
# mask the future tokens for decoder inputs at the 1st attention block
look_ahead_mask = tf.keras.layers.Lambda(
create_look_ahead_mask,
output_shape=(1, None, None),
name='look_ahead_mask')(dec_inputs)
# mask the encoder outputs for the 2nd attention block
dec_padding_mask = tf.keras.layers.Lambda(
create_padding_mask, output_shape=(1, 1, None),
name='dec_padding_mask')(inputs)
enc_outputs = encoder(
vocab_size=vocab_size,
num_layers=num_layers,
units=units,
d_model=d_model,
num_heads=num_heads,
dropout=dropout,
)(inputs=[inputs, enc_padding_mask])
dec_outputs = decoder(
vocab_size=vocab_size,
num_layers=num_layers,
units=units,
d_model=d_model,
num_heads=num_heads,
dropout=dropout,
)(inputs=[dec_inputs, enc_outputs, look_ahead_mask, dec_padding_mask])
outputs = tf.keras.layers.Dense(units=vocab_size, name="outputs")(dec_outputs)
return tf.keras.Model(inputs=[inputs, dec_inputs], outputs=outputs, name=name)
I don't understand why it's giving this error since the model trains perfectly fine.
Any help would be appreciated.
My saving code for reference:
print("Saving the model.")
saveloc = "C:/tmp/solar.h5"
model.save(saveloc)
print("Model saved to: " + saveloc + " succesfully.")
It's not a bug, it's a feature.
This error lets you know that TF can't save your model, because it won't be able to load it.
Specifically, it won't be able to reinstantiate your custom Layer classes: encoder and decoder.
To solve this, just override their get_config method according to the new arguments you've added.
A layer config is a Python dictionary (serializable) containing the configuration of a layer. The same layer can be reinstantiated later (without its trained weights) from this configuration.
For example, if your encoder class looks something like this:
class encoder(tf.keras.layers.Layer):
def __init__(
self,
vocab_size, num_layers, units, d_model, num_heads, dropout,
**kwargs,
):
super().__init__(**kwargs)
self.vocab_size = vocab_size
self.num_layers = num_layers
self.units = units
self.d_model = d_model
self.num_heads = num_heads
self.dropout = dropout
# Other methods etc.
then you only need to override this method:
def get_config(self):
config = super().get_config().copy()
config.update({
'vocab_size': self.vocab_size,
'num_layers': self.num_layers,
'units': self.units,
'd_model': self.d_model,
'num_heads': self.num_heads,
'dropout': self.dropout,
})
return config
When TF sees this (for both classes), you will be able to save the model.
Because now when the model is loaded, TF will be able to reinstantiate the same layer from config.
Layer.from_config's source code may give a better sense of how it works:
#classmethod
def from_config(cls, config):
return cls(**config)
This problem is caused by mixing imports between the keras and tf.keras libraries, which is not supported.
Use tf.keras.models or usr keras.models everywhere
You should never mix imports between these libraries, as it will not work and produces all kinds of strange error messages. These errors change with versions of keras and tensorflow.
I suggest You try the following:
model = tf.keras.Model(...)
model.save_weights("some_path")
...
model.load_weights("some_path")
I think simple solution is to install the tensorflow==2.4.2 for gpu tensorflow-gpu==2.4.2 , i faced the issue and debug the whole day but it was not resolved. finally i installed the older stable version and error is gone
Related
I'm trying to remove multiple layers form a pre-trained Keras model (ResNet50V2), but no matter what I do it's not working. I've read countless other questions on stack overflow, github issues, and forum posts related to this topic in the past month, and I still can't make it work... So I'll ask directly. What might I be doing wrong?
from ray.rllib.models.tf.tf_modelv2 import TFModelV2
from ray.rllib.utils.framework import try_import_tf
from ray.rllib.models import ModelCatalog
tf = try_import_tf()
def resnet_core(x):
x = tf.keras.applications.resnet_v2.preprocess_input(x)
resnet = tf.keras.applications.ResNet50V2(
include_top=False,
weights="imagenet",
)
remove_n = 130
for i in range(remove_n):
resnet._layers.pop()
print(len(resnet._layers))
s = tf.keras.models.Model(resnet.input, resnet._layers[-1].output, name='resnet-core')
for layer in s.layers:
print('adding layer',layer.name)
for layer in s.layers[:]:
layer.trainable = False
s.build(None)
return s(x)
class ImpalaCNN(TFModelV2):
def __init__(self, obs_space, action_space, num_outputs, model_config, name):
super().__init__(obs_space, action_space, num_outputs, model_config, name)
inputs = tf.keras.layers.Input(shape=obs_space.shape, name="observations")
x = inputs
x = resnet_core(x)
x = tf.keras.layers.Flatten()(x)
x = tf.keras.layers.ReLU()(x)
x = tf.keras.layers.Dense(units=256, activation="relu", name="hidden")(x)
logits = tf.keras.layers.Dense(units=num_outputs, name="pi")(x)
value = tf.keras.layers.Dense(units=1, name="vf")(x)
self.base_model = tf.keras.Model(inputs, [logits, value])
self.register_variables(self.base_model.variables)
def forward(self, input_dict, state, seq_lens):
obs = tf.cast(input_dict["obs"], tf.float32)
logits, self._value = self.base_model(obs)
return logits, state
def value_function(self):
return tf.reshape(self._value, [-1])
# Register model in ModelCatalog
ModelCatalog.register_custom_model("impala_cnn_tf", ImpalaCNN)
The error I'm getting is:
...
File "/Users/manu/anaconda3/envs/procgen/lib/python3.7/site-packages/ray/rllib/evaluation/rollout_worker.py", line 376, in __init__
self._build_policy_map(policy_dict, policy_config)
File "/Users/manu/anaconda3/envs/procgen/lib/python3.7/site-packages/ray/rllib/evaluation/rollout_worker.py", line 859, in _build_policy_map
policy_map[name] = cls(obs_space, act_space, merged_conf)
File "/Users/manu/anaconda3/envs/procgen/lib/python3.7/site-packages/ray/rllib/policy/tf_policy_template.py", line 143, in __init__
obs_include_prev_action_reward=obs_include_prev_action_reward)
File "/Users/manu/anaconda3/envs/procgen/lib/python3.7/site-packages/ray/rllib/policy/dynamic_tf_policy.py", line 163, in __init__
framework="tf")
File "/Users/manu/anaconda3/envs/procgen/lib/python3.7/site-packages/ray/rllib/models/catalog.py", line 317, in get_model_v2
registered))
ValueError: It looks like variables {<tf.Variable 'default_policy/
conv4_block4_1_conv/kernel:0' ... }
were created as part of <impala_cnn_tf.ImpalaCNN object at
0x19a8ccc90> but does not appear in model.variables()
({<tf.Variable 'default_policy/pi/
kernel:0' shape=(256, 15) dtype=float32> ...}). Did you forget to call
model.register_variables() on the variables in question?
The error seems to indicate some variables from the layers I'm trying to skip were not registered, but that's because I don't want to use them! Any ideas?
More context in case it helps:
If I set remove_n = 0 I don't see the error (but of course the whole ResNet50V2 is being used)
I'm a newbie with Keras and ML. This might be a very dumb question.
The reason I'm trying to remove many layers, not just the last one, is that I want the model to fit on a small GPU.
I'm trying to train the model using rllib for the aicrowd procgen competition (using the competition to learn Keras/RL)
Full code here: https://github.com/maraoz/neurips2020-procgen-starter-kit/blob/master/models/impala_cnn_tf.py#L55
Full error log here: https://pastebin.com/X0Dk7wdd
Thanks in advance!
Rather than popping off layers, you could try accessing the 130th layer from the last layer. Then, you can build a new model using the input of your original model and the output of this layer.
model = tf.keras.models.Model(resnet.input, resnet.layers[-130].output)
This will do essentially the same thing as what you tried but its much easier and safer since you aren't accessing any private properties of the model itself.
I'm trying to test an A3C-model that I was provided through this tutorial:
https://medium.com/tensorflow/deep-reinforcement-learning-playing-cartpole-through-asynchronous-advantage-actor-critic-a3c-7eab2eea5296
First, since the algorithm doesn't recognize all the packages (although installed), I tried to refer to them in different approaches:
from tensorflow import keras (didn't work)
from tensorflow._api.v1 import keras (worked)
Now I have the following base-concept of an A3C-model:
class ActorCriticModel(keras.Model):
def __init__(self, state_size, action_size):
super(ActorCriticModel, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.dense1 = layers.Dense(100, activation='relu')
self.policy_logits = layers.Dense(action_size)
self.dense2 = layers.Dense(100, activation='relu')
self.values = layers.Dense(1)
def call(self, inputs):
# Forward pass
x = self.dense1(inputs)
logits = self.policy_logits(x)
v1 = self.dense2(inputs)
values = self.values(v1)
return logits, values
model = ActorCriticModel()
Providing the packages:
from keras import layers (which is recognized as used package)
from keras.layers import Input (isn't recognized as used package)
I currently face the problem that inputs in my model aren't "matching signature of base method in class 'Network'".
How do I solve this?
About the versions of the programs and API's that I work with:
Python 3.7.3
Tensorflow 1.13.1
Keras 2.3.1
I am trying to utilize Model.fit_generator of Keras in tensorflow 1.10.
Simplified reproducible code:
import tensorflow as tf
import numpy as np
class TestNet(tf.keras.Model):
def __init__(self, class_count, name='TestNet', **kwargs):
super(TestNet, self).__init__(name=name, **kwargs)
self.convolution = tf.keras.layers.Conv1D(class_count, kernel_size=1, input_shape=(None, 3))
def call(self, points):
return self.convolution(points)
def segmentation_loss(labels, logits):
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=logits)
return tf.reduce_mean(cross_entropy)
def generate():
while True:
yield (np.zeros(shape=(100,3)), np.zeros(shape=(100)))
if __name__ == "__main__":
test_net = TestNet(class_count=5)
optimizer = tf.keras.optimizers.Adam()
test_net.compile(optimizer, loss=segmentation_loss)
history = test_net.fit_generator(generate, steps_per_epoch=1000, epochs=10)
While this works in tensorflow 1.14, executing this in 1.10 yields the NotImplementedError in the title:
NotImplementedError: fit_generator is not yet enabled for unbuilt Model subclasses
Anybody knows how to work around this?
I see a mistake in fit_generator, update to this:
history = test_net.fit_generator(generate(), steps_per_epoch=1000, epochs=10)
Now, I don't know how Tensorflow 1.10 works, but this kind of modeling is rather new, usually a Keras model is built like this:
#model's inputs
inputs = Input((None,3))
#model's layers
convolution = Conv1D(class_count, kernel_size=1)
#model's call
outputs = convolution(inputs)
#model finish
test_net = Model(inputs, outputs)
A workaround for you:
inputs = Input(shape)
test_net = TestNet(...)
outputs = test_net.call(inputs)
test_net_model = Model(inputs, outputs)
I am quite new to machine learning and I am trying to implement my custom layer in keras. I found a couple of tutorials and it seems comparatively straight forward. What I do not understand, though, is how to implement my new custom layer in Sequential(). See for example this classification problem that I took from the tensorflow website(https://www.tensorflow.org/tutorials/keras/basic_text_classification), posted here for your convenience:
from __future__ import absolute_import, division, print_function
import tensorflow as tf
from tensorflow import keras
import numpy as np
imdb = keras.datasets.imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)
# A dictionary mapping words to an integer index
word_index = imdb.get_word_index()
# The first indices are reserved
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2 # unknown
word_index["<UNUSED>"] = 3
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
def decode_review(text):
return ' '.join([reverse_word_index.get(i, '?') for i in text])
train_data = keras.preprocessing.sequence.pad_sequences(train_data,
value=word_index["<PAD>"],
padding='post',
maxlen=256)
test_data = keras.preprocessing.sequence.pad_sequences(test_data,
value=word_index["<PAD>"],
padding='post',
maxlen=256)
# input shape is the vocabulary count used for the movie reviews (10,000 words)
vocab_size = 10000
model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))
model.summary()
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
x_val = train_data[:10000]
partial_x_train = train_data[10000:]
y_val = train_labels[:10000]
partial_y_train = train_labels[10000:]
history = model.fit(partial_x_train,
partial_y_train,
epochs=40,
batch_size=512,
validation_data=(x_val, y_val),
verbose=1)
results = model.evaluate(test_data, test_labels)
print(results)
Do I have to change the source code for keras.Sequential() or is there an easy way?
Furthermore, looking at the source code for the class Sequential() made me wonder: I can't figure out how functions like 'summary()','compile()', 'fit()' and 'evaluate()' can be called if those are not even provided in the source code in this class. Here is the source code for Sequential():
https://github.com/keras-team/keras/blob/a1397169ddf8595736c01fcea084c8e34e1a3884/keras/engine/sequential.py
Sequential is a Model, and not a layer.
The functions you mentioned (summary, compile, fit, evaluate) are implemented in the Model class linked here, as Sequential is a subclass of Model.
If you're writing a custom layer, you should be subclassing Layer instead, and not Model or Sequential.
You would need to implement build, call, and compute_output_shape to create your own layer.
There's a few examples on the Keras documentation:
from keras import backend as K
from keras.layers import Layer
class MyLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[1], self.output_dim),
initializer='uniform',
trainable=True)
super(MyLayer, self).build(input_shape) # Be sure to call this at the end
def call(self, x):
return K.dot(x, self.kernel)
def compute_output_shape(self, input_shape):
return (input_shape[0], self.output_dim)
To use it, import the MyLayer class from whichever file you put it in, and then add it like the default Keras layers:
from custom.layers import MyLayer
model = keras.Sequential()
model.add(MyLayer())
I am trying to build an RCL block in Keras for recurrent convolution network as described in the paper : https://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Liang_Recurrent_Convolutional_Neural_2015_CVPR_paper.pdf
My model using my custom keras layer gets compiled however, after running 1 epoch when model is being saved i encounter an error thrown by h5py:
h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
h5py/h5o.pyx in h5py.h5o.link()
RuntimeError: Unable to create link (name already exists)
I am using already existing keras layers inside my custom layer:
class Recurrent_block(tf.keras.layers.Layer):
def __init__(self, ch_out,t):
self.num_outputs = num_outputs
self.t = t
self.ch_out = ch_out
super(Recurrent_block, self).__init__()
def build(self, input_shape)
self.shape = tf.TensorShape([input_shape[0].value,input_shape[1].value,input_shape[2].value,self.ch_out])
self.fc = tf.keras.models.Sequential([tf.keras.layers.Conv2D(filters=self.ch_out, kernel_size=(3,3), strides=(1, 1), padding='same'),tf.keras.layers.BatchNormalization(),tf.keras.layers.Activation('relu')])
self.fc.build(self.shape)
print(self.fc.trainable_weights)
self._trainable_weights = self.fc.trainable_weights
super(Recurrent_block, self).build(self.shape)
def call(self,x):
x = tf.keras.layers.Conv2D(self.ch_out,kernel_size=1,strides=(1, 1),padding='same')(x)
for i in range(self.t+1):
if i==0:
x1 = self.fc(x)
x1 = self.fc(x+x1)
return x1
Can someone please help either resolving this error or just guiding me on how to combine multiple keras layer into one custom layer and also to change the call function in the way i did.