Develop Reference

How to change the input shape of model in Keras

I have a model that I load this way:
def YOLOv3_pretrained(n_classes=12, n_bbox=3):
yolo3 = tf.keras.models.load_model("yolov3/yolo3.h5")
yolo3.trainable = False
l3 = yolo3.get_layer('leaky_re_lu_71').output
l3_flat = tf.keras.layers.Flatten()(l3)
out3 = tf.keras.layers.Dense(100*(4+1+n_classes))(l3_flat)
out3 = Reshape((100, (4+1+n_classes)), input_shape=(12,))(out3)
yolo3 = Model(inputs=yolo3.input, outputs=[out3])
return yolo3
I want to add a Dense at the end of it but since it takes an input with shape (None, 416,416,3) it doesn't let me do it and it returns an error:
ValueError: The last dimension of the inputs to a Dense layer should be defined. Found None. Full input shape received: (None, None)
I also tried this way with a Sequential (I want to use just the last output of yolo):
def YOLOv3_Dense(n_classes=12):
yolo3 = tf.keras.models.load_model("yolov3/yolo3.h5")
model = Sequential()
model.add(yolo3)
model.add(Flatten())
model.add(Dense(100*(4+1+n_classes)))
model.add(Reshape((100, (4+1+n_classes)), input_shape=(413,413,3)))
return model
But it returns another error:
ValueError: All layers in a Sequential model should have a single output tensor. For multi-output layers, use the functional API.
Is there a way to add the final Dense layer?

The problem is that you are trying to reduce (flatten) an output with multiple None dimensions, which will not work if you want to use the output as input to another layer. You can try using a GlobalAveragePooling2D or GlobalMaxPooling2D instead:
import tensorflow as tf
yolo3 = tf.keras.models.load_model("yolo3.h5")
yolo3.trainable = False
l3 = yolo3.get_layer('leaky_re_lu_71').output
l3_flat = tf.keras.layers.GlobalMaxPooling2D()(l3)
out3 = tf.keras.layers.Dense(100*(4+1+12))(l3_flat)
out3 = tf.keras.layers.Reshape((100, (4+1+12)), input_shape=(12,))(out3)
yolo3 = tf.keras.Model(inputs=yolo3.input, outputs=[out3])

How to save model properly to continue the training of VAE in keras

I have built the VAE in keras using functional API. The VAE has 3 models:
encoder
def _create_encoder(self):
# create convolutional layers for encoder
X = self.images
for i in range(len(self.encoder_filters)):
X = self._create_conv_layer(X,
"Conv2D",
self.encoder_filters[i],
self.encoder_kernel_size[i],
self.encoder_strides[i],
self.encoder_padding[i],
"encoder_conv"+str(i)
)
# keep track of tensor shape before flattening (we will need this to build decoder)
encoder_shape_before_flattening = K.int_shape(X)[1:]
# flatten the tensor
X = Flatten()(X)
# create dense layers for mu and sigma
self.encoder_mu = Dense(units=self.latent_space_size, name='encoder_mu')(X)
self.encoder_log_var = Dense(units=self.latent_space_size, name='encoder_log_var')(X)
self.encoder_parameters = Model(self.images, (self.encoder_mu, self.encoder_log_var))
# create encoder output by sampling from normal distribution
self.encoder_output = Lambda(self.sample_latent_space,name="encoder_output")([self.encoder_mu,self.encoder_log_var])
self.encoder = Model(inputs=self.images, outputs=self.encoder_output)
return encoder_shape_before_flattening
decoder
def _create_decoder(self, encoder_shape_before_flattening):
X = Dense(np.prod(encoder_shape_before_flattening))(self.decoder_input)
X = Reshape(encoder_shape_before_flattening)(X)
# create convolutional layers for decoder
for i in range(len(self.decoder_filters)):
is_not_last_layer = i < len(self.decoder_filters)-1
X = self._create_conv_layer(X,
"Conv2DTranspose",
self.decoder_filters[i],
self.decoder_kernel_size[i],
self.decoder_strides[i],
self.decoder_padding[i],
"decoder_conv"+str(i),
batch_norm=is_not_last_layer,
dropout=is_not_last_layer,
activation=is_not_last_layer
)
# output values should be between 0 and 1
self.decoder_output = Activation("sigmoid")(X)
self.decoder = Model(inputs=self.decoder_input, outputs=self.decoder_output)
the whole model
def _create_model(self):
self.images = Input(shape=self.input_dims, name="images")
# create encoder as separate model
encoder_shape_before_flattening = self._create_encoder()
# create decoder as separate model
self.decoder_input = Input(shape=(self.latent_space_size,), name="decoder_input")
self._create_decoder(encoder_shape_before_flattening)
# create unique model
self.model = Model(inputs=self.images, outputs=self.decoder(self.encoder_output))
I am using ModelCheckpoint callback to save the whole model after every epoch.
checkpoint_model = ModelCheckpoint(os.path.join(save_path, "model.h5"), verbose=1)
But when I load the model with load_model
def load_trained_model(self, load_path, r_loss_factor):
self.model = load_model(os.path.join(load_path, "model.h5"), custom_objects={"loss": self.penalized_loss(r_loss_factor),"sample_latent_space":self.sample_latent_space})
and call fit_generator again to continue the training I get the following error:
InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: You must feed a value for placeholder tensor 'images' with dtype float and shape [?,128,128,3]
[[{{node images}}]]
[[metrics_1/loss_1/Identity/_1971]]
(1) Invalid argument: You must feed a value for placeholder tensor 'images' with dtype float and shape [?,128,128,3]
[[{{node images}}]]
The code can be found here

Behavior difference when building TF Keras RNN with two different methods

I am building a RNN text generator, mostly going from the Tensorflow docs here.
My question, I have defined the model two ways:
Method (1):
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embed_dim,
batch_input_shape=[batch_size, None]),
tf.keras.layers.GRU(rnn_units, return_sequences=True, stateful=True,
recurrent_initializer='glorot_uniform'),
tf.keras.layers.Dense(vocab_size)
])
Method (2):
model = tf.keras.Sequential()
model.add(tf.keras.layers.Embedding(vocab_size, embed_dim,
batch_input_shape=[BATCH_SIZE, None]))
model.add(tf.keras.layers.GRU(rnn_units, return_sequences=True,
stateful=True,
recurrent_initializer='glorot_uniform'))
model.add(tf.keras.layers.Dense(vocab_size))
In my mind, these both do the same thing. However when generating text with:
def generate_text(model, start_string, length=1000):
# converting start string to numbers (vectorisation)
input_eval = [char2idx[s] for s in start_string]
input_eval = tf.expand_dims(input_eval, 0)
# initialise empty string to store results
text = []
model.reset_states()
for i in range(length):
predictions = model(input_eval)
# remove batch dimension
predictions = tf.squeeze(predictions, 0)
# use categorical distribution to predict character returned by model
predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()
# we pass the predicted character as the next input to the model
# along with previous hidden state
input_eval = tf.expand_dims([predicted_id], 0)
# append predicted text
text.append(idx2char[predicted_id])
return (start_string + ''.join(text))
Which I pass:
print(generate_text(model, start_string=u'From '))
Method (1) works perfectly, but method (2) throws the following error:
WARNING:tensorflow:Model was constructed with shape Tensor("embedding_1_input:0", shape=(64, None), dtype=float32) for input (64, None), but it was re-called on a Tensor with incompatible shape (1, 5).
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-33-eb814780c9fe> in <module>()
----> 1 print(generate_text(model, start_string=u'From ', length=PRINT))
14 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/ops.py in set_shape(self, shape)
1086 raise ValueError(
1087 "Tensor's shape %s is not compatible with supplied shape %s" %
-> 1088 (self.shape, shape))
1089
1090 # Methods not supported / implemented for Eager Tensors.
ValueError: Tensor's shape (5, 64, 1024) is not compatible with supplied shape [5, 1, 1024]
If anyone could help me understand what the difference is between these two methods that would be amazing, thankyou!
Edit:
Including model saving and loading code. I use this to save the model (with a batch size 64) and then load with a batch size of 1 for text generation.
Saving weights:
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath='./training_checkpoints/ckpt_{epoch}',
save_weights_only=True
)
Loading weights into new model (batch size = 1):
model = build_model(len(vocab_char), EMBED_DIM, UNITS, 1)
model.load_weights(tf.train.latest_checkpoint('./training_checkpoints'))
model.build(tf.TensorShape([1, None]))
model.summary()

Extracting the dropout mask from a keras dropout layer?

I would like to extract and store the dropout mask [array of 1/0s] from a dropout layer in a Sequential Keras model at each batch while training. I was wondering if there was a straight forward way way to do this within Keras or if I would need to switch over to tensorflow (How to get the dropout mask in Tensorflow).
Would appreciate any help! I'm quite new to TensorFlow and Keras.
There are a couple of functions (dropout_layer.get_output_mask(), dropout_layer.get_input_mask()) for the dropout layer that I tried using but got None after calling on the previous layer.
model = tf.keras.Sequential()
model.add(tf.keras.layers.Flatten(name="flat", input_shape=(28, 28, 1)))
model.add(tf.keras.layers.Dense(
512,
activation='relu',
name = 'dense_1',
kernel_initializer=tf.keras.initializers.GlorotUniform(seed=123),
bias_initializer='zeros'))
dropout = tf.keras.layers.Dropout(0.2, name = 'dropout') #want this layer's mask
model.add(dropout)
x = dropout.output_mask
y = dropout.input_mask
model.add(tf.keras.layers.Dense(
10,
activation='softmax',
name='dense_2',
kernel_initializer=tf.keras.initializers.GlorotUniform(seed=123),
bias_initializer='zeros'))
model.compile(...)
model.fit(...)

It's not easily exposed in Keras. It goes deep until it calls the Tensorflow dropout.
So, although you're using Keras, it's will also be a tensor in the graph that can be gotten by name (finding it's name: In Tensorflow, get the names of all the Tensors in a graph).
This option, of course will lack some keras information, you should probably have to do that inside a Lambda layer so Keras adds certain information to the tensor. And you must take extra care because the tensor will exist even when not training (where the mask is skipped)
Now, you can also use a less hacky way, that may consume a little processing:
def getMask(x):
boolMask = tf.not_equal(x, 0)
floatMask = tf.cast(boolMask, tf.float32) #or tf.float64
return floatMask
Use a Lambda(getMasc)(output_of_dropout_layer)
But instead of using a Sequential model, you will need a functional API Model.
inputs = tf.keras.layers.Input((28, 28, 1))
outputs = tf.keras.layers.Flatten(name="flat")(inputs)
outputs = tf.keras.layers.Dense(
512,
# activation='relu', #relu will be a problem here
name = 'dense_1',
kernel_initializer=tf.keras.initializers.GlorotUniform(seed=123),
bias_initializer='zeros')(outputs)
outputs = tf.keras.layers.Dropout(0.2, name = 'dropout')(outputs)
mask = Lambda(getMask)(outputs)
#there isn't "input_mask"
#add the missing relu:
outputs = tf.keras.layers.Activation('relu')(outputs)
outputs = tf.keras.layers.Dense(
10,
activation='softmax',
name='dense_2',
kernel_initializer=tf.keras.initializers.GlorotUniform(seed=123),
bias_initializer='zeros')(outputs)
model = Model(inputs, outputs)
model.compile(...)
model.fit(...)
Training and predicting
Since you can't train the masks (it doesn't make any sense), it should not be an output of the model for training.
Now, we could try this:
trainingModel = Model(inputs, outputs)
predictingModel = Model(inputs, [output, mask])
But masks don't exist in prediction, because dropout is only applied in training. So this doesn't bring us anything good in the end.
The only way for training is then using a dummy loss and dummy targets:
def dummyLoss(y_true, y_pred):
return y_true #but this might evoke a "None" gradient problem since it's not trainable, there is no connection to any weights, etc.
model.compile(loss=[loss_for_main_output, dummyLoss], ....)
model.fit(x_train, [y_train, np.zeros((len(y_Train),) + mask_shape), ...)
It's not guaranteed that these will work.

I found a very hacky way to do this by trivially extending the provided dropout layer. (Almost all code from TF.)
class MyDR(tf.keras.layers.Layer):
def __init__(self,rate,**kwargs):
super(MyDR, self).__init__(**kwargs)
self.noise_shape = None
self.rate = rate
def _get_noise_shape(self,x, noise_shape=None):
# If noise_shape is none return immediately.
if noise_shape is None:
return array_ops.shape(x)
try:
# Best effort to figure out the intended shape.
# If not possible, let the op to handle it.
# In eager mode exception will show up.
noise_shape_ = tensor_shape.as_shape(noise_shape)
except (TypeError, ValueError):
return noise_shape
if x.shape.dims is not None and len(x.shape.dims) == len(noise_shape_.dims):
new_dims = []
for i, dim in enumerate(x.shape.dims):
if noise_shape_.dims[i].value is None and dim.value is not None:
new_dims.append(dim.value)
else:
new_dims.append(noise_shape_.dims[i].value)
return tensor_shape.TensorShape(new_dims)
return noise_shape
def build(self, input_shape):
self.noise_shape = input_shape
print(self.noise_shape)
super(MyDR,self).build(input_shape)
#tf.function
def call(self,input):
self.noise_shape = self._get_noise_shape(input)
random_tensor = tf.random.uniform(self.noise_shape, seed=1235, dtype=input.dtype)
keep_prob = 1 - self.rate
scale = 1 / keep_prob
# NOTE: if (1.0 + rate) - 1 is equal to rate, then we want to consider that
# float to be selected, hence we use a >= comparison.
self.keep_mask = random_tensor >= self.rate
#NOTE: here is where I save the binary masks.
#the file grows quite big!
tf.print(self.keep_mask,output_stream="file://temp/droput_mask.txt")
ret = input * scale * math_ops.cast(self.keep_mask, input.dtype)
return ret

serving a tensorflow classifier

I have been fighting with tensorflow's builder to be able to serve my model, I am trying to feed data to my classifier after serving the model
My question is how would i feed the input to the model?
I have seen the code used by google's inception tutorial
and have tried to implement it
classify_inputs_tensor_info = utils.build_tensor_info(
serialized_tf_example)
classes_output_tensor_info = utils.build_tensor_info(classes)
scores_output_tensor_info = utils.build_tensor_info(values)
classification_signature = signature_def_utils.build_signature_def(
inputs={
signature_constants.CLASSIFY_INPUTS: classify_inputs_tensor_info
},
outputs={
signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor_info,
signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor_info
},
method_name=signature_constants.CLASSIFY_METHOD_NAME)
and from what i understand the input is passed to a tensor called serialized_tf_example which as the name suggests serializes the input to string but then they use tf.FixedLenFeature which i don't understand and then parses the serialized_tf_example with tf.parse_example and assigns it to x which is used within the model, but i would like to parse it to a classifier that accepts arrays as inputs but don't know how to go around this.
while trying to implement this i wrote this
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = { 'audio/encoded': tf.FixedLenFeature( shape=[193], dtype=tf.float32, default_value=input_x),}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf_example['audio/encoded']
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
# Define the dimensions in the feature columns
feature_columns = [tf.contrib.layers.real_valued_column("", dimension=5)]
classifier = tf.contrib.learn.DNNLinearCombinedClassifier(
dnn_feature_columns=feature_columns, dnn_hidden_units=[200,300], n_classes=10,
dnn_optimizer=tf.train.GradientDescentOptimizer(
learning_rate=0.01
)
)
#run training
classifier.fit(input_fn=get_train_inputs, steps=100)
#testing
accuracy_score = classifier.evaluate(input_fn=get_test_inputs, steps=10)["accuracy"]
print('Test accuracy : ', format(accuracy_score))
prediction = format(list(classifier.predict_classes(x, as_iterable=True)))
but x is a tensor and so is not able to be read. when i try use run or .eval() it asks me to feed a value to serialized_tf_example
InvalidArgumentError (see above for traceback): You must feed a value
for placeholder tensor 'tf_example' with dtype string [[Node:
tf_example = Placeholderdtype=DT_STRING, shape=[],
_device="/job:localhost/replica:0/task:0/cpu:0"]]
when i use prediction = format(list(classifier.predict_classes(np.array(x), as_iterable=True))
I get
InvalidArgumentError (see above for traceback): Shape in
shape_and_slice spec [1,200] does not match the shape stored in
checkpoint: [193,200] [[Node: save/RestoreV2_1 =
RestoreV2[dtypes=[DT_FLOAT],
_device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save/Const_0, save/RestoreV2_1/tensor_names, save/RestoreV2_1/shape_and_slices)]]

You can/should use classifier.predict without tf.Example.Your input_fn in train and eval returns x, y. you can write a predict_input_fn similar to other input functions.
predictoin = next(classifier.predict_classes(input_fn=predict_input_fn))
Please note that, if you get all predictions with list function should end by an exception. You can check tf.estimator.inputs.numpy_input_fn