I am training a feature extractor based on densenet, which looks like the following:
# Import the Sequential model and layers
from keras.models import Sequential
import keras
import tensorflow as tf
from keras.layers import Conv2D, MaxPooling2D, Lambda, Dropout, Concatenate
from keras.layers import Activation, Dropout, Flatten, Dense
import pandas as pd
from sklearn import preprocessing
import ast
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
size = 256
class DenseNetBase(tf.keras.Model):
def __init__(self, size, include_top = True):
super(DenseNetBase, self).__init__()
self.include_top = include_top
#base
self.base = tf.keras.applications.DenseNet201(weights='imagenet',include_top=False, pooling='avg',input_shape = (size,size,3))
#final layer
self.dense = Dense(1, activation='sigmoid', name='predictions')
def call(self, input_tensor):
input_image = input_tensor[0]
input_metafeatures = input_tensor[1]
#model
x = self.base(input_image)
if self.include_top:
x = self.dense(x)
return x
def build_graph(self):
x = self.base.input
y = tf.keras.Input(shape=(3,))
return tf.keras.Model(inputs=[x,y], outputs=self.call([x,y]))
I want to then take the DenseNetBase, keep the trained weights, but remove the final dense layer to use for extracting features. Simplified DenseClassifier looks like this:
class DenseClassifier(tf.keras.Model):
def __init__(self, size, feature_extractor):
super(DenseClassifier, self).__init__()
#base tf.keras.layers.Input(shape=(size,size,3))
self.feature_extractor = tf.keras.Model(inputs = tf.keras.Input(shape=(size,size,3)), outputs = feature_extractor.layers[-2].output)
#final layer
self.dense = Dense(1, activation='sigmoid', name='prediction')
def call(self, input_tensor):
input_image = input_tensor[0]
input_metafeatures = input_tensor[1]
#model
x = self.feature_extractor(input_image)
return self.dense(x)
def build_graph(self):
x = self.base.input
y = tf.keras.Input(shape=(3,))
return tf.keras.Model(inputs=[x,y], outputs=self.call([x,y]))
Tying it together:
#build densenet feature extractor we have trained
denseBase = DenseNetBase(256, include_top = True)
denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase.load_weights('./models/DenseBaseSimple.h5')
#this doesn't work
DenseClassifier = DenseClassifier(size = 256, feature_extractor = denseBase)
In the above example, I get an error for the input which I am not sure why. The expected behaviour would be that I could build the latter model, and compile, and the existing weights DenseNetBase would be used for feature extraction.
I have tried to replace the input section with inputs = feature_extractor.layers[-2].input which does compile, but does not seem to evaluate to the same accuracy as denseBase even though it is using the same weights (in the simple example above with no extra layers).
My goal/question:
How can I load the weights from the pre-trained denseBase but remove the last dense layer (so the output is (None, 1920) as from DenseNet without the top but with my weights).
How can I then load this model without dense into another subclassed model as above to extract features.
Thanks!
To answer my own question, I did some testing looking at the values of initialised weights using logic from here:
It's what's expected. DenseBaseClassifier (using denseBase) and using imagenet weights both have similar prediction weight initialisations. This is because both these layers are randomly initialised and not trained, while the prediction layer in denseBase has been optimised and hence is different.
For the denseNet section, DenseBaseClassifier (using denseBase) == denseBase (some noise due to only saving weights), whereas original imagenet weights are different.
Using denseBase_featureextractor = tf.keras.Model(inputs = denseBase.layers[-2].input, outputs = denseBase.layers[-2].output) does indeed preserve the weights.
Not sure why self.feature_extractor = tf.keras.Model(inputs = tf.keras.Input(shape=(size,size,3)), outputs = feature_extractor.layers[-2].output) doesn't work though.
denseBase = DenseNetBase(size, include_top = True)
denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase.load_weights('./models/DenseBaseSimple.h5')
denseBase_featureextractor = tf.keras.Model(inputs = denseBase.layers[-2].input, outputs = denseBase.layers[-2].output)
DenseClassifier_denseBase = DenseClassifier(size = 256, feature_extractor = denseBase_featureextractor)
DenseClassifier_denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase_imagenet = tf.keras.applications.DenseNet201(weights='imagenet',include_top=False, pooling='avg',input_shape = (size,size,3))
DenseClassifier_imagenet = DenseClassifier(size = 256, feature_extractor = denseBase_imagenet)
DenseClassifier_imagenet.build([(None, 256, 256, 3), (None,3)])
def get_weights_print_stats(layer):
W = layer.get_weights()
#print(len(W))
#for w in W:
# print(w.shape)
return W
def hist_weights(weights, title, bins=500):
for weight in weights[0:5]:
plt.hist(np.ndarray.flatten(weight), bins=bins)
plt.title(title)
fig = plt.figure(figsize=(15, 10))
fig.subplots_adjust(hspace=0.4, wspace=0.4)
W = get_weights_print_stats(denseBase.layers[1])
plt.subplot(2, 3, 1)
hist_weights(W, "denseBase")
y = plt.ylabel("Final prediction later weights")#, rotation="horizontal")
W = get_weights_print_stats(DenseClassifier_denseBase.layers[1])
plt.subplot(2, 3, 2)
hist_weights(W, "DenseBaseClassifier (using denseBase weights)")
W = get_weights_print_stats(DenseClassifier_imagenet.layers[1])
plt.subplot(2, 3, 3)
hist_weights(W, "DenseBaseClassifier (using imagenet weights)")
W = get_weights_print_stats(denseBase.layers[0])
plt.subplot(2, 3, 4)
hist_weights(W, "denseBase")
y = plt.ylabel("DenseNet base first 5 weights")#, rotation="horizontal")
W = get_weights_print_stats(DenseClassifier_denseBase.layers[0])
plt.subplot(2, 3, 5)
hist_weights(W, "DenseBaseClassifier (using denseBase weights)")
W = get_weights_print_stats(DenseClassifier_imagenet.layers[0])
plt.subplot(2, 3, 6)
hist_weights(W, "DenseBaseClassifier (using imagenet weights)")
Related
I am trying to implement a tensorflow model (encoder decoder like) in which I train initially with a small number of layers, and append the model with more layers after training. I thought it would be easiest to create the layers as Models as I intend on setting various layers to trainable = False at points and thought it'd be easiest this way.
The following code is a simple demonstration of an error I'm getting.
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.models import Model
from tensorflow.keras.layers import concatenate, Input
from tensorflow.keras.layers import MaxPool2D, UpSampling2D, ReLU
from tensorflow.keras.layers import BatchNormalization
def conv_block(x, filters, kernel_size=(3,3), padding="same", strides=1):
c = Conv2D(filters, kernel_size, padding=padding, strides=strides)(x)
c = ReLU()(c)
c=BatchNormalization()(c)
c = Conv2D(filters, kernel_size, padding=padding, strides=strides)(c)
c = ReLU()(c)
c=BatchNormalization()(c)
return c
def down_block(x, filters, kernel_size=(3,3), padding="same", strides=1):
c = conv_block(x, filters, kernel_size = kernel_size,
padding = padding, strides = strides)
p = MaxPool2D((2,2))(c)
return c,p
def up_block(x, skip, filters, kernel_size=(3,3), padding="same", strides=1):
us = UpSampling2D((2,2))(x)
concat = concatenate([us, skip])
c = conv_block(concat, filters, kernel_size = kernel_size,
padding = padding, strides = strides)
return c
def create_base_model():
inner_input = Input((None,None,128))
bn = conv_block(inner_input,128)
inner_model = Model(inputs=inner_input,outputs=bn)
return inner_model
def create_downblock_model():
model_input = Input((None,None,128))
c,p = down_block(model_input, 128)
down_model = Model(inputs = model_input, outputs = [c,p])
return down_model
def create_upblock_model():
input_u = Input((None,None,128))
input_c = Input((None,None,128))
u = up_block(input_u, input_c, 128)
up_model = Model(inputs=[input_u,input_c], outputs = u)
return up_model
bn_model = create_base_model()
# 1ST METHOD - This works
down_model1 = create_downblock_model()
up_model1 = create_upblock_model()
x = bn_model(down_model1.output[-1])
x = up_model1([x,down_model1.output[0]])
inner_model = Model(inputs=down_model1.input, outputs=x)
# 2ND METHOD - This doesn't work
down_model2 = create_downblock_model()
up_model2 = create_upblock_model()
x = down_model2(down_model1.output[-1])
x = bn_model(x[-1])
x = up_model2([x,down_model2.output[0]])
x = up_model1([x,down_model1.output[0]])
inner_model = Model(inputs=down_model1.input, outputs=x)
gets the following error for the second method.
Graph disconnected: cannot obtain value for tensor Tensor("input_5:0", shape=(None, None, None, 128), dtype=float32) at layer "input_5". The following previous layers were accessed without issue: ['input_2', 'conv2d_2', 're_lu_2']
Now down_model2 has the layer input_5:0, so I am assuming the issue is with the line x = down_model2(down_model1.output[-1]). I searched around and topics with a similar error would suggest that maybe the fact that: down_model1.output[-1] isn't an input layer is the issue, however I really don't understand why my method one works completely fine, but when I try to incorporate 2 downblocks, the same way of doing things fails? In my 1st method, I use down_block1.output[-1] as input when defining a new model fine, however it doesn't work in the second method?
I'm using tensorflow2.1.
Apologies if I'm overlooking something simple but I can't understand why this isn't working. Cheers
The problem is cause by x = up_model2([x,down_model2.output[0]]) at third-to-last line probably due to wrong repeated reference, you need change the last block of code to:
down_model2_output = down_model2(down_model1.output[-1])
x = bn_model(down_model2_output[-1])
x = up_model2([x,down_model2_output[0]])
x = up_model1([x,down_model1.output[0]])
inner_model = Model(inputs=down_model1.input, outputs=x)
I am trying to create a model with Normalized cross correlation custom layer, code taken from here
from keras import backend as K
from keras.layers import Conv2D, MaxPooling2D, Dense, Input, Flatten
from keras.models import Model, Sequential
from keras.engine import InputSpec, Layer
from keras import regularizers
from keras.optimizers import SGD, Adam
from keras.utils.conv_utils import conv_output_length
from keras import activations
import numpy as np
class Normalized_Correlation_Layer(Layer):
# create a class inherited from keras.engine.Layer.
def __init__(self, patch_size=(5, 5),
dim_ordering='tf',
border_mode='same',
stride=(1, 1),
activation=None,
**kwargs):
if border_mode != 'same':
raise ValueError('Invalid border mode for Correlation Layer '
'(only "same" is supported as of now):', border_mode)
self.kernel_size = patch_size
self.subsample = stride
self.dim_ordering = dim_ordering
self.border_mode = border_mode
self.activation = activations.get(activation)
super(Normalized_Correlation_Layer, self).__init__(**kwargs)
def compute_output_shape(self, input_shape):
return(input_shape[0][0], input_shape[0][1], input_shape[0][2], self.kernel_size[0] * input_shape[0][2]*input_shape[0][-1])
def get_config(self):
config = {'patch_size': self.kernel_size,
'activation': self.activation.__name__,
'border_mode': self.border_mode,
'stride': self.subsample,
'dim_ordering': self.dim_ordering}
base_config = super(Correlation_Layer, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def call(self, x, mask=None):
input_1, input_2 = x
stride_row, stride_col = self.subsample
inp_shape = input_1._keras_shape
output_shape = self.compute_output_shape([inp_shape, inp_shape])
padding_row = (int(self.kernel_size[0] / 2),int(self.kernel_size[0] / 2))
padding_col = (int(self.kernel_size[1] / 2),int(self.kernel_size[1] / 2))
input_1 = K.spatial_2d_padding(input_1, padding =(padding_row,padding_col))
input_2 = K.spatial_2d_padding(input_2, padding = ((padding_row[0]*2, padding_row[1]*2),padding_col))
output_row = output_shape[1]
output_col = output_shape[2]
output = []
for k in range(inp_shape[-1]):
xc_1 = []
xc_2 = []
# print("here")
for i in range(padding_row[0]):
for j in range(output_col):
xc_2.append(K.reshape(input_2[:, i:i+self.kernel_size[0], j:j+self.kernel_size[1], k],
(-1, 1,self.kernel_size[0]*self.kernel_size[1])))
for i in range(output_row):
slice_row = slice(i, i + self.kernel_size[0])
slice_row2 = slice(i + padding_row[0], i +self.kernel_size[0] + padding_row[0])
# print("dfg")
for j in range(output_col):
slice_col = slice(j, j + self.kernel_size[1])
xc_2.append(K.reshape(input_2[:, slice_row2, slice_col, k],
(-1, 1,self.kernel_size[0]*self.kernel_size[1])))
xc_1.append(K.reshape(input_1[:, slice_row, slice_col, k],
(-1, 1,self.kernel_size[0]*self.kernel_size[1])))
for i in range(output_row, output_row+padding_row[1]):
for j in range(output_col):
xc_2.append(K.reshape(input_2[:, i:i+ self.kernel_size[0], j:j+self.kernel_size[1], k],
(-1, 1,self.kernel_size[0]*self.kernel_size[1])))
xc_1_aggregate = K.concatenate(xc_1, axis=1)
xc_1_mean = K.mean(xc_1_aggregate, axis=-1, keepdims=True)
xc_1_std = K.std(xc_1_aggregate, axis=-1, keepdims=True)
xc_1_aggregate = (xc_1_aggregate - xc_1_mean) / xc_1_std
xc_2_aggregate = K.concatenate(xc_2, axis=1)
xc_2_mean = K.mean(xc_2_aggregate, axis=-1, keepdims=True)
xc_2_std = K.std(xc_2_aggregate, axis=-1, keepdims=True)
xc_2_aggregate = (xc_2_aggregate - xc_2_mean) / xc_2_std
xc_1_aggregate = K.permute_dimensions(xc_1_aggregate, (0, 2, 1))
block = []
len_xc_1= len(xc_1)
print("asdf")
for i in range(len_xc_1):
#This for loop is to compute the product of a given patch of feature map 1 and the feature maps on which it is supposed to
sl1 = slice(int(i/inp_shape[2])*inp_shape[2],
int(i/inp_shape[2])*inp_shape[2]+inp_shape[2]*self.kernel_size[0])
#This calculates which are the patches of feature map 2 to be considered for a given patch of first feature map.
block.append(K.reshape(K.batch_dot(xc_2_aggregate[:,sl1,:],
xc_1_aggregate[:,:,i]),(-1,1,1,inp_shape[2] *self.kernel_size[0])))
block = K.concatenate(block, axis=1)
# print("zxcv")
block= K.reshape(block,(-1,output_row,output_col,inp_shape[2] *self.kernel_size[0]))
output.append(block)
output = self.activation(output)
print(output)
return output
My model is a combination of cross correlation and Conv2D layers,
dt = 'float32'
def create_model():
ip = keras.layers.Input((50,50, 1))
ncx1_1 = Normalized_Correlation_Layer(patch_size=(1, 1))([ip,ip])
ncn1_1 = keras.layers.Conv2D(64, (1,1), activation = 'relu', dtype=dt)(ip)
ncn2_1 = keras.layers.Conv2D(64, (1,1), activation = 'relu', dtype=dt)(ncx1_1)
ncx2_1 = Normalized_Correlation_Layer(patch_size=(1, 1),dtype=dt)([ncn1_1,ncn2_1])
# ncx2_1 = keras.layers.Reshape((50, 50, 3200))(ncx2_1)
# Problem occurs here
ncn3 = keras.layers.Conv2D(filters=64,kernel_size=(1,1), activation = 'relu', dtype=dt)(ncx2_1)
ncn4 = keras.layers.Conv2D(12, (1,1), activation = 'sigmoid', dtype=dt)(ncn3)
model = keras.models.Model(ip,ncn4)
return model
The model till the last cross correlation layer is successfully created, but I get problem for ncn3 layer
ValueError: number of input channels does not match corresponding dimension of filter, 50 != 3200
The output shape printed from the ncx2_1 layer, while creating it is printed as (?, 50, 50, 50),
when I print ncx2_1.shape and also the outputs returned from call function of layer class ([<tf.Tensor 'normalized__correlation__layer_4/Reshape_10000:0' shape=(?, 50, 50, 50) dtype=float32>]).
But the model summary shows it as (?,50,50,3200) when I create the model till that layer only, ie. model = keras.models.Model(ip,ncx2_1)
When I reshape the layer using ncx2_1 = keras.layers.Reshape((50, 50, 3200))(ncx2_1) , I can create the model successfully, but when I try to fit the data on it, I get :
InvalidArgumentError: Input to reshape is a tensor with 6250000 values, but the requested shape has 400000000
[[node reshape_1/Reshape (defined at /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1781) ]]
[[node loss/mul (defined at /usr/local/lib/python3.6/dist-packages/keras/engine/training.py:865) ]]
Here, my batch size is 50, so for a layer with (B,H,W,C) inputs of (50,50,50,50), the size should be 6250000, butt for (50,50,50,3200), it should be 400000000, which means that the output of cross correlation layer is 50 channels.
I am either interpreting this wrong or I have made a mistake somewhere which I would like to know about.
I am using keras 2.1.2 with tensorflow 1.13.1 (That was the version in which the custom layer was written and I was getting other problems with latest version)
I am also using a custom generator if that is needed info and calling fit using md.fit_generator(train_gen,verbose=1). I can also add any other detail necessary.
I'm not sure how to modify my code to get keras activations. I've seen conflicting examples of K.function() inputs and am not sure if I'm getting outputs per layer our activations.
Here is my code
activity = 'Downstairs'
layer = 1
seg_x = create_segments_and_labels(df[df['ActivityEncoded']==mapping[activity]],TIME_PERIODS,STEP_DISTANCE,LABEL)[0]
get_layer_output = K.function([model_m.layers[0].input],[model_m.layers[layer].output])
layer_output = get_layer_output([seg_x])[0]
try:
ax = sns.heatmap(layer_output[0].transpose(),cbar=True,cbar_kws={'label':'Activation'})
except:
ax = sns.heatmap(layer_output.transpose(),cbar=True,cbar_kws={'label':'Activation','rotate':180})
ax.set_xlabel('Kernel',fontsize=30)
ax.set_yticks(range(0,len(layer_output[0][0])+1,10))
ax.set_yticklabels(range(0,len(layer_output[0][0])+1,10))
ax.set_xticks(range(0,len(layer_output[0])+1,5))
ax.set_xticklabels(range(0,len(layer_output[0])+1,5))
ax.set_ylabel('Filter',fontsize=30)
ax.xaxis.labelpad = 10
ax.set_title('Filter vs. Kernel\n(Layer=' + model_m.layers[layer].name + ')(Activity=' + activity + ')',fontsize=35)
Suggestions here on stack overflow just do it as I do:
Keras, How to get the output of each layer?
Example 4 adds k's learning phase to the mix but my output is still the same.
https://www.programcreek.com/python/example/93732/keras.backend.function
Am I getting output or activations? Documentation implies i might need layers.activations but I haven't made that work.
My code, or the code passing in learning phase both get this heatmap.
https://imgur.com/a/5fI6N0B
For layers defined as e.g. Dense(activation='relu'), layer.outputs will fetch the (relu) activations. To get layer pre-activations, you'll need to set activation=None (i.e. 'linear'), followed by an Activation layer. Example below.
from keras.layers import Input, Dense, Activation
from keras.models import Model
import numpy as np
import matplotlib.pyplot as plt
import keras.backend as K
ipt = Input(shape=(8,))
x = Dense(10, activation=None)(ipt)
x = Activation('relu')(x)
out = Dense(1, activation='sigmoid')(x)
model = Model(ipt, out)
model.compile('adam', 'binary_crossentropy')
X = np.random.randn(16, 8)
outs1 = get_layer_outputs(model, model.layers[1], X, 1) # Dense
outs2 = get_layer_outputs(model, model.layers[2], X, 1) # Activation
plt.hist(np.ndarray.flatten(outs1), bins=200); plt.show()
plt.hist(np.ndarray.flatten(outs2), bins=200); plt.show()
Function used:
def get_layer_outputs(model, layer, input_data, learning_phase=1):
layer_fn = K.function([model.input, K.learning_phase()], layer.output)
return layer_fn([input_data, learning_phase])
I am looking for a way where we can use tensorflow API to create a neural network with the number of layer and hidden units as user defined.
Lets say I have a neural network like this
hidden1 = tf.layers.dense(inp, units=32, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden1")
bn1 = tf.layers.batch_normalization(inputs=hidden1, name="bn1")
hidden2 = tf.layers.dense(bn1, units=16, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden2")
bn2 = tf.layers.batch_normalization(inputs=hidden2, name="bn2")
hidden3 = tf.layers.dense(bn2, units=8 , kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden3")
bn3 = tf.layers.batch_normalization(inputs=hidden3, name="bn3")
out = tf.layers.dense(bn3, units=1, kernel_initializer=tf.initializers.he_uniform(), activation=None, name="out")
In the above snippet you can notice, if I want 3 layers then I need to repeat the code for 3 times.
I am looking for a way, where we can use for loop to define the above code block. For example, if number of layers is defined as 3, then the for loop should iterate and assign units and activation value for each according to user defined.
# psuedocode
for i in range(number_of_layer):
hidden_(i) = tf.layers.dense(inp, units=32, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden_(i)")
bn_(i) = tf.layers.batch_normalization(inputs=hidden_(i), name="bn_(i)")
You can do it like this:
from keras.layers import Dense, BatchNormalization, Dropout
from keras.layers.advanced_activations import ReLU
from keras.models import Model
# Define the number of units per hidden layer
layer_widths = [128, 64, 32]
# Set up input layer
input_layer = Input(...) # change according to your input
x = input_layer.output
# Iteratively add the hidden layers
for n_neurons in layer_widths:
x = Dense(n_neurons)(x)
x = ReLU()(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
# Add the output layer
output = Dense(16, activation='softmax')(x) # change according to your output
# Stack the model together
model = Model(input, output)
Using tensorflow API
inp = tf.placeholder("float", [None,2],name="inp")
units = [32, 16, 8]
for unit in range(len(units)):
inp = tf.layers.dense(inp, units=units[unit], kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu,name="hidden" + str(unit + 1))
inp = tf.layers.batch_normalization(inputs=inp, name="bn"+str(unit + 1))
out = tf.layers.dense(inp, units=1, kernel_initializer=tf.initializers.he_uniform(), activation=None, name="out")
I am building a reinforcement learning algorithm in Tensorflow and I would like to be able to dynamically turn dropout off and then on within one single call to session.run().
Rationale: I need to (1) do a forward pass w/o dropout to calculate the targets; and (2) do a training step with the generated targets. If I execute these two steps in different calls to session.run(), everything is ok. But I would like to do it with one single call to session.run() (using tf.stop_gradients(targets)).
After trying several solutions w/o much success, I landed on a solution where I replace the learning_phase placeholder used by Keras with a variable (since placeholders are tensors and do not allow assignments) and use a custom layer to set that variable to True or False as desired. This solution is shown in the code below. Getting the value of either m1 or m2 separately (e.g., running sess.run(m1, feed_dict={ph:np.ones((1,1))})works as expected w/o error. However, getting the value of m3, or getting the values of m1 and m2 simultaneously, works sometimes and sometimes not (and the error message is uninformative).
Do you know what I am doing wrong or a better way to do what I want?
EDIT: The code shows a toy example. In reality I have a single model and I need to run two forward passes (one with dropout off and the other with dropout on) and one backward pass. And I want to do all this it w/o returning to python.
from tensorflow.keras.layers import Dropout, Dense, Input, Layer
from tensorflow.python.keras import backend as K
from tensorflow.keras import Model
import tensorflow as tf
import numpy as np
class DropoutSwitchLayer(Layer):
def __init__(self, stateful=True, **kwargs):
self.stateful = stateful
self.supports_masking = True
super(DropoutSwitchLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.lph = tf.Variable(True, dtype=tf.bool, name="lph", trainable=False)
K._GRAPH_LEARNING_PHASES[tf.get_default_graph()] = self.lph
super(DropoutSwitchLayer, self).build(input_shape)
def call(self, inputs, mask=None):
data_input, training = inputs
op = self.lph.assign(training[0], use_locking=True)
# ugly trick here to make the layer work
data_input = data_input + tf.multiply(tf.cast(op, dtype=tf.float32), 0.0)
return data_input
def compute_output_shape(self, input_shape):
return input_shape[0]
dropout_on = np.array([True], dtype=np.bool)
dropout_off = np.array([False], dtype=np.bool)
input_ph = tf.placeholder(tf.float32, shape=(None, 1))
drop = Input(shape=(), dtype=tf.bool)
input = Input(shape=(1,))
h = DropoutSwitchLayer()([input, drop])
h = Dense(1)(h)
h = Dropout(0.5)(h)
o = Dense(1)(h)
m = Model(inputs=[input, drop], outputs=o)
m1 = m([input_ph, dropout_on])
m2 = m([input_ph, dropout_off])
m3 = m([m2, dropout_on])
sess = tf.Session()
K.set_session(sess)
sess.run(tf.global_variables_initializer())
EDIT 2: Daniel Möller's solution below works when using a Dropout layer, but what if using dropout inside an LSTM layer?
input = Input(shape=(1,))
h = Dense(1)(input)
h = RepeatVector(2)(h)
h = LSTM(1, dropout=0.5, recurrent_dropout=0.5)(h)
o = Dense(1)(h)
Why not make a single continuous model?
#layers
inputs = Input(shape(1,))
dense1 = Dense(1)
dense2 = Dense(1)
#no drop pass:
h = dense1(inputs)
o = dense2(h)
#optionally:
o = Lambda(lambda x: K.stop_gradient(x))(o)
#drop pass:
h = dense1(o)
h = Dropout(.5)(h)
h = dense2(h)
modelOnlyFinalOutput = Model(inputs,h)
modelOnlyNonDrop = Model(inputs,o)
modelBothOutputs = Model(inputs, [o,h])
Choose one for training:
model.fit(x_train,y_train) #where y_train = [targets1, targets2] if using both outputs
It turns out Keras supports, out of the box, what I want to do. Using the training argument in the call to the Dropout/LSTM layer, in combination with Daniel Möller's approach to build the model (thanks!), does the trick.
In the code below (just a toy example), o1 and o3 should be equal and different than o2
from tensorflow.keras.layers import Dropout, Dense, Input, Lambda, Layer, Add, RepeatVector, LSTM
from tensorflow.python.keras import backend as K
from tensorflow.keras import Model
import tensorflow as tf
import numpy as np
repeat = RepeatVector(2)
lstm = LSTM(1, dropout=0.5, recurrent_dropout=0.5)
#Forward pass with dropout disabled
next_state = tf.placeholder(tf.float32, shape=(None, 1), name='next_state')
h = repeat(next_state)
# Use training to disable dropout
o1 = lstm(h, training=False)
target1 = tf.stop_gradient(o1)
#Forward pass with dropout enabled
state = tf.placeholder(tf.float32, shape=(None, 1), name='state')
h = repeat(state)
o2 = lstm(h, training=True)
target2 = tf.stop_gradient(o2)
#Forward pass with dropout disabled
ph3 = tf.placeholder(tf.float32, shape=(None, 1), name='ph3')
h = repeat(ph3)
o3 = lstm(h, training=False)
loss = target1 + target2 - o3
opt = tf.train.GradientDescentOptimizer(0.1)
train = opt.minimize(loss)
sess = tf.Session()
K.set_session(sess)
sess.run(tf.global_variables_initializer())
data = np.ones((1,1))
sess.run([o1, o2, o3], feed_dict={next_state:data, state:data, ph3:data})
How about this :
class CustomDropout(tf.keras.layers.Layer):
def __init__(self):
super(CustomDropout, self).__init__()
self.dropout1= Dropout(0.5)
self.dropout2= Dropout(0.1)
def call(self, inputs):
if xxx:
return self.dropout1(inputs)
else:
return self.dropout2(inputs)