I'm not sure how to modify my code to get keras activations. I've seen conflicting examples of K.function() inputs and am not sure if I'm getting outputs per layer our activations.
Here is my code
activity = 'Downstairs'
layer = 1
seg_x = create_segments_and_labels(df[df['ActivityEncoded']==mapping[activity]],TIME_PERIODS,STEP_DISTANCE,LABEL)[0]
get_layer_output = K.function([model_m.layers[0].input],[model_m.layers[layer].output])
layer_output = get_layer_output([seg_x])[0]
try:
ax = sns.heatmap(layer_output[0].transpose(),cbar=True,cbar_kws={'label':'Activation'})
except:
ax = sns.heatmap(layer_output.transpose(),cbar=True,cbar_kws={'label':'Activation','rotate':180})
ax.set_xlabel('Kernel',fontsize=30)
ax.set_yticks(range(0,len(layer_output[0][0])+1,10))
ax.set_yticklabels(range(0,len(layer_output[0][0])+1,10))
ax.set_xticks(range(0,len(layer_output[0])+1,5))
ax.set_xticklabels(range(0,len(layer_output[0])+1,5))
ax.set_ylabel('Filter',fontsize=30)
ax.xaxis.labelpad = 10
ax.set_title('Filter vs. Kernel\n(Layer=' + model_m.layers[layer].name + ')(Activity=' + activity + ')',fontsize=35)
Suggestions here on stack overflow just do it as I do:
Keras, How to get the output of each layer?
Example 4 adds k's learning phase to the mix but my output is still the same.
https://www.programcreek.com/python/example/93732/keras.backend.function
Am I getting output or activations? Documentation implies i might need layers.activations but I haven't made that work.
My code, or the code passing in learning phase both get this heatmap.
https://imgur.com/a/5fI6N0B
For layers defined as e.g. Dense(activation='relu'), layer.outputs will fetch the (relu) activations. To get layer pre-activations, you'll need to set activation=None (i.e. 'linear'), followed by an Activation layer. Example below.
from keras.layers import Input, Dense, Activation
from keras.models import Model
import numpy as np
import matplotlib.pyplot as plt
import keras.backend as K
ipt = Input(shape=(8,))
x = Dense(10, activation=None)(ipt)
x = Activation('relu')(x)
out = Dense(1, activation='sigmoid')(x)
model = Model(ipt, out)
model.compile('adam', 'binary_crossentropy')
X = np.random.randn(16, 8)
outs1 = get_layer_outputs(model, model.layers[1], X, 1) # Dense
outs2 = get_layer_outputs(model, model.layers[2], X, 1) # Activation
plt.hist(np.ndarray.flatten(outs1), bins=200); plt.show()
plt.hist(np.ndarray.flatten(outs2), bins=200); plt.show()
Function used:
def get_layer_outputs(model, layer, input_data, learning_phase=1):
layer_fn = K.function([model.input, K.learning_phase()], layer.output)
return layer_fn([input_data, learning_phase])
Related
I have a simple 2 layer model: 1 input, 1 output.
if I train the model on synthetic data like this :
x = np.linspace(0.5, 2.4, 8)
y = (x - 1.7) * (x - 1.3) - 10
then it's actually trying to curve and fit to it
but if I use my real data,
x = np.array([2.46512976, 2.49080819, 2.51648663, 2.54216506, 2.5678435 ,
2.59352193, 2.61920037, 2.6448788 ])
y = np.array([-1.03820421, -1.05035971, -1.05866843, -1.06354286, -1.06677236,
-1.06449571, -1.06124893, -1.055874 ])
then it goes in a straight line
what can I do to make it fit ?
code for network:
import numpy as np
import tensorflow as tf
from tensorflow import keras
from keras.models import Model
from keras.layers import Input, Dense
input_layer = Input(shape=(1,))
hidden11 = Dense(64, activation='tanh')(input_layer)
hidden22 = Dense(64, activation='tanh')(hidden11)
output_layer = Dense(1, activation='linear')(hidden22)
M = Model(input_layer, output_layer)
M.compile(loss='mse',optimizer='adam')
M.fit(x.reshape(-1,1), y, epochs=500)
I am training a feature extractor based on densenet, which looks like the following:
# Import the Sequential model and layers
from keras.models import Sequential
import keras
import tensorflow as tf
from keras.layers import Conv2D, MaxPooling2D, Lambda, Dropout, Concatenate
from keras.layers import Activation, Dropout, Flatten, Dense
import pandas as pd
from sklearn import preprocessing
import ast
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
size = 256
class DenseNetBase(tf.keras.Model):
def __init__(self, size, include_top = True):
super(DenseNetBase, self).__init__()
self.include_top = include_top
#base
self.base = tf.keras.applications.DenseNet201(weights='imagenet',include_top=False, pooling='avg',input_shape = (size,size,3))
#final layer
self.dense = Dense(1, activation='sigmoid', name='predictions')
def call(self, input_tensor):
input_image = input_tensor[0]
input_metafeatures = input_tensor[1]
#model
x = self.base(input_image)
if self.include_top:
x = self.dense(x)
return x
def build_graph(self):
x = self.base.input
y = tf.keras.Input(shape=(3,))
return tf.keras.Model(inputs=[x,y], outputs=self.call([x,y]))
I want to then take the DenseNetBase, keep the trained weights, but remove the final dense layer to use for extracting features. Simplified DenseClassifier looks like this:
class DenseClassifier(tf.keras.Model):
def __init__(self, size, feature_extractor):
super(DenseClassifier, self).__init__()
#base tf.keras.layers.Input(shape=(size,size,3))
self.feature_extractor = tf.keras.Model(inputs = tf.keras.Input(shape=(size,size,3)), outputs = feature_extractor.layers[-2].output)
#final layer
self.dense = Dense(1, activation='sigmoid', name='prediction')
def call(self, input_tensor):
input_image = input_tensor[0]
input_metafeatures = input_tensor[1]
#model
x = self.feature_extractor(input_image)
return self.dense(x)
def build_graph(self):
x = self.base.input
y = tf.keras.Input(shape=(3,))
return tf.keras.Model(inputs=[x,y], outputs=self.call([x,y]))
Tying it together:
#build densenet feature extractor we have trained
denseBase = DenseNetBase(256, include_top = True)
denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase.load_weights('./models/DenseBaseSimple.h5')
#this doesn't work
DenseClassifier = DenseClassifier(size = 256, feature_extractor = denseBase)
In the above example, I get an error for the input which I am not sure why. The expected behaviour would be that I could build the latter model, and compile, and the existing weights DenseNetBase would be used for feature extraction.
I have tried to replace the input section with inputs = feature_extractor.layers[-2].input which does compile, but does not seem to evaluate to the same accuracy as denseBase even though it is using the same weights (in the simple example above with no extra layers).
My goal/question:
How can I load the weights from the pre-trained denseBase but remove the last dense layer (so the output is (None, 1920) as from DenseNet without the top but with my weights).
How can I then load this model without dense into another subclassed model as above to extract features.
Thanks!
To answer my own question, I did some testing looking at the values of initialised weights using logic from here:
It's what's expected. DenseBaseClassifier (using denseBase) and using imagenet weights both have similar prediction weight initialisations. This is because both these layers are randomly initialised and not trained, while the prediction layer in denseBase has been optimised and hence is different.
For the denseNet section, DenseBaseClassifier (using denseBase) == denseBase (some noise due to only saving weights), whereas original imagenet weights are different.
Using denseBase_featureextractor = tf.keras.Model(inputs = denseBase.layers[-2].input, outputs = denseBase.layers[-2].output) does indeed preserve the weights.
Not sure why self.feature_extractor = tf.keras.Model(inputs = tf.keras.Input(shape=(size,size,3)), outputs = feature_extractor.layers[-2].output) doesn't work though.
denseBase = DenseNetBase(size, include_top = True)
denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase.load_weights('./models/DenseBaseSimple.h5')
denseBase_featureextractor = tf.keras.Model(inputs = denseBase.layers[-2].input, outputs = denseBase.layers[-2].output)
DenseClassifier_denseBase = DenseClassifier(size = 256, feature_extractor = denseBase_featureextractor)
DenseClassifier_denseBase.build([(None, 256, 256, 3), (None,3)])
denseBase_imagenet = tf.keras.applications.DenseNet201(weights='imagenet',include_top=False, pooling='avg',input_shape = (size,size,3))
DenseClassifier_imagenet = DenseClassifier(size = 256, feature_extractor = denseBase_imagenet)
DenseClassifier_imagenet.build([(None, 256, 256, 3), (None,3)])
def get_weights_print_stats(layer):
W = layer.get_weights()
#print(len(W))
#for w in W:
# print(w.shape)
return W
def hist_weights(weights, title, bins=500):
for weight in weights[0:5]:
plt.hist(np.ndarray.flatten(weight), bins=bins)
plt.title(title)
fig = plt.figure(figsize=(15, 10))
fig.subplots_adjust(hspace=0.4, wspace=0.4)
W = get_weights_print_stats(denseBase.layers[1])
plt.subplot(2, 3, 1)
hist_weights(W, "denseBase")
y = plt.ylabel("Final prediction later weights")#, rotation="horizontal")
W = get_weights_print_stats(DenseClassifier_denseBase.layers[1])
plt.subplot(2, 3, 2)
hist_weights(W, "DenseBaseClassifier (using denseBase weights)")
W = get_weights_print_stats(DenseClassifier_imagenet.layers[1])
plt.subplot(2, 3, 3)
hist_weights(W, "DenseBaseClassifier (using imagenet weights)")
W = get_weights_print_stats(denseBase.layers[0])
plt.subplot(2, 3, 4)
hist_weights(W, "denseBase")
y = plt.ylabel("DenseNet base first 5 weights")#, rotation="horizontal")
W = get_weights_print_stats(DenseClassifier_denseBase.layers[0])
plt.subplot(2, 3, 5)
hist_weights(W, "DenseBaseClassifier (using denseBase weights)")
W = get_weights_print_stats(DenseClassifier_imagenet.layers[0])
plt.subplot(2, 3, 6)
hist_weights(W, "DenseBaseClassifier (using imagenet weights)")
I'm trying to follow the tutorial given here.
This tutorial trains a Keras model using a genetic algorithm, with the PyGAD package. I'm interested in the binary classification case. My input matrix is of dimension 10000x20. Hence, I've created the following model using Keras:
input_layer = tensorflow.keras.layers.Input(20)
dense_layer1 = tensorflow.keras.layers.Dense(500, activation="relu")(input_layer)
dense_layer2 = tensorflow.keras.layers.Dense(500, activation="relu")(dense_layer1)
output_layer = tensorflow.keras.layers.Dense(1, activation="softmax")(dense_layer2)
model = tensorflow.keras.Model(inputs=input_layer, outputs=output_layer)
keras_ga = pygad.kerasga.KerasGA(model=model,
num_solutions=10)
However, when I go to run the algorithm, using ga_instance.run(), I get the error:
ValueError: Shapes (10000,) and (10000, 1) are incompatible
I can't figure out why I'm getting this error? I want my Keras model to have 2 hidden layers, each with 500 hidden nodes and 1 output node.
I think the problem is related to how each output is represented in the array. if you have a single output for 10000 instances, then this is an example of preparing the data that works with PyGAD. Its shape is (1000, 1).
numpy.random.uniform(0, 1, (1000, 1))
Here is a code that works but for a simple network architecture because, based on the fitness function you used, the fitness sometimes is NaN.
As I do not have the same data you used, I generated the input/output data randomly.
import tensorflow.keras
import pygad.kerasga
import numpy
import pygad
def fitness_func(solution, sol_idx):
global data_inputs, data_outputs, keras_ga, model
model_weights_matrix = pygad.kerasga.model_weights_as_matrix(model=model,
weights_vector=solution)
model.set_weights(weights=model_weights_matrix)
predictions = model.predict(data_inputs)
cce = tensorflow.keras.losses.CategoricalCrossentropy()
solution_fitness = 1.0 / (cce(data_outputs, predictions).numpy() + 0.00000001)
# print("solution_fitness", cce(data_outputs, predictions).numpy(), solution_fitness)
return solution_fitness
def callback_generation(ga_instance):
print("Generation = {generation}".format(generation=ga_instance.generations_completed))
print("Fitness = {fitness}".format(fitness=ga_instance.best_solution(ga_instance.last_generation_fitness)[1]))
data_inputs = numpy.random.uniform(0, 1, (1000, 20))
data_outputs = numpy.random.uniform(0, 1, (1000, 1))
# create model
from tensorflow.keras.layers import Dense, Dropout
l1_rate=1e-6
l2_rate = 1e-6
input_layer = tensorflow.keras.layers.InputLayer(20)
dense_layer1 = tensorflow.keras.layers.Dense(10, activation="relu",kernel_regularizer=tensorflow.keras.regularizers.l1_l2(l1=l1_rate, l2=l2_rate))
output_layer = tensorflow.keras.layers.Dense(1, activation="sigmoid")
model = tensorflow.keras.Sequential()
model.add(input_layer)
model.add(dense_layer1)
model.add(Dropout(0.2))
model.add(output_layer)
keras_ga = pygad.kerasga.KerasGA(model=model,
num_solutions=10)
# Run pygad
num_generations = 30
num_parents_mating = 5
initial_population = keras_ga.population_weights
ga_instance = pygad.GA(num_generations=num_generations,
num_parents_mating=num_parents_mating,
initial_population=initial_population,
fitness_func=fitness_func,
on_generation=callback_generation)
ga_instance.run()
Thanks for using PyGAD!
I am looking for a way where we can use tensorflow API to create a neural network with the number of layer and hidden units as user defined.
Lets say I have a neural network like this
hidden1 = tf.layers.dense(inp, units=32, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden1")
bn1 = tf.layers.batch_normalization(inputs=hidden1, name="bn1")
hidden2 = tf.layers.dense(bn1, units=16, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden2")
bn2 = tf.layers.batch_normalization(inputs=hidden2, name="bn2")
hidden3 = tf.layers.dense(bn2, units=8 , kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden3")
bn3 = tf.layers.batch_normalization(inputs=hidden3, name="bn3")
out = tf.layers.dense(bn3, units=1, kernel_initializer=tf.initializers.he_uniform(), activation=None, name="out")
In the above snippet you can notice, if I want 3 layers then I need to repeat the code for 3 times.
I am looking for a way, where we can use for loop to define the above code block. For example, if number of layers is defined as 3, then the for loop should iterate and assign units and activation value for each according to user defined.
# psuedocode
for i in range(number_of_layer):
hidden_(i) = tf.layers.dense(inp, units=32, kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu, name="hidden_(i)")
bn_(i) = tf.layers.batch_normalization(inputs=hidden_(i), name="bn_(i)")
You can do it like this:
from keras.layers import Dense, BatchNormalization, Dropout
from keras.layers.advanced_activations import ReLU
from keras.models import Model
# Define the number of units per hidden layer
layer_widths = [128, 64, 32]
# Set up input layer
input_layer = Input(...) # change according to your input
x = input_layer.output
# Iteratively add the hidden layers
for n_neurons in layer_widths:
x = Dense(n_neurons)(x)
x = ReLU()(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
# Add the output layer
output = Dense(16, activation='softmax')(x) # change according to your output
# Stack the model together
model = Model(input, output)
Using tensorflow API
inp = tf.placeholder("float", [None,2],name="inp")
units = [32, 16, 8]
for unit in range(len(units)):
inp = tf.layers.dense(inp, units=units[unit], kernel_initializer=tf.initializers.he_uniform(),activation=tf.nn.relu,name="hidden" + str(unit + 1))
inp = tf.layers.batch_normalization(inputs=inp, name="bn"+str(unit + 1))
out = tf.layers.dense(inp, units=1, kernel_initializer=tf.initializers.he_uniform(), activation=None, name="out")
I am building a reinforcement learning algorithm in Tensorflow and I would like to be able to dynamically turn dropout off and then on within one single call to session.run().
Rationale: I need to (1) do a forward pass w/o dropout to calculate the targets; and (2) do a training step with the generated targets. If I execute these two steps in different calls to session.run(), everything is ok. But I would like to do it with one single call to session.run() (using tf.stop_gradients(targets)).
After trying several solutions w/o much success, I landed on a solution where I replace the learning_phase placeholder used by Keras with a variable (since placeholders are tensors and do not allow assignments) and use a custom layer to set that variable to True or False as desired. This solution is shown in the code below. Getting the value of either m1 or m2 separately (e.g., running sess.run(m1, feed_dict={ph:np.ones((1,1))})works as expected w/o error. However, getting the value of m3, or getting the values of m1 and m2 simultaneously, works sometimes and sometimes not (and the error message is uninformative).
Do you know what I am doing wrong or a better way to do what I want?
EDIT: The code shows a toy example. In reality I have a single model and I need to run two forward passes (one with dropout off and the other with dropout on) and one backward pass. And I want to do all this it w/o returning to python.
from tensorflow.keras.layers import Dropout, Dense, Input, Layer
from tensorflow.python.keras import backend as K
from tensorflow.keras import Model
import tensorflow as tf
import numpy as np
class DropoutSwitchLayer(Layer):
def __init__(self, stateful=True, **kwargs):
self.stateful = stateful
self.supports_masking = True
super(DropoutSwitchLayer, self).__init__(**kwargs)
def build(self, input_shape):
self.lph = tf.Variable(True, dtype=tf.bool, name="lph", trainable=False)
K._GRAPH_LEARNING_PHASES[tf.get_default_graph()] = self.lph
super(DropoutSwitchLayer, self).build(input_shape)
def call(self, inputs, mask=None):
data_input, training = inputs
op = self.lph.assign(training[0], use_locking=True)
# ugly trick here to make the layer work
data_input = data_input + tf.multiply(tf.cast(op, dtype=tf.float32), 0.0)
return data_input
def compute_output_shape(self, input_shape):
return input_shape[0]
dropout_on = np.array([True], dtype=np.bool)
dropout_off = np.array([False], dtype=np.bool)
input_ph = tf.placeholder(tf.float32, shape=(None, 1))
drop = Input(shape=(), dtype=tf.bool)
input = Input(shape=(1,))
h = DropoutSwitchLayer()([input, drop])
h = Dense(1)(h)
h = Dropout(0.5)(h)
o = Dense(1)(h)
m = Model(inputs=[input, drop], outputs=o)
m1 = m([input_ph, dropout_on])
m2 = m([input_ph, dropout_off])
m3 = m([m2, dropout_on])
sess = tf.Session()
K.set_session(sess)
sess.run(tf.global_variables_initializer())
EDIT 2: Daniel Möller's solution below works when using a Dropout layer, but what if using dropout inside an LSTM layer?
input = Input(shape=(1,))
h = Dense(1)(input)
h = RepeatVector(2)(h)
h = LSTM(1, dropout=0.5, recurrent_dropout=0.5)(h)
o = Dense(1)(h)
Why not make a single continuous model?
#layers
inputs = Input(shape(1,))
dense1 = Dense(1)
dense2 = Dense(1)
#no drop pass:
h = dense1(inputs)
o = dense2(h)
#optionally:
o = Lambda(lambda x: K.stop_gradient(x))(o)
#drop pass:
h = dense1(o)
h = Dropout(.5)(h)
h = dense2(h)
modelOnlyFinalOutput = Model(inputs,h)
modelOnlyNonDrop = Model(inputs,o)
modelBothOutputs = Model(inputs, [o,h])
Choose one for training:
model.fit(x_train,y_train) #where y_train = [targets1, targets2] if using both outputs
It turns out Keras supports, out of the box, what I want to do. Using the training argument in the call to the Dropout/LSTM layer, in combination with Daniel Möller's approach to build the model (thanks!), does the trick.
In the code below (just a toy example), o1 and o3 should be equal and different than o2
from tensorflow.keras.layers import Dropout, Dense, Input, Lambda, Layer, Add, RepeatVector, LSTM
from tensorflow.python.keras import backend as K
from tensorflow.keras import Model
import tensorflow as tf
import numpy as np
repeat = RepeatVector(2)
lstm = LSTM(1, dropout=0.5, recurrent_dropout=0.5)
#Forward pass with dropout disabled
next_state = tf.placeholder(tf.float32, shape=(None, 1), name='next_state')
h = repeat(next_state)
# Use training to disable dropout
o1 = lstm(h, training=False)
target1 = tf.stop_gradient(o1)
#Forward pass with dropout enabled
state = tf.placeholder(tf.float32, shape=(None, 1), name='state')
h = repeat(state)
o2 = lstm(h, training=True)
target2 = tf.stop_gradient(o2)
#Forward pass with dropout disabled
ph3 = tf.placeholder(tf.float32, shape=(None, 1), name='ph3')
h = repeat(ph3)
o3 = lstm(h, training=False)
loss = target1 + target2 - o3
opt = tf.train.GradientDescentOptimizer(0.1)
train = opt.minimize(loss)
sess = tf.Session()
K.set_session(sess)
sess.run(tf.global_variables_initializer())
data = np.ones((1,1))
sess.run([o1, o2, o3], feed_dict={next_state:data, state:data, ph3:data})
How about this :
class CustomDropout(tf.keras.layers.Layer):
def __init__(self):
super(CustomDropout, self).__init__()
self.dropout1= Dropout(0.5)
self.dropout2= Dropout(0.1)
def call(self, inputs):
if xxx:
return self.dropout1(inputs)
else:
return self.dropout2(inputs)