When using tf.layers.conv2d, setting the initializer is easy, it can be done through its parameter. But what if I use tf.nn.conv2d? I use this code. Is this equivalent to setting the kernel_initializer parameter in tf.layers.conv2d? Although the program runs without errors, I don't know how to verify whether it does what it is expected do.
with tf.name_scope('conv1_2') as scope:
kernel = tf.get_variable(initializer=tf.contrib.layers.xavier_initializer(),
shape=[3, 3, 32, 32], name='weights')
conv = tf.nn.conv2d(conv1_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
The operation underneath is the same (see here).
As for the kernel and its initialization, I took a glimpse in the code and it looked the same... the layers.conv2d call a tf.get_variable at the end of the day.
But I wanted to see it empirically, so here is a test code that declares a conv2d using each method (tf.layers.conv2d and tf.nn.conv2d), evaluates the initialized kernels and compares them.
I've arbitrarily set the things that shouldn't interfere in the comparison, such as an input tensor and the strides.
import tensorflow as tf
import numpy as np
# the way you described in your question
def _nn(input_tensor, initializer, filters, size):
kernel = tf.get_variable(
initializer=initializer,
shape=[size, size, 32, filters],
name='kernel')
conv = tf.nn.conv2d(
input=input_tensor,
filter=kernel,
strides=[1, 1, 1, 1],
padding='SAME')
return kernel
# the other way
def _layer(input_tensor, initializer, filters, size):
tf.layers.conv2d(
inputs=input_tensor,
filters=filters,
kernel_size=size,
kernel_initializer=initializer)
# 'conv2d/kernel:0' is the name of the generated kernel
return tf.get_default_graph().get_tensor_by_name('conv2d/kernel:0')
def _get_kernel(method):
# an isolated context for each conv2d
graph = tf.Graph()
sess = tf.Session(graph=graph)
with graph.as_default(), sess.as_default():
# important so that same randomness doesnt play a role
tf.set_random_seed(42)
# arbitrary input tensor with compatible shape
input_tensor = tf.constant(1.0, shape=[1, 64, 64, 32])
initializer = tf.contrib.layers.xavier_initializer()
kernel = method(
input_tensor=input_tensor,
initializer=initializer,
filters=32,
size=3)
sess.run(tf.global_variables_initializer())
return sess.run(kernel)
if __name__ == '__main__':
kernel_nn = _get_kernel(_nn)
kernel_layer = _get_kernel(_layer)
print('kernels are ', end='')
# compares shape and values
if np.array_equal(kernel_layer, kernel_nn):
print('exactly the same')
else:
print('not the same!')
And the output is... kernels are exactly the same.
The docs, btw: tf.nn.conv2d and tf.layers.conv2d.
Related
Hi I'm new to neural networks and I'm currently working on Tensoflow.
First I did the MNIST tutorial which worked quite well. Now I wanted to deepen the whole by means of an own network for Cifar10 in Google Colab. For this purpose I wrote the following code:
def conv2d(input, size, inputDim, outputCount):
with tf.variable_scope("conv2d"):
## -> This area causes problems <- ##
##########variant1
weight = tf.Variable(tf.truncated_normal([size, size, inputDim, outputCount], stddev=0.1),name="weight")
bias = tf.Variable( tf.constant(0.1, shape=[outputCount]),name="bias")
##########variant2
weight = tf.get_variable("weight", tf.truncated_normal([size, size, inputDim, outputCount], stddev=0.1))
bias = tf.get_variable("bias", tf.constant(0.1, shape=[outputCount]))
##################
conv = tf.nn.relu(tf.nn.conv2d(input, weight, strides=[1, 1, 1, 1], padding='SAME') + bias)
return conv
def maxPool(conv2d):....
def fullyConnect(input, inputSize, outputCount, relu):
with tf.variable_scope("fullyConnect"):
## -> This area causes problems <- ##
##########variant1
weight = tf.Variable( tf.truncated_normal([inputSize, outputCount], stddev=0.1),name="weight")
bias = tf.Variable( tf.constant(0.1, shape=[outputCount]),name="bias")
##########variant2
weight = tf.get_variable("weight", tf.truncated_normal([inputSize, outputCount], stddev=0.1))
bias = tf.get_variable("bias", tf.constant(0.1, shape=[outputCount]))
##################
fullyIn = tf.reshape(input, [-1, inputSize])
fullyCon = fullyIn
if relu:
fullyCon = tf.nn.relu(tf.matmul(fullyIn, weight) + bias)
return fullyCon
#Model Def.
def getVGG16A(grafic,width,height,dim):
with tf.name_scope("VGG16A"):
img = tf.reshape(grafic, [-1,width,height,dim])
with tf.name_scope("Layer1"):
with tf.variable_scope("Layer1"):
with tf.variable_scope("conv1"):
l1_c = conv2d(img,3, dim, 64)
with tf.variable_scope("mp1"):
l1_mp = maxPool(l1_c) #32 > 16
with tf.name_scope("Layer2"):
with tf.variable_scope("Layer2"):
with tf.variable_scope("conv1"):
l2_c = conv2d(l1_mp,3, 64, 128)
with tf.variable_scope("mp1"):
l2_mp = maxPool(l2_c) #16 > 8
with tf.name_scope("Layer6"):
with tf.variable_scope("Layer6"):
with tf.variable_scope("fully1"):
L6_fc1 = fullyConnect(l2_mp, 8*8*128 , 1024, True)
with tf.variable_scope("fully2"):
L6_fc2 = fullyConnect(L6_fc1, 1024, 1024, True)
keep_prob = tf.placeholder(tf.float32)
drop = tf.nn.dropout(L6_fc2, keep_prob)
with tf.variable_scope("fully3"):
L6_fc3 = fullyConnect(drop,1024, 3, False)
return L6_fc3, keep_prob
x = tf.placeholder(tf.float32, [None, 3072]) #input
y_ = tf.placeholder(tf.float32, [None, 3]) #output
# Build the graph for the deep net
y_conv, keep_prob = getVGG16A(x,32,32,3) #create Model
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-3).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for batch in getBatchData(prep_filter_dataBatch1,2): #a self-written method for custom batch return
train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.8})
print('test accuracy %g' % accuracy.eval(feed_dict={
x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))
For the definition of the tensorflow variables I first used variant1 (tf.variable).
This caused an overflow of the graphics memory after repeated execution.
Then I used variant2 (tf.get_variable). If I have understood the documentation correctly, this should use already existing variables if they exist.
But as soon as I do this I get the following error message:
TypeError: Tensor objects are not iterable when eager execution is not enabled. To iterate over this tensor use tf.map_fn.
I've been looking the hole day, but I haven't found an explanation for this.
Now I hope that there is someone here who can explain to me why this is not possible, or where I can find further information. The error message is getting me nowhere. I don't want a solution because I want to and have to understand this, because I want to write my bachelor thesis in the field of CNN.
Why can I use tf.variable but not tf.get_variable which should do the same?
Thanks for the help,
best regards, Pascal :)
I found my mistake.
I forgot the keyword initializer.
the correct line looks like this:
weight = tf.get_variable("weight",initializer=tf.truncated_normal([size, size, inputDim, outputCount], stddev=anpassung))
I am trying to implement a Fully Convolutional network (5 layers) in TensorFlow.
But after few time of training, all my logits fall to 0.
Did anyone have the same problem before ?
Here is how I implemented my CONV-ReLU-maxPOOL layer :
def conv_relu_layer (in_data, nb_filters, filter_shape) :
nb_in_channels = int (in_data_reshaped.shape[3])
conv_shape = [filter_shape[0], filter_shape[1],
nb_in_channels, nb_filters]
weights = tf.Variable (
tf.truncated_normal (conv_shape, mean=0., stddev=.05))
bias = tf.Variable (
tf.truncated_normal ([nb_filters], mean=0., stddev=1.))
output = tf.nn.conv2d (in_data_reshaped, weights,
[1,1,1,1], padding="SAME")
output += bias
output = tf.nn.relu (output)
return output
def conv_relu_pool_layer (in_data, nb_filters, filter_shape, pool_shape,
pooling=tf.nn.max_pool) :
conv_out = conv_relu_layer (in_data, nb_filters, filter_shape)
ksize = [1, pool_shape[0], pool_shape[1], 1]
strides = [1, pool_shape[0], pool_shape[1], 1]
return pooling (conv_out, ksize=ksize, strides=strides, padding="SAME")
Here is my network :
def create_network_5C (in_data, name="5C") :
c1 = conv_relu_pool_layer (in_data, 64, [5,5], [2,2])
c2 = conv_relu_pool_layer (c1, 128, [5,5], [2,2])
c3 = conv_relu_pool_layer (c2, 256, [5,5], [2,2])
c4 = conv_relu_pool_layer (c3, 64, [5,5], [2,2])
return conv_relu_layer (c4, 2, [5,5])
The loss function :
def loss (logits, labels, num_classes) :
with tf.name_scope('loss'):
logits = tf.reshape(logits, (-1, num_classes))
epsilon = tf.constant(value=1e-4)
labels = tf.to_float(tf.reshape(labels, (-1, num_classes)))
softmax = tf.nn.softmax(logits) + epsilon
cross_entropy = - tf.reduce_sum (
tf.multiply (labels * tf.log (softmax), head),
reduction_indices=[1])
cross_entropy_mean = tf.reduce_mean (cross_entropy)
tf.add_to_collection('losses', cross_entropy_mean)
loss = tf.add_n(tf.get_collection('losses'))
return loss
My main routine :
batch_size = 5
# Load data
x = tf.placeholder (tf.float32, [None, 416, 416, 3], name="x")
y = tf.placeholder (tf.float32, [None, 416, 416, 1], name="y")
# Contrast normalization and computation
x_gcn = tf.map_fn (lambda img : tf.image.per_image_standardization (img), x)
logits = create_network_5C (x_gcn)
# Having label at the same dimension as the output
y_p = tf.nn.avg_pool (tf.sign (y),
ksize=[1,16,16,1], strides=[1,16,16,1], padding="SAME")
y_rshp = tf.reshape (y_p, [batch_size, 416//16, 416//16])
y_bin = tf.cast (y_rshp > .5, tf.int32)
y_1hot = tf.one_hot (y_bin, 2)
# Compute error
error = loss (logits, y_1hot, 2)
optimizer = tf.train.AdamOptimizer (learning_rate=args.eta).minimize (error)
# Run the session
init_op = tf.global_variables_initializer ()
with tf.Session () as session :
session.run (init_op)
err, _ = session.run ([error, optimizer],
feed_dict={ x: image_batch,
y: label_batch })
I note that, if I reduce my network to 2 layers only, it won't drop the logits to 0, but it won't learn anything either. If I reduce it to 3 layers, it will drop to 0, but after a many iterations (while 5 layers drop to 0 in few batches).
Can this be linked to what is called "gradient vanish" ?
If it's relevant, my spec are : Ubuntu 16.04 - Python 3.6.4 - tensorflow 1.6.0
[EDIT] My problem really look like dead-ReLU, as mentioned here : StackOverflow : FCN training error, but my data is normalized (between something like -2 and +2, and I already tried to change the mean and stddev initial value of my weights and biases
[EDIT 2] I tried to replace the ReLUs with Leaky ReLU, or a softplus, in both cases, logits get stucked under 0.1 and loss stay between 0.6 and 0.7
Using some leaky relu was actually enough, then I just needed to let him train for a hudge amount of time.
Suppose I am trying to connect the output of a pooling layer to a dense layer. In order to do this, I need to flatten the pooled tensor. Consider the layers below:
def conv_layer(input, in_channels, out_channels, name="conv"):
w = tf.get_variable("W", initializer=tf.truncated_normal([3, 3, in_channels, out_channels], stddev=0.1))
b = tf.get_variable("B", initializer=tf.constant(0.1, shape=[out_channels]))
conv = tf.nn.conv2d(input, w, strides=[1,1,1,1], padding="SAME")
act = tf.nn.relu(conv + b)
return act
def pool_layer(input, name="pool"):
pool = tf.nn.max_pool(input, ksize=[1,2,2,1], strides=[1,2,2,1], padding="SAME")
return pool
def dense_layer(input, size_in, size_out, name="dense"):
w = tf.get_variable("W", initializer=tf.truncated_normal([size_in, size_out], stddev=0.1))
b = tf.get_variable("B", initializer=tf.constant(0.1, shape=[size_out]))
act = tf.nn.relu(tf.matmul(input, w) + b)
return act
I am using them to create a network:
def cnn_model(x):
x_image = tf.reshape(x, [-1, nseries, present_window, 1])
conv1 = conv_layer(x_image, 1, 32, "conv1")
pool1 = pool_layer(conv1, "pool1")
conv2 = conv_layer(pool1, 32, 64, "conv2")
pool2 = pool_layer(conv2, "pool2")
nflat = 17*15*64 # hard-coded
flat = tf.reshape(pool2, [-1, nflat])
yhat = dense_layer(flat, nflat, future_window, "dense1")
return yhat
As you can see I am hard-coding the variable nflat. How to avoid this?
If it's a tensor pool.get_shape() should work on Keras or Tensorflow.
This will actually return a tuple with the size of each dimension, so you need to choose from it, probably it's the 2nd in your case.
If input is actually your input (without any other layer), why are you max-pooling? aren't you looking for dropout ?
Indeed you will find a problem if your batch size is variable, since there's no way of telling the model the size of the reshape
I am trying out the Dataset API for my input pipeline shown in the TensorFlow documentation and use almost the same code:
tr_data = Dataset.from_tensor_slices((train_images, train_labels))
tr_data = tr_data.map(input_parser, NUM_CORES, output_buffer_size=2000)
tr_data = tr_data.batch(BATCH_SIZE)
tr_data = tr_data.repeat(EPOCHS)
iterator = dataset.make_one_shot_iterator()
next_example, next_label = iterator.get_next()
# Script throws error here
loss = model_function(next_example, next_label)
with tf.Session(...) as sess:
sess.run(tf.global_variables_initializer())
while True:
try:
train_loss = sess.run(loss)
except tf.errors.OutOfRangeError:
print("End of training dataset.")
break
This should be faster since it avoids using the slow feed_dicts. But I can't make it work with my model, which is a simplified LeNet architecture. The problem is the tf.layers.dense in my model_function() which expects an known input shape (I guess because it has to know the number of weights beforehand). But next_example and next_label only get their shape by running them in the session. Before evaluating them their shape is just undefined ?
Declaring the model_function() throws this error:
ValueError: The last dimension of the inputs to Dense should be
defined. Found None.
Right now, I don't know if I am using this Dataset API in the intended way or if there is a workaround.
Thanks in advance!
Edit 1:
Below is my model and it throws the error at the first dense layer
def conv_relu(input, kernel_shape):
# Create variable named "weights".
weights = tf.get_variable("weights", kernel_shape,
initializer=tf.random_normal_initializer())
# Create variable named "biases".
biases = tf.get_variable("biases", kernel_shape[3],
initializer=tf.constant_initializer(0.0))
conv = tf.nn.conv2d(input, weights,
strides=[1, 1, 1, 1], padding='VALID')
return tf.nn.relu(conv + biases)
def fully(input, output_dim):
assert len(input.get_shape())==2, 'Wrong input shape, need flattened tensor as input'
input_dim = input.get_shape()[1]
weight = tf.get_variable("weight", [input_dim, output_dim],
initializer=tf.random_normal_initializer())
bias = tf.get_variable('bias', [output_dim],
initializer=tf.random_normal_initializer())
fully = tf.nn.bias_add(tf.matmul(input, weight), bias)
return fully
def simple_model(x):
with tf.variable_scope('conv1'):
conv1 = conv_relu(x, [3,3,1,10])
conv1 = tf.nn.max_pool(conv1,[1,2,2,1],[1,2,2,1],'SAME')
with tf.variable_scope('conv2'):
conv2 = conv_relu(conv1, [3,3,10,10])
conv2 = tf.nn.max_pool(conv2,[1,2,2,1],[1,2,2,1],'SAME')
with tf.variable_scope('conv3'):
conv3 = conv_relu(conv2, [3,3,10,10])
conv3 = tf.nn.max_pool(conv3,[1,2,2,1],[1,2,2,1],'SAME')
flat = tf.contrib.layers.flatten(conv3)
with tf.variable_scope('fully1'):
fully1 = tf.layers.dense(flat, 1000)
fully1 = tf.nn.relu(fully1)
with tf.variable_scope('fully2'):
fully2 = tf.layers.dense(fully1, 100)
fully2 = tf.nn.relu(fully2)
with tf.variable_scope('output'):
output = tf.layers.dense(fully2, 4)
fully1 = tf.nn.relu(output)
return output
Edit 2:
Here you see the print of the tensors. Notice that next_example does not have a shape
next_example: Tensor("IteratorGetNext:0", dtype=float32)
next_label: Tensor("IteratorGetNext:1", shape=(?, 4), dtype=float32)
I found the answer myself.
Following this thread the easy fix is to just set the shape with tf.Tensor.set_shape if you know your image sizes beforehand.
def input_parser(img_path, label):
# read the img from file
img_file = tf.read_file(img_path)
img_decoded = tf.image.decode_image(img_file, channels=1)
img_decoded = tf.image.convert_image_dtype(img_decoded, dtype=tf.float32)
img_decoded.set_shape([90,160,1]) # This line was missing
return img_decoded, label
It would have been nice if the tensorflow documentation included this line.
I am currently trying to get familiar with the Tensorflow library and I have a rather fundamental question that bugs me.
While building a convolutional neural network for MNIST classification I tried to use my own model_fn. In which usually the following line occurs to reshape the input features.
x = tf.reshape(x, shape=[-1, 28, 28, 1]), with the -1 referring to the input batch size.
Since I use this node as input to my convolutional layer,
x = tf.reshape(x, shape=[-1, 28, 28, 1])
conv1 = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu)
does this mean that the size all my networks layers are dependent on the batch size?
I tried freezing and running the graph on a single test input, which will only work if I provide n=batch_size test images.
Can you give me a hint on how to make my network run on any input batchsize while predicting?
Also I guess using the tf.reshape node (see first node in cnn_layout) in the network definition is not the best input for serving.
I will append my network layer-up and the model_fn
def cnn_layout(features,reuse,is_training):
with tf.variable_scope('cnn',reuse=reuse):
# resize input to [batchsize,height,width,channel]
x = tf.reshape(features['x'], shape=[-1,30,30,1], name='input_placeholder')
# conv1, 32 filter, 5 kernel
conv1 = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu, name='conv1')
# pool1, 2 stride, 2 kernel
pool1 = tf.layers.max_pooling2d(conv1, 2, 2, name='pool1')
# conv2, 64 filter, 3 kernel
conv2 = tf.layers.conv2d(pool1, 64, 3, activation=tf.nn.relu, name='conv2')
# pool2, 2 stride, 2 kernel
pool2 = tf.layers.max_pooling2d(conv2, 2, 2, name='pool2')
# flatten pool2
flatten = tf.contrib.layers.flatten(pool2)
# fc1 with 1024 neurons
fc1 = tf.layers.dense(flatten, 1024, name='fc1')
# 75% dropout
drop = tf.layers.dropout(fc1, rate=0.75, training=is_training, name='dropout')
# output logits
output = tf.layers.dense(drop, 1, name='output_logits')
return output
def model_fn(features, labels, mode):
# setup two networks one for training one for prediction while sharing weights
logits_train = cnn_layout(features=features,reuse=False,is_training=True)
logits_test = cnn_layout(features=features,reuse=True,is_training=False)
# predictions
predictions = tf.round(tf.sigmoid(logits_test),name='predictions')
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# define loss and optimizer
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_train,labels=labels),name='loss')
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE, name='optimizer')
train = optimizer.minimize(loss, global_step=tf.train.get_global_step(),name='train')
# accuracy for evaluation
accuracy = tf.metrics.accuracy(labels=labels,predictions=predictions,name='accuracy')
# summarys for tensorboard
tf.summary.scalar('loss',loss)
# return training and evalution spec
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train,
eval_metric_ops={'accuracy':accuracy}
)
Thanks!
In the typical scenario, the rank of features['x'] is already going to be 4, with the outer dimension being the actual batch size, so there's no need to resize it.
Let me try to explain.
You haven't shown your serving_input_receiver_fn yet and there are several ways to do that, although in the end the principle is similar across them all. If you're using TensorFlow Serving, then you probably use build_parsing_serving_input_receiver_fn. It's informative to look at the source code:
def build_parsing_serving_input_receiver_fn(feature_spec,
default_batch_size=None):
serialized_tf_example = array_ops.placeholder(
dtype=dtypes.string,
shape=[default_batch_size],
name='input_example_tensor')
receiver_tensors = {'examples': serialized_tf_example}
features = parsing_ops.parse_example(serialized_tf_example, feature_spec)
return ServingInputReceiver(features, receiver_tensors)
So in your client, you're going to prepare a request that has one or more Examples in it (let's say the length is N). The server treats the serialized examples as a list of strings which get "fed" into the input_example_tensor placeholder. The shape (which is None) dynamically gets filled in to be the size of the list (N).
Then the parse_example op parses each item in the placeholder and out pops a Tensor for each feature whose outer dimension is N. In your case, you'll have x with shape=[N, 30, 30, 1].
(Note that other serving systems, such as CloudML Engine, do not operate on Example objects, but the principles are the same).
I just want to briefly provide my found solution. Since I did not want to build a scalable production grade model, but a simple model runner in python to execute my CNN locally.
To export the model I used,
input_size = 900
def serving_input_receiver_fn():
inputs = {"x": tf.placeholder(shape=[None, input_size], dtype=tf.float32)}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
model.export_savedmodel(
export_dir_base=model_dir,
serving_input_receiver_fn=serving_input_receiver_fn)
To load and run it (without needing the model definition again) I used the tensorflow predictor class.
from tensorflow.contrib import predictor
class TFRunner:
""" runs a frozen cnn graph """
def __init__(self,model_dir):
self.predictor = predictor.from_saved_model(model_dir)
def run(self, input_list):
""" runs the input list through the graph, returns output """
if len(input_list) > 1:
inputs = np.vstack(input_list)
predictions = self.predictor({"x": inputs})
elif len(input_list) == 1:
predictions = self.predictor({"x": input_list[0]})
else:
predictions = []
return predictions