Does tensorflow need to set the reuse ==True when finish training and inference?
I have a network like this:
def __build_net(self,placeholder,reuse=False):
with tf.variable_scope('siamse',reuse=reuse):
layer = tf.layers.dense(placeholder,3000,activation=tf.nn.leaky_relu)
layer = tf.layers.batch_normalization(layer)
embedding= tf.layers.dense(layer,300,activation = tf.nn.leaky_relu)
print('Siamse Net has built',flush=True)
return embedding
And I create two network share same parameter:
self.embedding1=self.__build_net(self.centers_placeholder)
self.embedding2=self.__build_net(self.neighbors_placeholder,reuse=True)
I used this network to generate embeddings of some kind of data.
My question is: Do I need to set the reuse to True when doing inference(generate embedding) like this:
with tf.Session() as sess:
self.saver.restore(sess,self.store_path+self.model_type+'_model_'+str(self.model_num)+'_'+str(self.center_size)+'_'+str(self.neighbor_size)+'.ckpt')
embedding = self.__build_net(self.centers_placeholder,reuse=True)
embeddings = sess.run(embedding,feed_dict = {self.centers_placeholder : data})
Or like this:
with tf.Session() as sess:
self.saver.restore(sess,self.store_path+self.model_type+'_model_'+str(self.model_num)+'_'+str(self.center_size)+'_'+str(self.neighbor_size)+'.ckpt')
embedding = self.__build_net(self.centers_placeholder,reuse=False)
embeddings = sess.run(embedding,feed_dict = {self.centers_placeholder : data})
And then, When set the variable scope, do I need to give a name to each layer?
Thanks!
No....reuse means whether you need to use a previously defined variable.
Say, you've created a variable called 'foo/v':
with tf.variable_scope("foo"):
v = tf.get_variable("v", [1])
print(v.name) ---> foo/v:0
Running the following will give:
with tf.variable_scope("foo"):
v1 = tf.get_variable("v", [1]) ---> gives error as name 'foo/v' exists
print(v1.name)
with tf.variable_scope("foo", reuse=False):
v1 = tf.get_variable("v", [1]) ---> gives error as name 'foo/v' exists
print(v1.name)
with tf.variable_scope("foo", reuse=True):
v1 = tf.get_variable("v", [1])
print(v1.name) ---> foo/v:0
with tf.variable_scope("foo", reuse=tf.AUTO_REUSE):
v1 = tf.get_variable("v", [1])
print(v1.name) ---> foo/v:0
But if you run the following from the very begining:
with tf.variable_scope("foo", reuse=True):
v1 = tf.get_variable("v", [1])
print(v1.name) ---> gives error as 'foo/v' does not exist (thus cannot be reused).
Thus I prefer setting reuse=tf.AUTO_REUSE all the time.
For a detailed explanation, please read How Does Variable Scope Work? from the TensorFlow official guide.
By the way, tf.layers.batch_normalization has a training option that needs to be set False during inference. See the explanations here.
I'm trying to load a saved model by saver and evaluate the output of the network over the test data, restoring a saved model and using the network as a function:
def model(x, x_size):
with tf.variable_scope("my_net", reuse=tf.AUTO_REUSE):
W1 = tf.get_variable('w1', [x_size, x_size],
initializer=tf.random_normal_initializer())
b1 = tf.get_variable('b1', [x_size],
initializer=tf.random_normal_initializer())
y1 = tf.nn.relu(tf.matmul(x, W1) + b1,'y1')
return y1
eval_x = tf.placeholder(tf.float32, name='eval_x', shape=[None, x_size])
eval_probs = model(eval_x, x_size)
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph('save_model/model.ckpt.meta')
new_saver.restore(sess, "save_model/model.ckpt")
probs = sess.run(eval_probs, feed_dict={eval_x: test_x})
The error I get is:
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value my_net/w1
[[Node: my_net/w1/read = IdentityT=DT_FLOAT, _class=["loc:#my_net/w1"], _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Can anybody help?
You need to run a special operation to initialize variables like w1. An easy way to do this is to execute sess.run(tf.global_variables_initializer()) after you create the session but before you use your variables.
From the code it seems that you are not passing any value for
x_size
so I don't think your variables get initialised.
I am trying to run the following tensorflow code and it's working fine the first time. If I try running it again, it keeps throwing an error saying
ValueError: Variable layer1/weights1 already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
File "C:\Users\owner\Anaconda3\envs\DeepLearning_NoGPU\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
File "C:\Users\owner\Anaconda3\envs\DeepLearning_NoGPU\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Users\owner\Anaconda3\envs\DeepLearning_NoGPU\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
If I restart the console and then run it, once again it runs just fine.
Given below is my implementation of the neural network.
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
import tensorflow as tf
learning_rate = 0.001
training_epochs = 100
n_input = 9
n_output = 1
n_layer1_node = 100
n_layer2_node = 100
X_train = np.random.rand(100, 9)
y_train = np.random.rand(100, 1)
with tf.variable_scope('input'):
X = tf.placeholder(tf.float32, shape=(None, n_input))
with tf.variable_scope('output'):
y = tf.placeholder(tf.float32, shape=(None, 1))
#layer 1
with tf.variable_scope('layer1'):
weight_matrix1 = {'weights': tf.get_variable(name='weights1',
shape=[n_input, n_layer1_node],
initializer=tf.contrib.layers.xavier_initializer()),
'biases': tf.get_variable(name='biases1',
shape=[n_layer1_node],
initializer=tf.zeros_initializer())}
layer1_output = tf.nn.relu(tf.add(tf.matmul(X, weight_matrix1['weights']), weight_matrix1['biases']))
#Layer 2
with tf.variable_scope('layer2'):
weight_matrix2 = {'weights': tf.get_variable(name='weights2',
shape=[n_layer1_node, n_layer2_node],
initializer=tf.contrib.layers.xavier_initializer()),
'biases': tf.get_variable(name='biases2',
shape=[n_layer2_node],
initializer=tf.zeros_initializer())}
layer2_output = tf.nn.relu(tf.add(tf.matmul(layer1_output, weight_matrix2['weights']), weight_matrix2['biases']))
#Output layer
with tf.variable_scope('layer3'):
weight_matrix3 = {'weights': tf.get_variable(name='weights3',
shape=[n_layer2_node, n_output],
initializer=tf.contrib.layers.xavier_initializer()),
'biases': tf.get_variable(name='biases3',
shape=[n_output],
initializer=tf.zeros_initializer())}
prediction = tf.nn.relu(tf.add(tf.matmul(layer2_output, weight_matrix3['weights']), weight_matrix3['biases']))
cost = tf.reduce_mean(tf.squared_difference(prediction, y))
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
for epoch in range(training_epochs):
session.run(optimizer, feed_dict={X: X_train, y: y_train})
train_cost = session.run(cost, feed_dict={X: X_train, y:y_train})
print(epoch, " epoch(s) done")
print("training complete")
As the error suggests I tried adding reuse=True as a parameter in with tf.variable_scope(): but that again is not working.
I am running this inside a conda environment. I am using Python 3.5 and CUDA 8(But it shouldn't matter because this is not configured to run in the GPU) in windows 10.
This is a matter of how TF works. One needs to understand that TF has a "hidden" state - a graph being built. Most of the tf functions create ops in this graph (like every tf.Variable call, every arithmetic operation and so on). On the other hand actual "execution" happens in the tf.Session(). Consequently your code will usually look like this:
build_graph()
with tf.Session() as sess:
process_something()
since all actual variables, results etc. leave in session only, if you want to "run it twice" you would do
build_graph()
with tf.Session() as sess:
process_something()
with tf.Session() as sess:
process_something()
Notice that I am building graph once. Graph is an abstract representation of how things look like, it does not hold any state of computations. When you try to do
build_graph()
with tf.Session() as sess:
process_something()
build_graph()
with tf.Session() as sess:
process_something()
you might get errors during second build_graph() due to trying to create variables with the same names (what happens in your case), graph being finalised etc. If you really need to run things this way you simply have to reset graph in between
build_graph()
with tf.Session() as sess:
process_something()
tf.reset_default_graph()
build_graph()
with tf.Session() as sess:
process_something()
will work fine.
How can I assign a value to a tf Variable inside a function?
Based on the link here, it say thats you have to run a sess on the tf tensor. I want to update the tf variable inside the function after few calculations.
Example:
def update(weights):
value_1 = 0
value_2 = 2
........... some code here ...........
weights['layer_1'] = tf.multiply(weights['layer_1'],value_1)
weights['layer_2'] = tf.multiply(weights['layer_2'],value_2)
............some code here.............
I can't do the above code. But how do I use assign to make this code work?
You have to use assign, which take a Tensor which has to be exactly the same shape as the original Variable. If you want to have different shape use the validate_shape=False. But you have to keep in mind that you'll get the actual changes on run time, thus you will code the behavior of your variable not assigning values.
Here an example that shows variable assignment with variable shapes:
import tensorflow as tf
var = tf.Variable(tf.zeros((1, 3)))
new_v = tf.assign(var, tf.ones((5, 7)), validate_shape=False)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
print sess.run([var])
print sess.run([new_v])
For your particular example you could try:
def update(weights):
value_1 = tf.constant(0)
value_2 = tf.constant(2)
........... some code here ...........
weights['layer_1'] = tf.assign(weights['layer_1'], tf.multiply(weights['layer_1'],value_1))
weights['layer_2'] = tf.assign(weights['layer_2'], tf.multiply(weights['layer_2'],value_2))
............some code here.............
This works for me -
import tensorflow as tf
import numpy as np
# function to randomly initialize weights for a specific layer
def assign_var(layer_number):
weight_value = np.random.rand(5,3) # or any calculations you need
weight_var = tf.get_variable('weights_layer_'+str(layer_number))
return tf.assign(weight_var,weight_value)
with tf.Session() as sess:
sess.run(assign_var(1))
sess.run(assign_var(2))
EDIT The problem with the above code is - it keeps adding to the graph every time you call the function.
Alternatively, I think this should be better.
import tensorflow as tf
import numpy as np
var_name = tf.placeholder(tf.string)
weight_value = tf.placeholder(tf.float32)
weight_var = tf.get_variable(var_name)
assign_weights = tf.assign(weight_var,weight_value)
sess = tf.Session()
# function to randomly initialize weights for a specific layer
def assign_var(layer_number):
rand_weight_value = np.random.rand(5,3) # or any calculations you need
sess.run(assign_weights,{var_name:'weights_layer'+str(layer_number),weight_value:rand_weight_value})
assign_var(1) # assigns random weight values to layer 1
I've found that indexing still is an open issue in tensorflow (#206), so I'm wondering what I could use as a workaround at the moment. I want to index/slice a row/column of a matrix based on a variable that changes for every training example.
What I've tried so far:
Slicing based on placeholder (doesn't work)
The following (working) code slices based on a fixed number.
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
y = tf.slice(x,[0],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5]})
print(result)
However, it seems that I can't simply replace one of these fixed numbers with a tf.placeholder. The following code gives me the error "TypeError: List of Tensors when single Tensor expected."
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5],i:0})
print(result)
This sounds like the brackets around [i] are too much, but removing them doesn't help either. How to use a placeholder/variable as index?
Slicing based on python variable (doesn't backprop/update properly)
I've also tried using a normal python variable as index. This does not lead to an error, but the network doesn't learn anything while training. I suppose because the changing variable is not properly registered, the graph is malformed and updates don't work?
Slicing via one-hot vector + multiplication (works, but is slow)
One workaround I found is using a one-hot vector. Making a one-hot vector in numpy, passing this using a placeholder, then doing the slicing via matrix multiplication. This works, but is quite slow.
Any ideas how to efficiently slice/index based on a variable?
Slicing based on a placeholder should work just fine. It looks like you are running into a type error, due to some subtle issues of shapes and types. Where you have the following:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
...you should instead have:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,i,[1])
...and then you should feed i as [0] in the call to sess.run().
To make this a little clearer, I would recommend rewriting the code as follows:
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[None]) # 1-D tensor
i = tf.placeholder(tf.int32, shape=[1])
y = tf.slice(x, i, [1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x: [1, 2, 3, 4, 5], i: [0]})
print(result)
The additional shape arguments to the tf.placeholder op help to ensure that the values you feed have the appropriate shapes, and also that TensorFlow will raise an error if the shapes are not correct.
If you have an extra dimension, this works.
import tensorflow as tf
import numpy as np
def reorder0(e, i, length):
'''
e: a two dimensional tensor
i: a one dimensional int32 tensor, of shape (e.shape[0])
returns: a tensor of the same shape as e, where the jth entry is entry i[j] from e
'''
return tf.concat(
[ tf.expand_dims( e[i[j],:], axis=0) for j in range(length) ],
axis=0
)
e = tf.placeholder(tf.float32, shape=(2,3,5), name='e' ) # sentences, words, embedding
i = tf.placeholder(tf.int32, shape=(2,3), name='i' ) # for each word, index of parent
p = tf.concat(
[ tf.expand_dims(reorder0(e[k,:,:], i[k,:], 3), axis=0) for k in range(2) ],
axis=0,
name='p'
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(p, feed_dict={
e: [
( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
],
i: [ (1,1,1), (2,0,2)]
})
print(result)
If the sizes are not known when building the model, use TensorArray.
e = tf.placeholder(tf.float32, shape=(3,5) ) # words, embedding
i = tf.placeholder(tf.int32, shape=(3) ) # for each word, index of parent
#p = reorder0(e, i, 3)
a = tf.TensorArray(
tf.float32,
size=e.get_shape()[0],
dynamic_size=True,
infer_shape= True,
element_shape=e.get_shape()[1],
clear_after_read = False
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(
a.unstack(e).gather(i),
feed_dict={
e: ( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
#( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
i: (2,0,2)
}
)
print(result)