TensorFlow: "Attempting to use uninitialized value" in variable initialization - python

I am trying to implement multivariate linear regression in Python using TensorFlow, but have run into some logical and implementation issues. My code throws the following error:
Attempting to use uninitialized value Variable
Caused by op u'Variable/read'
Ideally the weights output should be [2, 3]
def hypothesis_function(input_2d_matrix_trainingexamples,
output_matrix_of_trainingexamples,
initial_parameters_of_hypothesis_function,
learning_rate, num_steps):
# calculate num attributes and num examples
number_of_attributes = len(input_2d_matrix_trainingexamples[0])
number_of_trainingexamples = len(input_2d_matrix_trainingexamples)
#Graph inputs
x = []
for i in range(0, number_of_attributes, 1):
x.append(tf.placeholder("float"))
y_input = tf.placeholder("float")
# Create Model and Set Model weights
parameters = []
for i in range(0, number_of_attributes, 1):
parameters.append(
tf.Variable(initial_parameters_of_hypothesis_function[i]))
#Contruct linear model
y = tf.Variable(parameters[0], "float")
for i in range(1, number_of_attributes, 1):
y = tf.add(y, tf.multiply(x[i], parameters[i]))
# Minimize the mean squared errors
loss = tf.reduce_mean(tf.square(y - y_input))
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train = optimizer.minimize(loss)
#Initialize the variables
init = tf.initialize_all_variables()
# launch the graph
session = tf.Session()
session.run(init)
for step in range(1, num_steps + 1, 1):
for i in range(0, number_of_trainingexamples, 1):
feed = {}
for j in range(0, number_of_attributes, 1):
array = [input_2d_matrix_trainingexamples[i][j]]
feed[j] = array
array1 = [output_matrix_of_trainingexamples[i]]
feed[number_of_attributes] = array1
session.run(train, feed_dict=feed)
for i in range(0, number_of_attributes - 1, 1):
print (session.run(parameters[i]))
array = [[0.0, 1.0, 2.0], [0.0, 2.0, 3.0], [0.0, 4.0, 5.0]]
hypothesis_function(array, [8.0, 13.0, 23.0], [1.0, 1.0, 1.0], 0.01, 200)


Run this:
init = tf.global_variables_initializer()
sess.run(init)
Or (depending on the version of TF that you have):
init = tf.initialize_all_variables()
sess.run(init)


It's not 100% clear from the code example, but if the list initial_parameters_of_hypothesis_function is a list of tf.Variable objects, then the line session.run(init) will fail because TensorFlow isn't (yet) smart enough to figure out the dependencies in variable initialization. To work around this, you should change the loop that creates parameters to use initial_parameters_of_hypothesis_function[i].initialized_value(), which adds the necessary dependency:
parameters = []
for i in range(0, number_of_attributes, 1):
parameters.append(tf.Variable(
initial_parameters_of_hypothesis_function[i].initialized_value()))


There is another the error happening which related to the order when calling initializing global variables. I've had the sample of code has similar error FailedPreconditionError (see above for traceback): Attempting to use uninitialized value W
def linear(X, n_input, n_output, activation = None):
W = tf.Variable(tf.random_normal([n_input, n_output], stddev=0.1), name='W')
b = tf.Variable(tf.constant(0, dtype=tf.float32, shape=[n_output]), name='b')
if activation != None:
h = tf.nn.tanh(tf.add(tf.matmul(X, W),b), name='h')
else:
h = tf.add(tf.matmul(X, W),b, name='h')
return h
from tensorflow.python.framework import ops
ops.reset_default_graph()
g = tf.get_default_graph()
print([op.name for op in g.get_operations()])
with tf.Session() as sess:
# RUN INIT
sess.run(tf.global_variables_initializer())
# But W hasn't in the graph yet so not know to initialize
# EVAL then error
print(linear(np.array([[1.0,2.0,3.0]]).astype(np.float32), 3, 3).eval())
You should change to following
from tensorflow.python.framework import ops
ops.reset_default_graph()
g = tf.get_default_graph()
print([op.name for op in g.get_operations()])
with tf.Session() as
# NOT RUNNING BUT ASSIGN
l = linear(np.array([[1.0,2.0,3.0]]).astype(np.float32), 3, 3)
# RUN INIT
sess.run(tf.global_variables_initializer())
print([op.name for op in g.get_operations()])
# ONLY EVAL AFTER INIT
print(l.eval(session=sess))


Normally there are two ways of initializing variables, 1) using the sess.run(tf.global_variables_initializer()) as the previous answers noted; 2) the load the graph from checkpoint.
You can do like this:
sess = tf.Session(config=config)
saver = tf.train.Saver(max_to_keep=3)
try:
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.model_dir))
# start from the latest checkpoint, the sess will be initialized
# by the variables in the latest checkpoint
except ValueError:
# train from scratch
init = tf.global_variables_initializer()
sess.run(init)
And the third method is to use the tf.train.Supervisor. The session will be
Create a session on 'master', recovering or initializing the model as needed, or wait for a session to be ready.
sv = tf.train.Supervisor([parameters])
sess = sv.prepare_or_wait_for_session()


I want to give my resolution, it work when i replace the line [session = tf.Session()] with [sess = tf.InteractiveSession()]. Hope this will be useful to others.


run both:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())

Related

Training while loop in Tensorflow

I've attempted converting a Python-side training loop to Tensorflow to (hypothetically) make the code run faster - not having to pass control over to cpu constantly. However, I can't manage using tf.while_loop.
Here's the code that works:
import numpy as np
import tensorflow as tf
from tqdm import tqdm
from sklearn.datasets import load_iris
from sklearn.preprocessing import RobustScaler
x, y = load_iris(True)
x = RobustScaler().fit_transform(x)
shape = (10, 10)
max_epochs = 1000
graph = tf.Graph()
sess = tf.Session(graph=graph)
x = x.astype(np.float64)
# Construct graph
with graph.as_default():
weights = tf.get_variable(
'weights', shape, initializer=tf.constant_initializer, dtype=tf.float64
)
curr_epoch = tf.placeholder(dtype=tf.int64, shape=())
with tf.name_scope('data'):
data = tf.data.Dataset.from_tensor_slices(x)
data = data.shuffle(buffer_size=10000)
data = data.repeat(max_epochs)
data = data.batch(1)
data = data.make_one_shot_iterator().get_next()
with tf.name_scope('update'):
update_op = make_update_op(weights)
init = tf.global_variables_initializer()
sess.run(init)
for i in tqdm(range(max_epochs)):
for _ in range(x.shape[0]):
sess.run(update_op, feed_dict={
curr_epoch: i
})
np_weights = sess.run(weights)
print(np_weights) # Correctly prints an array of 150's.
Now, if I create an update function to pass tf.while_loop, an error is thrown.
def make_update_op(w):
return w.assign(
w + 0.001
)
# In the code above:
update_op = tf.while_loop(lambda _: True, make_update_op, (weights,), maximum_iterations=x.shape[0])
# No inner loop:
for i in tqdm(range(max_epochs)):
sess.run(update_op, feed_dict={
curr_epoch: i
})
Line 22, in make_update_op
return w.assign(
AttributeError: 'Tensor' object has no attribute 'assign'
I don't quite understand what is happening even after reading the documentation. weights is a Variable after all. What could be done to correctly make the training loop?

The tensor that you're trying to assign a new value within a while loop is a result of a sequence of multiple operations-tensors (operation is node in the graph, while tensor is a directed edge). In particular, the while loop will produce:
Variable/Read-->while/Enter-->while/Merge-->while/Switch-->while/Identity
What you're trying to assign here is a tensor while/Identity.
tf.while_loop is usually used to iterate over the dimensions of a tensor (also over the None - the unknown dimension). You're trying to iterate over the variables that are fully defined. You don't need to create a tf.while_loop for that. Just create operations that update each variable and group these operations together:
update_ops = [w.assign(w + 0.001) for w in weights]
update_op = tf.group(update_ops)
Now, when you execute the update_op with tf.Session() interface it will update all variables.
Example:
import tensorflow as tf
v1 = tf.Variable(tf.ones((1, 2), dtype=tf.float32))
v2 = tf.Variable(2*tf.ones((1, 3), dtype=tf.float32))
update_ops = [w.assign(w + 0.001) for w in [v1, v2]]
update_op = tf.group(update_ops)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print('before update:')
print(v1.eval(), v2.eval())
print('after update:')
sess.run(update_op) # <-- update your variables
print(v1.eval(), v2.eval())
# before update:
# [[1. 1.]] [[2. 2. 2.]]
# after update:
# [[1.001 1.001]] [[2.001 2.001 2.001]]

Turns out, all that was missing was the fact that one cannot assign to a variable inside a loop as Vlad pointed out. Instead, one can return the new value of a variable.
def make_update_op(w):
return w + 0.001
new_w = tf.while_loop(lambda _: True, make_update_op, (weights,), maximum_iterations=x.shape[0])
update_op = weights.assign(new_w)
To use more variables one would need to return the same amount from the function and unpack them in Python, but the principle is the same.
def make_update_op(w, d):
return w + 0.001, d
new_w, _ = tf.while_loop(lambda *_: True, make_update_op, (weights, data), maximum_iterations=x.shape[0])
update_op = weights.assign(new_w)

Why does this TensorFlow snippet throw an error in feeding?

Code
#!/usr/bin/env python3
import tensorflow as tf
import numpy as np
def customOps(n):
x = tf.placeholder(tf.float32)
v1 = tf.reduce_sum(x,1)
v2 = tf.reduce_sum(x,0)
v = tf.nn.softmax(tf.concat([v1, v2], 0))
index = np.argmax(v)
if index > n/3:
finalval = tf.norm(v1-v2, ord='euclidean')
else:
finalval = tf.norm(v1+v2, ord='euclidean')
return finalval
if __name__ == '__main__':
mat = np.asarray([[0, 1], [1, 0]], dtype = np.float32)
n = mat.shape[0]
finalVal = customOps(n)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
outVal = sess.run(finalVal, feed_dict={x:mat})
print(outVal)
sess.close()
Error Thrown
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder_5' with dtype float [[{{node Placeholder_5}} = Placeholder[dtype=DT_FLOAT, shape=<unknown>, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
The error is thrown at sess.run(init) line in the above snippet. I am feeding a float type array through feed_dict and I am not sure why the error is being thrown.
Where is the error and why?

Why the error:
Because you ran the same snippet multiple times in an unclean graph (i.e, your graph has multiple copies of the network).
The reason I can say this is the _5 at the end of the node name in the error message. TF assigns a default name to all tensors in the graph using incremental indices in case a name is already taken. Placeholder_5 means that in the same graph there is at least 5 Placeholder instances without a custom default name assigned which, given your code, should be impossible unless you called the function multiple times without cleaning up the graph.
How to fix it:
Run in a clean graph: Put tf.reset_default_graph() before finalVal = customOps(n).
Note: Your code has more issues than that (for example, you have x in the main branch, but x is a local variable of customOps), but the cause of the error you have is the one stated above.
Below you find a tested and working version of your code that addresses both issues.
import tensorflow as tf
import numpy as np
def customOps(n):
x = tf.placeholder(tf.float32)
v1 = tf.reduce_sum(x,1)
v2 = tf.reduce_sum(x,0)
v = tf.nn.softmax(tf.concat([v1, v2], 0))
index = np.argmax(v)
if index > n/3:
finalval = tf.norm(v1-v2, ord='euclidean')
else:
finalval = tf.norm(v1+v2, ord='euclidean')
return x, finalval
if __name__ == '__main__':
mat = np.asarray([[0, 1], [1, 0]], dtype = np.float32)
n = mat.shape[0]
tf.reset_default_graph()
x, finalVal = customOps(n)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
outVal = sess.run(finalVal, feed_dict={x:mat})
print(outVal)
sess.close()

Simplest ever apply_gradients throws 'op' attribute error

I am learning to use tensorflow library. Every time I try to use simplest (I guess) possible example of gradient-based learning I get the same error, which makes me stuck.
Here is the code:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
los = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for counter in range(100):
grd = sess.run(optim.compute_gradients(loss=los))
sess.run(optim.apply_gradients(grads_and_vars=grd))
On the last line, I get following error:
AttributeError: 'numpy.float32' object has no attribute 'op'.
Thank you in advance for any tips.

The input of apply_gradients must be a tensor not a numpy array. Because, it is not meant to be called at each step of the learning but just once when building your graph. In fact, it "creates" the "training step" in which variables are updated using the computed gradient
You should use that kind of code instead:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
los = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
grads, vars = zip(*optimizer.compute_gradients(loss))
train_step = optim.apply_gradients(grads_and_vars=grads)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for counter in range(100):
grd = sess.run(train_step, feed_dict = {...})

The tensorflow converts tensor into numpy array when stored and evaluated using sess.run(). The apply_gradients method of GradientDescentOptimizer raises TypeError since it requires tensor as input as described here. This causing problem in the original code.
for counter in range(100):
grd = sess.run(optim.compute_gradients(loss=los))
sess.run(optim.apply_gradients(grads_and_vars=grd))
The solution to this problem is to define all the operations before starting session.
There may be cases where gradients need to be post-processed before updating weights. All such post-processing logic need to be defined before session begins.
Here is a simple example:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
loss = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
grads = optim.compute_gradients(loss)
train_step = optim.apply_gradients(grads_and_vars=grads)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for counter in range(100):
sess.run(train_step)

Tensorflow program give different answers after deployed on aws lambda

I have wrote a program with Tensorflow that identifies a number of figures in an image. The model is trained with a function and then used with another function to label the figures. The training have been done on my computer and the resulting model upload to aws with the solve function.
I my computer it works well, but when create a lambda in aws it works strange and start giving different answers with the same test data.
The model in the solve function is this:
# Recreate neural network from model file generated during training
# input
x = tf.placeholder(tf.float32, [None, size_of_image])
# weights
W = tf.Variable(tf.zeros([size_of_image, num_chars]))
# biases
b = tf.Variable(tf.zeros([num_chars]))
The solve function code to label the figures is this:
for testi in range(captcha_letters_num):
# load model from file
saver = tf.train.import_meta_graph(model_path + '.meta',
clear_devices=True)
saver.restore(sess, model_path)
# Data to label
test_x = np.asarray(char_imgs[testi], dtype=np.float32)
predict_op = model(test_x, W, b)
op = sess.run(predict_op, feed_dict={x: test_x})
# find max probability from the probability distribution returned by softmax
max_probability = op[0][0]
max_probability_index = -1
for i in range(num_chars):
if op[0][i] > max_probability:
max_probability = op[0][i]
max_probability_index = i
# append it to final output
final_text += char_map_list[max_probability_index]
# Reset the model so it can be used again
tf.reset_default_graph()
With the same test data it gives different answers, don't know why.

Solved!
What I finally do was to keep the Session outside the loop and initialize the variables. After ending the loop, reset the graph.
saver = tf.train.Saver()
sess = tf.Session()
# Initialize variables
sess.run(tf.global_variables_initializer())
.
.
.
# passing each of the 5 characters through the NNet
for testi in range(captcha_letters_num):
# Data to label
test_x = np.asarray(char_imgs[testi], dtype=np.float32)
predict_op = model(test_x, W, b)
op = sess.run(predict_op, feed_dict={x: test_x})
# find max probability from the probability distribution returned by softmax
max_probability = op[0][0]
max_probability_index = -1
for i in range(num_chars):
if op[0][i] > max_probability:
max_probability = op[0][i]
max_probability_index = i
# append it to final output
final_text += char_map_list[max_probability_index]
# Reset the model so it can be used again
tf.reset_default_graph()
sess.close()

Tensorflow slicing based on variable

I've found that indexing still is an open issue in tensorflow (#206), so I'm wondering what I could use as a workaround at the moment. I want to index/slice a row/column of a matrix based on a variable that changes for every training example.
What I've tried so far:
Slicing based on placeholder (doesn't work)
The following (working) code slices based on a fixed number.
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
y = tf.slice(x,[0],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5]})
print(result)
However, it seems that I can't simply replace one of these fixed numbers with a tf.placeholder. The following code gives me the error "TypeError: List of Tensors when single Tensor expected."
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5],i:0})
print(result)
This sounds like the brackets around [i] are too much, but removing them doesn't help either. How to use a placeholder/variable as index?
Slicing based on python variable (doesn't backprop/update properly)
I've also tried using a normal python variable as index. This does not lead to an error, but the network doesn't learn anything while training. I suppose because the changing variable is not properly registered, the graph is malformed and updates don't work?
Slicing via one-hot vector + multiplication (works, but is slow)
One workaround I found is using a one-hot vector. Making a one-hot vector in numpy, passing this using a placeholder, then doing the slicing via matrix multiplication. This works, but is quite slow.
Any ideas how to efficiently slice/index based on a variable?

Slicing based on a placeholder should work just fine. It looks like you are running into a type error, due to some subtle issues of shapes and types. Where you have the following:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
...you should instead have:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,i,[1])
...and then you should feed i as [0] in the call to sess.run().
To make this a little clearer, I would recommend rewriting the code as follows:
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[None]) # 1-D tensor
i = tf.placeholder(tf.int32, shape=[1])
y = tf.slice(x, i, [1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x: [1, 2, 3, 4, 5], i: [0]})
print(result)
The additional shape arguments to the tf.placeholder op help to ensure that the values you feed have the appropriate shapes, and also that TensorFlow will raise an error if the shapes are not correct.

If you have an extra dimension, this works.
import tensorflow as tf
import numpy as np
def reorder0(e, i, length):
'''
e: a two dimensional tensor
i: a one dimensional int32 tensor, of shape (e.shape[0])
returns: a tensor of the same shape as e, where the jth entry is entry i[j] from e
'''
return tf.concat(
[ tf.expand_dims( e[i[j],:], axis=0) for j in range(length) ],
axis=0
)
e = tf.placeholder(tf.float32, shape=(2,3,5), name='e' ) # sentences, words, embedding
i = tf.placeholder(tf.int32, shape=(2,3), name='i' ) # for each word, index of parent
p = tf.concat(
[ tf.expand_dims(reorder0(e[k,:,:], i[k,:], 3), axis=0) for k in range(2) ],
axis=0,
name='p'
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(p, feed_dict={
e: [
( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
],
i: [ (1,1,1), (2,0,2)]
})
print(result)

If the sizes are not known when building the model, use TensorArray.
e = tf.placeholder(tf.float32, shape=(3,5) ) # words, embedding
i = tf.placeholder(tf.int32, shape=(3) ) # for each word, index of parent
#p = reorder0(e, i, 3)
a = tf.TensorArray(
tf.float32,
size=e.get_shape()[0],
dynamic_size=True,
infer_shape= True,
element_shape=e.get_shape()[1],
clear_after_read = False
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(
a.unstack(e).gather(i),
feed_dict={
e: ( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
#( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
i: (2,0,2)
}
)
print(result)

Categories

Resources