Now i'm trying lstm tutorial, look some one's book. But it didn't work. What's the problem? :
import tensorflow as tf
import numpy as np
from tensorflow.contrib import rnn
import pprint
pp = pprint.PrettyPrinter(indent=4)
sess = tf.InteractiveSession()
a = [1, 0, 0, 0]
b = [0, 1, 0, 0]
c = [0, 0, 1, 0]
d = [0, 0, 0, 1]
init=tf.global_variables_initializer()
with tf.variable_scope('one_cell') as scope:
hidden_size = 2
cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden_size)
print(cell.output_size, cell.state_size)
x_data = np.array([[a]], dtype=np.float32)
pp.pprint(x_data)
outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)
sess.run(init)
pp.pprint(outputs.eval())
Error message is like that. Please solve this problem.
Attempting to use uninitialized value one_cell/rnn/basic_rnn_cell/weights
[[Node: one_cell/rnn/basic_rnn_cell/weights/read = Identity[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](one_cell/rnn/basic_rnn_cell/weights)]]
You haven't initialized some graph variables, as the error mentioned. Shift your code to this and it will work.
outputs, _states = tf.nn.dynamic_rnn(cell, x_data, dtype=tf.float32)
init=tf.global_variables_initializer()
sess.run(init)
Best practice is to have init right at the end of your graph and before sess.run.
EDIT: Refer to What does tf.global_variables_initializer() do under the hood? for more insights.
You define the operation init before creating your variables. Thus this operation will be performed only on the variables defined at that time, even if you run it after creating your variables.
So just move the definition of init and you will be fine.
Related
I am trying to get the maximum indexes of logits using tf.argmax() function. My code is show below:
import tensorflow as tf
import numpy as np
logits = tf.random_uniform([1,3,3,21], maxval=255, dtype=tf.float32, seed=0)
logits = logits / tf.norm(logits)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
logits_eval = sess.run(logits)
logits_argmax_np = np.argmax(logits_eval, axis=-1)
ypredT = tf.argmax(logits, axis=-1)
logits_argmax_tf = ypredT.eval()
I can get the right indexes using np.argmax(), but I don't whytf.argmax() is returning wrong indexes.
Thanks a lot in advance.
Edit: I am using tensorflow 1.13
The issue here is that each time you call sess.run you are instigating a separate execution of the session, from the bottom up. Since there are random numbers generated this will not produce the same result in each run, hence your argmax for each run is different. But they are doing the same thing.
To see this you can get both argmaxs from the same session execution, using square brackets to get both the ypredT tensor and the logits tensors from the same sess.run:
# tensorflow graph
logits = tf.random_uniform([1,3,3,21], maxval=255, dtype=tf.float32, seed=0)
logits = logits / tf.norm(logits)
ypredT = tf.argmax(logits, axis=-1) # tensorflow argmax
# run session
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
logits_eval, logits_argmax_tf = sess.run([logits, ypredT])
# after session has closed
logits_argmax_np = np.argmax(logits_eval, axis=-1) # get numpy argmax
print(logits_argmax_np)
print(logits_argmax_tf)
output:
[[[14 0 4]
[ 0 8 0]
[10 12 3]]]
[[[14 0 4]
[ 0 8 0]
[10 12 3]]]
I am trying to understand what tf.layers.dense does to an array and am using the code below. However, I get an error while running the code.
I tried debugging and it seems that there might be some issue while computing the rank of the tensor. However, sess.run(tf.rank(a)) successfully returns 3. So I suppose there is some other issue with the tensor itself.
import numpy as np
import tensorflow as tf
a = np.array([[[1, 0, 0], [1, 1, 0]], [[0, 0, 0], [0, 1, 1]]])
hidden_layer = tf.layers.dense(a, 5, activation=tf.nn.relu)
sess = tf.Session()
print(sess.run(hidden_layer))
The above code throws the error AttributeError: 'tuple' object has no attribute 'ndims', but I expect that a fully connected layer with weights and biases should be created.
What am I doing wrong?
Also, it would be really helpful if someone can maybe show a Python/NumPy equivalent of this implementation (without using tensorflow's dense), so that it's intuitive to follow.
There are two problems with the code: The Dense layer expects a Tensor as input instead of a Numpy array and the weights of the hidden layer need to be explicitly initialized. Here is the corrected code with some comments:
import numpy as np
import tensorflow as tf
# Make sure Dense layer is always initialized with the same values.
tf.set_random_seed(0)
# Dense layer will need float input
a = np.array([[[1, 0, 0], [1, 1, 0]], [[0, 0, 0], [0, 1, 1]]],
dtype=np.float32)
# Convert numpy array to Tensor
t = tf.convert_to_tensor(a)
# Create hidden layer with the array as input and random initializer that
# draws from a uniform distribution.
hidden_layer = tf.layers.dense(t, 5, activation=tf.nn.relu)
sess = tf.Session()
# Initialize Dense layer with the random initializer
sess.run(tf.global_variables_initializer())
# Print result of running the array through the Dense layer
print(sess.run(hidden_layer))
Btw as long as you're experimenting, you might benefit from using TensorFlow in eager mode or using PyTorch which has a friendlier interface.
First, you should reshape a. Then input the a to hidden layer. And you should initialize the parameter.
import numpy as np
import tensorflow as tf
a = np.array([[[1, 0, 0], [1, 1, 0]], [[0, 0, 0], [0, 1, 1]]], dtype=np.int32)
a = tf.reshape(a, [-1, 4*3])
hidden_layer = tf.layers.dense(a, 5, activation=tf.nn.relu)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print(sess.run(hidden_layer))
Code
#!/usr/bin/env python3
import tensorflow as tf
import numpy as np
def customOps(n):
x = tf.placeholder(tf.float32)
v1 = tf.reduce_sum(x,1)
v2 = tf.reduce_sum(x,0)
v = tf.nn.softmax(tf.concat([v1, v2], 0))
index = np.argmax(v)
if index > n/3:
finalval = tf.norm(v1-v2, ord='euclidean')
else:
finalval = tf.norm(v1+v2, ord='euclidean')
return finalval
if __name__ == '__main__':
mat = np.asarray([[0, 1], [1, 0]], dtype = np.float32)
n = mat.shape[0]
finalVal = customOps(n)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
outVal = sess.run(finalVal, feed_dict={x:mat})
print(outVal)
sess.close()
Error Thrown
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder_5' with dtype float [[{{node Placeholder_5}} = Placeholder[dtype=DT_FLOAT, shape=<unknown>, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
The error is thrown at sess.run(init) line in the above snippet. I am feeding a float type array through feed_dict and I am not sure why the error is being thrown.
Where is the error and why?
Why the error:
Because you ran the same snippet multiple times in an unclean graph (i.e, your graph has multiple copies of the network).
The reason I can say this is the _5 at the end of the node name in the error message. TF assigns a default name to all tensors in the graph using incremental indices in case a name is already taken. Placeholder_5 means that in the same graph there is at least 5 Placeholder instances without a custom default name assigned which, given your code, should be impossible unless you called the function multiple times without cleaning up the graph.
How to fix it:
Run in a clean graph: Put tf.reset_default_graph() before finalVal = customOps(n).
Note: Your code has more issues than that (for example, you have x in the main branch, but x is a local variable of customOps), but the cause of the error you have is the one stated above.
Below you find a tested and working version of your code that addresses both issues.
import tensorflow as tf
import numpy as np
def customOps(n):
x = tf.placeholder(tf.float32)
v1 = tf.reduce_sum(x,1)
v2 = tf.reduce_sum(x,0)
v = tf.nn.softmax(tf.concat([v1, v2], 0))
index = np.argmax(v)
if index > n/3:
finalval = tf.norm(v1-v2, ord='euclidean')
else:
finalval = tf.norm(v1+v2, ord='euclidean')
return x, finalval
if __name__ == '__main__':
mat = np.asarray([[0, 1], [1, 0]], dtype = np.float32)
n = mat.shape[0]
tf.reset_default_graph()
x, finalVal = customOps(n)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
outVal = sess.run(finalVal, feed_dict={x:mat})
print(outVal)
sess.close()
I want to do something as simple as this a = a + b, example code as follow
sess = tf.InteractiveSession()
embed = tf.Variable(tf.random_uniform([10, 2], -1, 1))
saver = tf.train.Saver([embed])
saver.restore(sess, 'save/model.ckpt')
new_embed = tf.Variable(tf.random_uniform([5, 2], -1, 1))
init = tf.initialize_variables([new_embed])
sess.run(init)
embed = tf.Variable(tf.concat(0, [embed, new_embed]))
However the last line won't execute because embed becomes an uninitialized value.
What I wish to accomplish here is to restore a variable from a file and concat with a new variable, i.e. make the [10, 2] variable to be a [15, 2] variable, where the first 10 rows are from the stored variable.
I was thinking to restore the [10, 2] variable to a new variable say old_ebmed, but I couldn't find a way to do so.
Any help would be appreciated.
I found a way to restore the variable to a varialbe with a different name
import tensorflow as tf
sess = tf.InteractiveSession()
old_embed = tf.Variable(tf.constant(0.0, shape = [10, 2]))
restorer = tf.train.Saver({'embed': old_embed})
restorer.restore(sess, 'test/d.ckpt')
new_embed = tf.Variable(tf.random_uniform([5, 2], -1, 1))
init_new = tf.initialize_variables([new_embed])
sess.run(init_new)
embed = tf.Variable(tf.concat(0, [old_embed, new_embed]))
init_embed = tf.initialize_variables([embed])
sess.run(init_embed)
saver = tf.train.Saver({'embed': embed})
saver.save(sess, 'test/d.ckpt')
I've found that indexing still is an open issue in tensorflow (#206), so I'm wondering what I could use as a workaround at the moment. I want to index/slice a row/column of a matrix based on a variable that changes for every training example.
What I've tried so far:
Slicing based on placeholder (doesn't work)
The following (working) code slices based on a fixed number.
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
y = tf.slice(x,[0],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5]})
print(result)
However, it seems that I can't simply replace one of these fixed numbers with a tf.placeholder. The following code gives me the error "TypeError: List of Tensors when single Tensor expected."
import tensorflow as tf
import numpy as np
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x:[1,2,3,4,5],i:0})
print(result)
This sounds like the brackets around [i] are too much, but removing them doesn't help either. How to use a placeholder/variable as index?
Slicing based on python variable (doesn't backprop/update properly)
I've also tried using a normal python variable as index. This does not lead to an error, but the network doesn't learn anything while training. I suppose because the changing variable is not properly registered, the graph is malformed and updates don't work?
Slicing via one-hot vector + multiplication (works, but is slow)
One workaround I found is using a one-hot vector. Making a one-hot vector in numpy, passing this using a placeholder, then doing the slicing via matrix multiplication. This works, but is quite slow.
Any ideas how to efficiently slice/index based on a variable?
Slicing based on a placeholder should work just fine. It looks like you are running into a type error, due to some subtle issues of shapes and types. Where you have the following:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,[i],[1])
...you should instead have:
x = tf.placeholder("float")
i = tf.placeholder("int32")
y = tf.slice(x,i,[1])
...and then you should feed i as [0] in the call to sess.run().
To make this a little clearer, I would recommend rewriting the code as follows:
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[None]) # 1-D tensor
i = tf.placeholder(tf.int32, shape=[1])
y = tf.slice(x, i, [1])
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(y, feed_dict={x: [1, 2, 3, 4, 5], i: [0]})
print(result)
The additional shape arguments to the tf.placeholder op help to ensure that the values you feed have the appropriate shapes, and also that TensorFlow will raise an error if the shapes are not correct.
If you have an extra dimension, this works.
import tensorflow as tf
import numpy as np
def reorder0(e, i, length):
'''
e: a two dimensional tensor
i: a one dimensional int32 tensor, of shape (e.shape[0])
returns: a tensor of the same shape as e, where the jth entry is entry i[j] from e
'''
return tf.concat(
[ tf.expand_dims( e[i[j],:], axis=0) for j in range(length) ],
axis=0
)
e = tf.placeholder(tf.float32, shape=(2,3,5), name='e' ) # sentences, words, embedding
i = tf.placeholder(tf.int32, shape=(2,3), name='i' ) # for each word, index of parent
p = tf.concat(
[ tf.expand_dims(reorder0(e[k,:,:], i[k,:], 3), axis=0) for k in range(2) ],
axis=0,
name='p'
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(p, feed_dict={
e: [
( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
],
i: [ (1,1,1), (2,0,2)]
})
print(result)
If the sizes are not known when building the model, use TensorArray.
e = tf.placeholder(tf.float32, shape=(3,5) ) # words, embedding
i = tf.placeholder(tf.int32, shape=(3) ) # for each word, index of parent
#p = reorder0(e, i, 3)
a = tf.TensorArray(
tf.float32,
size=e.get_shape()[0],
dynamic_size=True,
infer_shape= True,
element_shape=e.get_shape()[1],
clear_after_read = False
)
#initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
#run
result = sess.run(
a.unstack(e).gather(i),
feed_dict={
e: ( (1.0,1.1,1.2,1.3,1.4),(2.0,2.1,2.2,2.3,2.4),(3.0,3.1,3.2,3.3,3.4) ),
#( (21.0,21.1,21.2,21.3,21.4),(22.0,22.1,22.2,22.3,22.4),(23.0,23.1,23.2,23.3,23.4) ),
i: (2,0,2)
}
)
print(result)