I'm trying to load a saved model by saver and evaluate the output of the network over the test data, restoring a saved model and using the network as a function:
def model(x, x_size):
with tf.variable_scope("my_net", reuse=tf.AUTO_REUSE):
W1 = tf.get_variable('w1', [x_size, x_size],
initializer=tf.random_normal_initializer())
b1 = tf.get_variable('b1', [x_size],
initializer=tf.random_normal_initializer())
y1 = tf.nn.relu(tf.matmul(x, W1) + b1,'y1')
return y1
eval_x = tf.placeholder(tf.float32, name='eval_x', shape=[None, x_size])
eval_probs = model(eval_x, x_size)
with tf.Session() as sess:
new_saver = tf.train.import_meta_graph('save_model/model.ckpt.meta')
new_saver.restore(sess, "save_model/model.ckpt")
probs = sess.run(eval_probs, feed_dict={eval_x: test_x})
The error I get is:
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value my_net/w1
[[Node: my_net/w1/read = IdentityT=DT_FLOAT, _class=["loc:#my_net/w1"], _device="/job:localhost/replica:0/task:0/device:CPU:0"]]
Can anybody help?
You need to run a special operation to initialize variables like w1. An easy way to do this is to execute sess.run(tf.global_variables_initializer()) after you create the session but before you use your variables.
From the code it seems that you are not passing any value for
x_size
so I don't think your variables get initialised.
Related
I've a simple MNIST which I've successfully saved, being the code the next:
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
import tensorflow as tf
sess = tf.InteractiveSession()
tf_save_file = './mnist-to-save-saved'
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
y = tf.matmul(x, W) + b
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = y_, logits = y))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
saver.save(sess, tf_save_file)
for _ in range(1000):
batch = mnist.train.next_batch(100)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver.save(sess, tf_save_file, global_step=1000)
print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))
Then, the next files are generated:
checkpoint
mnist-to-save-saved-1000.data-00000-of-00001
mnist-to-save-saved-1000.index
mnist-to-save-saved-1000.meta
mnist-to-save-saved.data-00000-of-00001
mnist-to-save-saved.index
mnist-to-save-saved.meta
Now, in order to use it in production (and so, for example, pass it a number image), I want to be able to execute the trained model by passing it any number image to make the prediction (I mean, not deploying yet a server but making this prediction "locally", having in the same directory that "fixed" number image, so using the model would be like when you run an executable).
But, considering the (mid-low?) API level of my code, I'm confused about what would be the easiest correct next step (if restoring, using an Estimator, etc...), and how to do it.
Although I've read the official documentation, I insist that they seem to be many ways, but some are a bit complex and "noisy" for a simple model like this.
Edit:
I've edit and re-run the mnist file, whose code is the same as above except for those lines:
...
x = tf.placeholder(tf.float32, shape=[None, 784], name='input')
...
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1), name='result')
...
Then, I try to run this another .py code (in the same directory as the above code) in order to pass a local handwritten number image ("mnist-input-image.png") located in the same directory:
import tensorflow as tf
from PIL import Image
import numpy as np
image_test = Image.open("mnist-input-image.png")
image = np.array(image_test)
with tf.Session() as sess:
saver = tf.train.import_meta_graph('/Users/username/.meta')
new = saver.restore(sess, tf.train.latest_checkpoint('/Users/username/'))
graph = tf.get_default_graph()
input_x = graph.get_tensor_by_name("input:0")
result = graph.get_tensor_by_name("result:0")
feed_dict = {input_x: image}
predictions = result.eval(feed_dict=feed_dict)
print(predictions)
Now, if I correctly understand, I've to pass the image as numpy array. Then, my questions are:
1) Which is the exact file reference of those lines (since I've no .meta folder in my User folder)?
saver = tf.train.import_meta_graph('/Users/username/.meta')
new = saver.restore(sess, tf.train.latest_checkpoint('/Users/username/'))
I mean, to which exact files refer those lines (from my generated files list above)?
2) Translasted to my case, is correct this line to pass my numpy array into the feed dict?
feed_dict = {input_x: image}
A simple solution is to use your session object. When you have generated the checkpoint file, you can restore it with a Saver object.
By the way, do you know why most tutorials have their graph creation inside of a function? One good reason is because you can deserialize the graph quickly with your inputs.
The correct method to start a session is with the following:
# Use your placeholders, variables, etc to create the entire graph.
# Usually you return the input placeholder,
# prediction and the loss/accuracy here.
# You don't need the accuracy.
x, y, _ = make_your_graph(test_X, test_y)
# This object is the interface for serialization in tf
saver = tf.train.Saver()
with tf.Session() as sess:
# Takes your current model's checkpoint. "./checkpoint" is your checkpoint file.
saver.restore(sess, tf.train.latest_checkpoint("./checkpoint"))
prediction = sess.run(y)
Want to run more than 1 data point for your already-booted up session?
Then replace the last line with a feed dict:
while waiting_for_new_y():
another_y = get_new_y()
feed_dict = {x: [another_y]}
another_prediction = sess.run(y, feed_dict)
First of all , give value to name parameter in each object which you want to use later , so that you can use it later by it's name:
change this :
x = tf.placeholder(tf.float32, shape=[None, 784])
to
x = tf.placeholder(tf.float32, shape=[None, 784],name='input')
and
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
to
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1),name='result')
Now run this small script to store model :
import tensorflow as tf
with tf.Session() as sess:
saver = tf.train.import_meta_graph('/Users/dummy/.meta')
new=saver.restore(sess, tf.train.latest_checkpoint('/Users/dummy/'))
graph = tf.get_default_graph()
input_x = graph.get_tensor_by_name("input:0")
result = graph.get_tensor_by_name("result:0")
feed_dict = {input_x: mnist.test.images,} #here you feed your new data for example i am feeding mnist
predictions = result.eval(feed_dict=feed_dict)
print(predictions)
And you will get output.
I have the following code trying to optimize for a linear model with two inputs and three parameters (m_1, m_2 and b). Initially, I had issues with importing the data in a way such that the feed_dict would accept them, which I solved by putting it in a numpy array instead.
Now the optimizer function will run smoothly (and the outputs look roughly like it is optimizing the parameters), but as soon as I try to return the cost with the line at the end:
cost_val = sess.run(cost)
It returns the following error:
tensorflow.python.framework.errors_impl.InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder_2' with dtype float and shape [?,1]
[[Node: Placeholder_2 = Placeholder[dtype=DT_FLOAT, shape=[?,1], _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]
If I comment out that line alone, everything runs smoothly.
I tried changing the cost function from the more complicated one I was using to something simpler, but the error persists. I know this is probably related to the data input shape(?), but can't figure how the data would work for the optimizer but not the cost function.
# reading in data
filename = tf.train.string_input_producer(["file.csv"])
reader = tf.TextLineReader(skip_header_lines=1)
key, value = reader.read(filename)
rec_def = [[1], [1], [1]]
input_1, input_2, col3 = tf.decode_csv(value, record_defaults=rec_def)
# parameters
learning_rate = 0.001
training_steps = 300
x = tf.placeholder(tf.float32, [None,1])
x2 = tf.placeholder(tf.float32, [None,1])
m = tf.Variable(tf.zeros([1,1]))
m2 = tf.Variable(tf.zeros([1,1]))
b = tf.Variable(tf.zeros([1]))
y_ = tf.placeholder(tf.float32, [None,1])
y = tf.matmul(x,m) + tf.matmul(x2,m2) + b
# cost function
# cost = tf.reduce_mean(tf.log(1+tf.exp(-y_*y)))
cost = tf.reduce_sum(tf.pow((y_-y),2))
# Gradient descent optimizer
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
# initializing variables
init = tf.global_variables_initializer()
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
sess.run(init)
for i in range(training_steps):
xs = np.array([[sess.run(input_1)]])
ys = np.array([[sess.run(input_2)]])
label = np.array([[sess.run(col3)]])
feed = {x:xs, x2:ys, y_:label}
sess.run(optimizer, feed_dict=feed)
cost_val = sess.run(cost)
coord.request_stop()
coord.join(threads)
The cost tensor is a function of the placeholder tensors and this requires them to have a value. Since the call to sess.run(cost) isn't feeding those placeholders, you're seeing the error. (Putting it another way - what values of x and y_ do you want to compute the cost for?)
So you want to change the line:
cost_val = sess.run(cost)
to:
cost_val = sess.run(cost, feed_dict=feed)
Hope that helps.
I am learning to use tensorflow library. Every time I try to use simplest (I guess) possible example of gradient-based learning I get the same error, which makes me stuck.
Here is the code:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
los = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for counter in range(100):
grd = sess.run(optim.compute_gradients(loss=los))
sess.run(optim.apply_gradients(grads_and_vars=grd))
On the last line, I get following error:
AttributeError: 'numpy.float32' object has no attribute 'op'.
Thank you in advance for any tips.
The input of apply_gradients must be a tensor not a numpy array. Because, it is not meant to be called at each step of the learning but just once when building your graph. In fact, it "creates" the "training step" in which variables are updated using the computed gradient
You should use that kind of code instead:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
los = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
grads, vars = zip(*optimizer.compute_gradients(loss))
train_step = optim.apply_gradients(grads_and_vars=grads)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for counter in range(100):
grd = sess.run(train_step, feed_dict = {...})
The tensorflow converts tensor into numpy array when stored and evaluated using sess.run(). The apply_gradients method of GradientDescentOptimizer raises TypeError since it requires tensor as input as described here. This causing problem in the original code.
for counter in range(100):
grd = sess.run(optim.compute_gradients(loss=los))
sess.run(optim.apply_gradients(grads_and_vars=grd))
The solution to this problem is to define all the operations before starting session.
There may be cases where gradients need to be post-processed before updating weights. All such post-processing logic need to be defined before session begins.
Here is a simple example:
import tensorflow as tf
x = tf.constant(1, dtype=tf.float32, name='X')
a = tf.Variable(1, dtype=tf.float32, name= 'A')
y = tf.constant(10, dtype=tf.float32, name='Y')
ey = tf.multiply(x, a)
loss = (y - ey)**2
optim = tf.train.GradientDescentOptimizer(learning_rate=0.2)
grads = optim.compute_gradients(loss)
train_step = optim.apply_gradients(grads_and_vars=grads)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for counter in range(100):
sess.run(train_step)
I have saved a trained LSTM model and I want to restore the prediction to use it in testing. I was trying to follow this post. But I am getting errors. Here is what I tried:
x = tf.placeholder('tf.float32', [None, input_vec_size, 1])
y = tf.placeholder('tf.float32')
def recurrent_neural_network(x):
layer = {'weights': tf.Variable(tf.random_normal([n_hidden, n_classes])),
'biases': tf.Variable(tf.random_normal([n_classes]))}
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, 1])
x = tf.split(x, input_vec_size, 0)
lstm_cell = rnn.BasicLSTMCell(n_hidden, state_is_tuple=True)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.add(tf.matmul(outputs[-1], layer['weights']), layer['biases'])
return output
def train_neural_network(x):
prediction = recurrent_neural_network(x)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
Training ...
saver.save(sess, os.path.join(os.getcwd(), 'my_test_model'))
After that, in the training phase I am trying
def test_neural_network(input_data):
with tf.Session() as sess:
#sess.run(tf.global_variables_initializer())
new_saver = tf.train.import_meta_graph('my_test_model.meta')
new_saver.restore(sess, tf.train.latest_checkpoint('./'))
prediction = tf.get_default_graph().get_tensor_by_name("prediction:0")
Calculate features from input_data ...
result = sess.run(tf.argmax(prediction.eval(feed_dict={x: features}), 1))
But this throws the following error:
KeyError: "The name 'prediction:0' refers to a Tensor which does not exist. The operation, 'prediction', does not exist in the graph."
Then I tried adding :
tf.add_to_collection('prediction', prediction) before saving and replacing by prediction = tf.get_collection('prediction')[0] after restoring. But this gives me the following error:
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder_2' with dtype float and shape [?,34,1]
[[Node: Placeholder_2 = Placeholderdtype=DT_FLOAT, shape=[?,34,1], _device="/job:localhost/replica:0/task:0/cpu:0"]]
I know for the first error, I am supposed to assign a name in order to restore but prediction is not a tensorflow variable. I went through few previous posts and articles but unable to come up with a working solution. So, my questions are:
Am I doing something conceptually wrong? If so, what?
If not, is there an implementation error? And how do I solve it?
Thanks.
I could save my trained model at last and so posting an answer in case anyone comes across this question. I did not get a solution for the exact problem but I could build and save my model using tflearn. In order to train and store:
model = tflearn.DNN(lstm_model(n_classes, input_vec_size))
model.fit(train_x, train_y, validation_set=(test_x, test_y), n_epoch=20,
show_metric=True, snapshot_epoch=True, run_id='lstm_model')
model.save("../Models/lstm_model")
And later, to restore:
model.load(filepath+"lstm_model")
This seemed to be a far easier way to work with the model, and provides a compact and novel way to do the same task which I posted in the question.
I have saved a model and now I am trying to restore it, after restoring it works for the first time correctly but when I press 'TEST' button on the same running program to test another image it gives error
ValueError: At least two variables have the same name: Variable_2/Adam
def train_neural_network(x):
prediction = neural_network_model(x)#logits
softMax=tf.nn.softmax_cross_entropy_with_logits(
logits=prediction, labels=y)#prediction and original comapriosn
cost = tf.reduce_mean(softMax)#total loss
optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)#learning_rate=0.01
hm_epochs = 20
new_saver = tf.train.Saver()
with tf.Session() as sess:
global s
s=sess
sess.run(tf.global_variables_initializer())
new_saver = tf.train.import_meta_graph('../MY_MODELS/my_MNIST_Model_1.meta')
new_saver.restore(s, tf.train.latest_checkpoint('../MY_MODELS'))
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
print('Accuracy:', accuracy.eval(
{x: mnist.test.images, y: mnist.test.labels}))
The graph which you have loaded already has all the variables needed for inference. You need to load the tensors like accuracy from the saved graph. In your case you have same variables declared outside, which conflicts with the variables in the graph.
During training, if you have named your tensor accuracy with a name='accuracy', then you can load it from the graph using: get_tensor_by_name('accuracy:0'). In your example, you need to load the input tensors x and y also from the graph. Your code should be like:
def inference():
loaded_graph = tf.Graph()
new_saver = tf.train.Saver()
with tf.Session(graph=loaded_graph) as sess:
new_saver = tf.train.import_meta_graph('../MY_MODELS/my_MNIST_Model_1.meta')
new_saver.restore(s, tf.train.latest_checkpoint('../MY_MODELS'))
#Get the tensors by their variable name
# Note: the names of the following tensors have to be declared in your train graph for this to work. So just name them appropriately.
_accuracy = loaded_graph.get_tensor_by_name('accuracy:0')
_x = loaded_graph.get_tensor_by_name('x:0')
_y = loaded_graph.get_tensor_by_name('y:0')
print('Accuracy:', _accuracy.eval(
{_x: mnist.test.images, _y: mnist.test.labels}))