Develop Reference

How can I save a Tensorflow Core model?

I'm a beginner in Tensorflow and I found this neural network for binary classification which is giving me decent results, I would like to know after I run the session how can I save the model? I already try from the official website but nothing is working.
class AnnMLP():
def train(self,X_input,y_input,test,range_iteration,learning_rate): X = tf.compat.v1.placeholder(tf.float32, [None,27]) Y = tf.compat.v1.placeholder(tf.float32, [None,1])
# input
W1 = tf.Variable(tf.random.normal([27,60], seed=0), name='weight1')
b1 = tf.Variable(tf.random.normal([60], seed=0), name='bias1')
layer1 = tf.nn.sigmoid(tf.matmul(X,W1) + b1)
dropout_layer = keras.layers.Dropout(rate=0.4)
layer1=dropout_layer(layer1)
# hidden1
W2 = tf.Variable(tf.random.normal([60,60], seed=0), name='weight2')
b2 = tf.Variable(tf.random.normal([60], seed=0), name='bias2')
layer2 = tf.nn.sigmoid(tf.matmul(layer1,W2) + b2)
dropout_layer = keras.layers.Dropout(rate=0.4)
layer2=dropout_layer(layer2)
# hidden2
W3 = tf.Variable(tf.random.normal([60,90], seed=0), name='weight3')
b3 = tf.Variable(tf.random.normal([90], seed=0), name='bias3')
layer3 = tf.nn.sigmoid(tf.matmul(layer2,W3) + b3)
dropout_layer = keras.layers.Dropout(rate=0.4)
layer3=dropout_layer(layer3)
# output
W4 = tf.Variable(tf.random.normal([90,1], seed=0), name='weight4')
b4 = tf.Variable(tf.random.normal([1], seed=0), name='bias4')
logits = tf.matmul(layer3,W4) + b4
hypothesis = tf.nn.sigmoid(logits)
cost_i = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,labels=Y)
cost = tf.reduce_mean(cost_i)
train =tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost)
#train = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(cost) train = tf.train.AdadeltaOptimizer(learning_rate=learning_rate).minimize(cost)
prediction = tf.cast(hypothesis > 0.5, dtype=tf.float32)
correct_prediction = tf.equal(prediction, Y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, dtype=tf.float32))
print("\n============Processing============")
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in range(range_iteration):
sess.run(train, feed_dict={X: X_input, Y: y_input})
if step % 1000 == 0:
loss, acc = sess.run([cost, accuracy], feed_dict={X: X_input, Y: y_input})
print("Step: {:5}\tLoss: {:.3f}\tAcc: {:.2%}".format(step, loss, acc))
train_acc = sess.run(accuracy, feed_dict={X: X_input, Y: y_input})
if test == True:
test_acc,test_predict,test_correct = sess.run([accuracy,prediction,correct_prediction], feed_dict={X: X_test, Y: y_test})
return test_predict

Tensorflow provides privilege to load and restore entire model and checkpoints. For more details find the link here.
Tensorflow < 2
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run()
# Do some work with the model.
# Save the variables to disk.
save_path = saver.save(sess, "/tmp/model.ckpt")
print("Model saved in path: %s" % save_path)
Tensorflow > 2
# Create and train a new model instance.
model = create_model()
model.fit(train_images, train_labels, epochs=10)
# Save the entire model to a HDF5 file.
model.save('my_model.h5')

Tensorflow provides privilege to load and restore entire model and checkpoints
Tensorflow < 2
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run()
# Do some work with the model.
# Save the variables to disk.
save_path = saver.save(sess, "/tmp/model.ckpt")
print("Model saved in path: %s" % save_path)
Tensorflow>2
# Create and train a new model instance.
model = create_model()
model.fit(train_images, train_labels, epochs=10)
# Save the entire model to a HDF5 file.
model.save('my_model.h5'

With Tensorflow, 2 class classification using Neural Network

I'm trying 2 class classification of images by neural network using Tensorflow.
I want to extract 1000 pixels randomly.
However, I am in trouble with error:
"logits = inference(images_placeholder, keep_prob)
File "train5.py", line 83, in inference
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
TypeError: 'Tensor' object is not callable"
Please tell me what should I do.
I will attach a code below.
import sys
sys.path.append('/usr/local/opt/opencv3/lib/python3.5.4/site-packages')
import cv2
import numpy as np
import tensorflow as tf
import tensorflow.python.platform
import tensorboard as tb
import os
import math
import time
import random
start_time = time.time()
# TensorBoard output information directory
log_dir = '/tmp/data1' #tensorboard --logdir=/tmp/data1
#directory delete and reconstruction
if tf.gfile.Exists(log_dir):
tf.gfile.DeleteRecursively(log_dir)
tf.gfile.MakeDirs(log_dir)
# Reserve memory
config = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True))
sess = sess = tf.Session(config=config)
NUM_CLASSES = 2
IMAGE_SIZE_x = 1024
IMAGE_SIZE_y = 768
IMAGE_CHANNELS = 1
IMAGE_PIXELS = IMAGE_SIZE_x*IMAGE_SIZE_y*IMAGE_CHANNELS
SAMPLE_PIXELS = 1000
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('train', 'train.txt', 'File name of train data')
flags.DEFINE_string('test', 'test.txt', 'File name of train data')
flags.DEFINE_string('image_dir', 'trdata', 'Directory of images')
flags.DEFINE_string('train_dir', '/tmp/data', 'Directory to put the training data.')
flags.DEFINE_integer('max_steps', 20000, 'Number of steps to run trainer.')
flags.DEFINE_integer('batch_size', 10, 'Batch size'
'Must divide evenly into the dataset sizes.')
flags.DEFINE_float('learning_rate', 1e-5, 'Initial learning rate.')
def inference(images_placeholder, keep_prob):
""" Function to create predictive model
argument:
images_placeholder: image placeholder
keep_prob: dropout rate placeholder
Return:
y_conv:
"""
# Initialie with normal distribution with weight of 0.1
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
# Initialized with normal distribution with bias of 0.1
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
# Reshape input
x_image = images_placeholder
# ramdom sumpling pixels
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
x_list = [samples[i] for i in list]
# Input
with tf.name_scope('fc1') as scope:
W_fc1 = weight_variable([x_list,10])
b_fc1 = bias_variable([10])
h_fc1 = tf.nn.relu(tf.matmul(x_image,W_fc1) + b_fc1)
# Affine1
with tf.name_scope('fc2') as scope:
W_fc2 = weight_variable([10,10])
b_fc2 = bias_variable([10])
h_fc2 = tf.nn.relu(tf.matmul(h_fc1,W_fc2) + b_fc2)
# Affine2
with tf.name_scope('fc3') as scope:
W_fc3 = weight_variable([10,10])
b_fc3 = bias_variable([10])
h_fc3 = tf.nn.relu(tf.matmul(h_fc2,W_fc3) + b_fc3)
# Affine3
with tf.name_scope('fc4') as scope:
W_fc4 = weight_variable([10,10])
b_fc4 = bias_variable([10])
h_fc4 = tf.nn.relu(tf.matmul(h_fc3,W_fc4) + b_fc4)
# Affine4
with tf.name_scope('fc5') as scope:
W_fc5 = weight_variable([10,2])
b_fc5 = bias_variable([2])
# softmax regression
with tf.name_scope('softmax') as scope:
y_out=tf.nn.softmax(tf.matmul(h_fc4, W_fc5) + b_fc5)
# return
return y_out
def loss(logits, labels):
""" loss function
引数:
logits: logit tensor, float - [batch_size, NUM_CLASSES]
labels: labrl tensor, int32 - [batch_size, NUM_CLASSES]
返り値:
cross_entropy:tensor, float
"""
# cross entropy
cross_entropy = -tf.reduce_sum(labels*tf.log(tf.clip_by_value(logits,1e-10,1.0)))
# TensorBoard
tf.summary.scalar("cross_entropy", cross_entropy)
return cross_entropy
def training(loss, learning_rate):
#Adam
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
return train_step
def accuracy(logits, labels):
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
tf.summary.scalar("accuracy", accuracy)
return accuracy
if __name__ == '__main__':
f = open(FLAGS.train, 'r')
# array data
train_image = []
train_label = []
for line in f:
# Separate space and remove newlines
line = line.rstrip()
l = line.split()
# Load data and resize
img = cv2.imread(FLAGS.image_dir + '/' + l[0])
img = cv2.resize(img, (IMAGE_SIZE_x, IMAGE_SIZE_y))
#transrate grayscale
img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# transrate one row and 0-1 float
train_image.append(img_gry.flatten().astype(np.float32)/255.0)
# Prepare with label 1-of-k method
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
train_label.append(tmp)
# transrate numpy
train_image = np.asarray(train_image)
train_label = np.asarray(train_label)
f.close()
f = open(FLAGS.test, 'r')
test_image = []
test_label = []
for line in f:
line = line.rstrip()
l = line.split()
img = cv2.imread(FLAGS.image_dir + '/' + l[0])
img = cv2.resize(img, (IMAGE_SIZE_x, IMAGE_SIZE_y))
#transrate grayscale
img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# transrate one row and 0-1 float
test_image.append(img_gry.flatten().astype(np.float32)/255.0)
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
test_label.append(tmp)
test_image = np.asarray(test_image)
test_label = np.asarray(test_label)
f.close()
with tf.Graph().as_default():
# Put the image Tensor
images_placeholder = tf.placeholder("float", shape=(None, IMAGE_PIXELS))
# Put the label Tensor
labels_placeholder = tf.placeholder("float", shape=(None, NUM_CLASSES))
# Put dropout rate Tensor
keep_prob = tf.placeholder("float")
# Load inference() and make model
logits = inference(images_placeholder, keep_prob)
# Load loss() and calculate loss
loss_value = loss(logits, labels_placeholder)
# Load training() and train
train_op = training(loss_value, FLAGS.learning_rate)
# calculate accuracy
acc = accuracy(logits, labels_placeholder)
# save
saver = tf.train.Saver()
# Make Session
sess = tf.Session()
# Initialize variable
sess.run(tf.global_variables_initializer())
# TensorBoard
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
# Start training
for step in range(FLAGS.max_steps):
for i in range(int(len(train_image)/FLAGS.batch_size)):
batch = FLAGS.batch_size*i
sess.run(train_op, feed_dict={
images_placeholder: train_image[batch:batch+FLAGS.batch_size],
labels_placeholder: train_label[batch:batch+FLAGS.batch_size],
keep_prob: 0.5})
# Accuracy calculation for every steps
train_accuracy = sess.run(acc, feed_dict={
images_placeholder: train_image,
labels_placeholder: train_label,
keep_prob: 1.0})
print("step %d, training accuracy %g" %(step, train_accuracy))
# Added value to be displayed in Tensorflow every 1step
summary_str = sess.run(summary_op, feed_dict={
images_placeholder: train_image,
labels_placeholder: train_label,
keep_prob: 1.0})
summary_writer.add_summary(summary_str, step)
# Display accuracy on test data after training
print(" test accuracy %g"%sess.run(acc, feed_dict={
images_placeholder: test_image,
labels_placeholder: test_label,
keep_prob: 1.0}))
duration = time.time() - start_time
print('%.3f sec' %duration)
# Save model
save_path = saver.save(sess, os.getcwd() + "\\model.ckpt")

The error is this:
images_placeholder = tf.placeholder("float", shape=(None, IMAGE_PIXELS))
...
x_image = images_placeholder
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
x_image, just like images_placeholder is a variable node, so x_image(...) doesn't make sense and obviously leads to an error "TypeError: 'Tensor' object is not callable".
I assume you're trying to sample SAMPLE_PIXELS from each image in a batch. Note that random.sample won't work here, because x_image is a symbolic variable, its value is only known during the session. You have to use tf.boolean_mask with a random mask in order to select random pixels from the image.

tensorflow calculations without retraining

I want to train a model once in tensorflow, and then want to use the trained model for predicting some functions. Before we get into specifics, lets define a couple of functions ...
def runTF(func, inpDict):
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
result = sess.run(func, feed_dict = inpDict)
sess.close()
return result
and
def optTF(opt, res, others, inpDict, nSteps, printSteps=50):
os, re = [], []
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
for i in range(nSteps):
# First run the optimizer ...
sess.run(opt, feed_dict = inpDict)
# Save all the data you want to save
temp = sess.run( [res] + others, feed_dict = inpDict)
re.append(temp[0])
os.append(temp[1:])
if (i%printSteps) == 0:
print('{:5d}'.format(i))
sess.close()
return re, os
Here are a couple of steps for what I am doing ...
A. Generating some data
N = 500
features = 2
nSteps = 1000
X = np.array([np.random.random(N), np.random.random(N)])
data = [X.T, X[0].reshape(-1, 1)]
B. create a simple linear model
d = tf.placeholder(shape = np.shape(data[0]), dtype = tf.float32, name='d') # input layer
dOut = tf.placeholder(shape = np.shape(data[1]), dtype = tf.float32, name='dOut') # output layer
W = tf.Variable( np.random.randn(features, 1), dtype = tf.float32, name='W')
b = tf.Variable( np.zeros((1,1)), dtype = tf.float32, name='b')
result = tf.matmul(d, W)+b
cost = tf.reduce_mean((dOut - result)**2)
optimizer = tf.train.AdamOptimizer(learning_rate = 0.1).minimize(cost)
C. Lets run the optimiser
inpDict = {d: data[0], dOut:data[1]}
ot = optTF(optimizer, result, [W, cost], inpDict, 200, 50)
Here, I have checked the result, and see that it is what I want. So the optimiser is working fine. The model has been optimised. Now, I want to make a prediction with some other data. So I do ...
r = runTF(result, inpDict)
This new result is not what I would expect from the same trained model.
Now, as long as I stay within the same tf.Session(), we are ok. However, I do want to be able to do predictions even when I an done with a session. So my question is, how do I use a model once we have trained it in one session in a different session?
Note, the entire thing is something that I did in a different session?
Edited:
I edited the two functions to incorporate the saving ...
def runTF(func, inpDict, modelFile=None):
if modelFile is not None:
saver = tf.train.Saver()
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
if modelFile is not None:
ckpt = tf.train.get_checkpoint_state(modelFile)
print(modelFile, ckpt)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print('Session restored')
result = sess.run(func, feed_dict = inpDict)
sess.close()
return result
and
def optTF(opt, res, others, inpDict, nSteps, printSteps=50, modelFile='models/temp.ckpt'):
os, re = [], []
saver = tf.train.Saver()
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
for i in range(nSteps):
# First run the optimizer ...
sess.run(opt, feed_dict = inpDict)
# Save all the data you want to save
temp = sess.run( [res] + others, feed_dict = inpDict)
re.append(temp[0])
os.append(temp[1:])
if (i%printSteps) == 0:
print('{:5d}'.format(i))
path = saver.save(sess, modelFile)
print('Model saved in: {}'.format(path))
sess.close()
return re, os
And running the model as:
ot = optTF(optimizer, result, [cost]+weights+biases, inpDict, 200, 50)
r = runTF([result], inpDict, 'models/temp.ckpt')
Still nothing! I checked that:
The value of ckpt is None
The models folder has the following files:
checkpoint
temp.ckpt.index
temp.ckpt.data-00000-of-00001
temp.ckpt.meta

You need to save and restore the session you are creating and training. As in
init = tf.initialize_all_variables()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
if restore:
ckpt = tf.train.get_checkpoint_state(save_path)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
# ... training code omitted ...
saver.save(sess, save_path)
checkout also https://www.tensorflow.org/programmers_guide/saved_model if you want to create a full model with estimator instead of just one session.

Tensorflow restored weights is not set

I'm trying to restore a model in Tensorflow which I've trained. The problem is that it does not seem like the weights are properly restored.
For the training I've got the weights and biases defined as:
W = {
'h1': tf.Variable(tf.random_normal([n_inputs, n_hidden_1]), name='wh1'),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2]), name='wh2'),
'o': tf.Variable(tf.random_normal([n_hidden_2, n_classes]), name='wo')
}
b = {
'b1': tf.Variable(tf.random_normal([n_hidden_1]), name='bh1'),
'b2': tf.Variable(tf.random_normal([n_hidden_2]), name='bh2'),
'o': tf.Variable(tf.random_normal([n_classes]), name='bo')
}
Then I do some training on my own custom 2D image dataset and save the model by calling the tf.saver
saver = tf.train.Saver()
saver.save(sess, 'tf.model')
Later I want to restore that model with the exact same weights, so I build the model as before (also with the random_normal initialization) and call the tf.saver.restore
saver = tf.train.import_meta_graph('tf.model.meta')
saver.restore(sess, tf.train.latest_checkpoint('./'))
Now, if i call:
temp = sess.run(W['h1'][0][0])
print temp
I get random values, and not the restored value of the weight.
I've drawn a blank on this one, can somebody point me in the right direction?
FYI, I've tried (without) luck to simply declare the tf.Variables, but I keep getting:
ValueError: initial_value must be specified.
even though Tensorflow themselves state that it should be possible to simply declare with no initial value (https://www.tensorflow.org/programmers_guide/variables part: Restoring Values)
Update 1
When I, as suggested, run
all_vars = tf.global_variables()
for v in all_vars:
print v.name
I get the following output:
wh1:0
wh2:0
wo:0
bh1:0
bh2:0
bo:0
wh1:0
wh2:0
wo:0
bh1:0
bh2:0
bo:0
beta1_power:0
beta2_power:0
wh1/Adam:0
wh1/Adam_1:0
wh2/Adam:0
wh2/Adam_1:0
wo/Adam:0
wo/Adam_1:0
bh1/Adam:0
bh1/Adam_1:0
bh2/Adam:0
bh2/Adam_1:0
bo/Adam:0
bo/Adam_1:0
Which shows that the variables indeed is read. However invoking
print sess.run("wh1:0")
Results in the error: Attempting to use uninitialized value wh1

So with the help of you guys, I ended up dividing the saving and restoring parts of my program into two files, to ensure that no unwanted variables were initialized.
Train and Save routines fnn.py
def build(self, topology):
"""
Builds the topology of the model
"""
# Sanity check
assert len(topology) == 4
n_inputs = topology[0]
n_hidden_1 = topology[1]
n_hidden_2 = topology[2]
n_classes = topology[3]
# Sanity check
assert self.img_h * self.img_w == n_inputs
# Instantiate TF Placeholders
self.x = tf.placeholder(tf.float32, [None, n_inputs], name='x')
self.y = tf.placeholder(tf.float32, [None, n_classes], name='y')
self.W = {
'h1': tf.Variable(tf.random_normal([n_inputs, n_hidden_1]), name='wh1'),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2]), name='wh2'),
'o': tf.Variable(tf.random_normal([n_hidden_2, n_classes]), name='wo')
}
self.b = {
'b1': tf.Variable(tf.random_normal([n_hidden_1]), name='bh1'),
'b2': tf.Variable(tf.random_normal([n_hidden_2]), name='bh2'),
'o': tf.Variable(tf.random_normal([n_classes]), name='bo')
}
# Create model
self.l1 = tf.nn.sigmoid(tf.add(tf.matmul(self.x, self.W['h1']), self.b['b1']))
self.l2 = tf.nn.sigmoid(tf.add(tf.matmul(self.l1, self.W['h2']), self.b['b2']))
logits = tf.add(tf.matmul(self.l2, self.W['o']), self.b['o'])
# Define predict operation
self.predict_op = tf.argmax(logits, 1)
probs = tf.nn.softmax(logits, name='probs')
# Define cost function
self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, self.y))
# Adding these to collection so we can restore them again
tf.add_to_collection('inputs', self.x)
tf.add_to_collection('inputs', self.y)
tf.add_to_collection('outputs', logits)
tf.add_to_collection('outputs', probs)
tf.add_to_collection('outputs', self.predict_op)
def train(self, X, Y, n_epochs=10, learning_rate=0.001, logs_path=None):
"""
Trains the Model
"""
self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)
costs = []
# Instantiate TF Saver
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# start the threads used for reading files
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Compute total number of batches
total_batch = int(self.get_num_examples() / self.batch_size)
# start training
for epoch in range(n_epochs):
for i in range(total_batch):
batch_xs, batch_ys = sess.run([X, Y])
# run the training step with feed of images
_, cost = sess.run([self.optimizer, self.cost], feed_dict={self.x: batch_xs,
self.y: batch_ys})
costs.append(cost)
print "step %d" % (epoch * total_batch + i)
#costs.append(cost)
print "Epoch %d" % epoch
saver.save(sess, self.model_file)
temp = sess.run(self.W['h1'][0][0])
print temp
if self.visu:
plt.plot(costs)
plt.show()
# finalize
coord.request_stop()
coord.join(threads)
Predict routine fnn_eval.py:
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
g = tf.get_default_graph()
# restore the model
self.saver = tf.train.import_meta_graph(self.model_file)
self.saver.restore(sess, tf.train.latest_checkpoint('./tfmodels/fnn/'))
wh1 = g.get_tensor_by_name("wh1:0")
print sess.run(wh1[0][0])
x, y = tf.get_collection('inputs')
logits, probs, predict_op = tf.get_collection('outputs')
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
predictions = []
print Y.eval()
for i in range(1):#range(self.get_num_examples()):
batch_xs = sess.run(X)
# Reshape batch_xs if only a single image is given
# (numpy is 4D: batch_size * heigth * width * channels)
batch_xs = np.reshape(batch_xs, (-1, self.img_w * self.img_h))
prediction, probabilities, logit = sess.run([predict_op, probs, logits], feed_dict={x: batch_xs})
predictions.append(prediction[0])
# finalize
coord.request_stop()
coord.join(threads)

I guess the problem might be caused by creating a new variable when you restore the model, not getting the already existed variable. I tried this code
saver = tf.train.import_meta_graph('./model.ckpt-10.meta')
w1 = None
for v in tf.global_variables():
print v.name
w1 = tf.get_variable('wh1', [])
init = tf.global_variables_initializer()
sess.run(init)
saver.restore(sess, './model.ckpt-10')
for v in tf.global_variables():
print v.name
and clearly you can see the output that it creates a new variable called wh1_1:0.
If you try this
w1 = None
for v in tf.global_variables():
print v.name
if v.name == 'wh1:0':
w1 = v
init = [tf.global_variables_initializer(), tf.local_variables_initializer()]
sess.run(init)
saver.restore(sess, './model.ckpt-10')
for v in tf.global_variables():
print v.name
temp = sess.run(w1)
print temp[0][0]
There will be no problem.
Tensorflow suggests that it is better to use tf.variable_scope() (link) like this
with tf.variable_scope("foo"):
v = tf.get_variable("v", [1])
with tf.variable_scope("foo", reuse=True):
v1 = tf.get_variable("v", [1])
assert v1 == v

I have meet the same problem when saving model to saved_model format. Anyone using the function add_meta_graph_and_variables to save the model for serving, be careful about this parameter "legacy_init_op: Legacy support for op or group of ops to execute after the restore op upon a load."

You want to pass in a var_list to the Saver.
In your case, the variable list would come from your W and b dictionaries: var_list = list(W.values())+list(b.values()). Then, to restore the model, pass in var_list to the Saver: saver = tf.train.Saver(var_list=var_list).
Next, you need to get your checkpoint state: model = tf.train.get_checkpoint_state(<your saved model directory>). After that you can restore the trained weights.
var_list = list(W.values())+list(b.values())
saver = tf.train.Saver(var_list=var_list)
model = tf.get_checkpoint_state('./model/')
with tf.Session() as sess:
saver.restore(sess,model.model_checkpoint_path)
#Now use the pretrained weights

Transform map to mapPartition using pyspark

I am trying to load a tensorflow model from disk and predicting the values.
Code
def get_value(row):
print("**********************************************")
graph = tf.Graph()
rowkey = row[0]
checkpoint_file = "/home/sahil/Desktop/Relation_Extraction/data/1485336002/checkpoints/model-300"
print("Loading model................................")
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
input_x = graph.get_operation_by_name("X_train").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
batch_predictions = sess.run(predictions, {input_x: [row[1]], dropout_keep_prob: 1.0})
print(batch_predictions)
return (rowkey, batch_predictions)
I have a RDD which consists of a tuple (rowkey, input_vector). I want to use the loaded model to predict the score/class of the input.
Code to call get_value()
result = data_rdd.map(lambda iter: get_value(iter))
result.foreach(print)
The problem is every time I call the map, the model is loaded everytime for each tuple and it takes a lot of time.
I am thinking of loading the model using mapPartitions and then use map to call get_value function.
I have no clue as how to convert the code to a mapPartition where I load the tensorflow model only once per parition and reduce the running time.
Thanks in advance.

I am not sure if I get your question correctly, but we can optimise your code a bit here.
graph = tf.Graph()
checkpoint_file = "/home/sahil/Desktop/Relation_Extraction/data/1485336002/checkpoints/model-300"
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement)
sess = tf.Session(config=session_conf)
s = sess.as_default()
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
input_x = graph.get_operation_by_name("X_train").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
session_pickle = cPickle.dumps(sess)
def get_value(key, vector, session_pickle):
sess = cPickle.loads(session_pickle)
rowkey = key
batch_predictions = sess.run(predictions, {input_x: [vector], dropout_keep_prob: 1.0})
print(batch_predictions)
return (rowkey, batch_predictions
result = data_rdd.map(lambda (key, row): get_value(key=key, vector = row , session_pickle = session_pickle))
result.foreach(print)
So you can serialize your tensorflow session. Though I haven't tested your code here. Run this and leave a comment.

I guess that the below code is a huge improvement as it uses mapPartitions.
Code
def predict(rows):
graph = tf.Graph()
checkpoint_file = "/home/sahil/Desktop/Relation_Extraction/data/1485336002/checkpoints/model-300"
print("Loading model................................")
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=allow_soft_placement,
log_device_placement=log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
print("**********************************************")
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("X_train").outputs[0]
dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("output/predictions").outputs[0]
# Generate batches for one epoch
for row in rows:
X_test = [row[1]]
batch_predictions = sess.run(predictions, {input_x: X_test, dropout_keep_prob:
yield (row[0], batch_predictions)
result = data_rdd.mapPartitions(lambda iter: predict(iter))
result.foreach(print)