We are using tensorflow library for face recognition. Our code works fine for single images. But when we run it as an API, the prediction time increases for every subsequent request. This happens because it searches for previously predicted images as well which should ideally not happen. Please find below the code I am using.
def train:
with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess:
test_set = _get_test_data(input_directory)
images, labels = _load_images_and_labels(test_set, image_size=160, batch_size=batch_size,
num_threads=4, num_epochs=1)
_load_model(model_filepath=model_path)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)
images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
embedding_layer = tf.get_default_graph().get_tensor_by_name("embeddings:0")
phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,coord=coord)
emb_array, label_array = _create_embeddings(embedding_layer, images, labels, images_placeholder,
phase_train_placeholder, sess)
classifier_filename = classifier_output_path
class_name, prob = _evaluate_classifier(emb_array, label_array, classifier_filename)
coord.request_stop()
coord.join(threads)
def _create_embeddings(embedding_layer, images, labels, images_placeholder, phase_train_placeholder, sess):
emb_array = None
label_array = None
try:
i = 0
while True:
print("batch images")
batch_images, batch_labels = sess.run([images, labels])
print('Processing iteration {} batch of size: {}'.format(i, len(batch_labels)))
emb = sess.run(embedding_layer,
feed_dict={images_placeholder: batch_images, phase_train_placeholder: False})
emb_array = np.concatenate([emb_array, emb]) if emb_array is not None else emb
label_array = np.concatenate([label_array, batch_labels]) if label_array is not None else batch_labels
i += 1
except tf.errors.OutOfRangeError:
pass
return emb_array, label_array
It searches for previously predicted images at
`batch_images, batch_labels = sess.run([images, labels])`
in the create embedding function. I think this is the problem of some unclosed threads because of which sess.run runs for all queued threads. Can anyone help me with this
During debugging I found that previously predicted images information was in default graph which is picked during execution of following lines
images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
embedding_layer = tf.get_default_graph().get_tensor_by_name("embeddings:0")
so by resetting graph before start of session will solve the problem of scanning previously predicted images as
tf.reset_default_graph()
Related
I finetuned two Mobilenet models on diferent datasets based on the tensorflow object_detection API example from here. When I use eager mode (tf.executing_eagerly() is True) using only one model then the inference runs at 0.036 seconds per image. When I load two models Keras required to convert to graph mode (tf.executing_eagerly() is False) and the inference runs at 1.8 seconds per image. What I'm doing wrong?
def inference(pipeline_config, checkpoint_path):
print('Building model and restoring weights', flush=True)
num_classes = 3
# Load pipeline config and build a detection model.
configs = config_util.get_configs_from_pipeline_file(pipeline_config)
model_config = configs['model']
model_config.ssd.num_classes = num_classes
detection_model = model_builder.build(
model_config=model_config, is_training=False)
ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
ckpt.restore(checkpoint_path).expect_partial()
# Run model through a dummy image so that variables are created
image, shapes = detection_model.preprocess(tf.zeros([1, 320, 320, 3]))
prediction_dict = detection_model.predict(image, shapes)
_ = detection_model.postprocess(prediction_dict, shapes)
print('Weights restored!')
return detection_model
def get_model_detection_function(detection_model):
"""Get a tf.function for detection."""
# Again, uncomment this decorator if you want to run inference eagerly
#tf.function
def detect(input_tensor):
"""Run detection on an input image.
Args:
input_tensor: A [1, height, width, 3] Tensor of type tf.float32.
Note that height and width can be anything since the image will be
immediately resized according to the needs of the model within this
function.
Returns:
A dict containing 3 Tensors (`detection_boxes`, `detection_classes`,
and `detection_scores`).
"""
preprocessed_image, shapes = detection_model.preprocess(input_tensor)
prediction_dict = detection_model.predict(preprocessed_image, shapes)
return detection_model.postprocess(prediction_dict, shapes)
return detect
def mainProcess():
print('Loading model 1...')
g1 = tf.Graph()
s1 = tf.compat.v1.Session(graph=g1)
with g1.as_default(), s1.as_default():
detection_model_1 = inference('config_1/pipeline.config', 'Checkpoint_1/ckpt-1')
detect_fn_1 = get_model_detection_function(detection_model_1)
s1.run(tf.compat.v1.global_variables_initializer())
print('Loading model 2...')
g2 = tf.Graph()
s2 = tf.compat.v1.Session(graph=g2)
with g2.as_default():
detection_model_2 = inference('config_2/pipeline.config', 'Checkpoint_2/ckpt-1')
detect_fn_2 = get_model_detection_function(detection_model_2)
s2.run(tf.compat.v1.global_variables_initializer())
for i, f in enumerate(listdir('images_dir/')):
...
... read the image
...
with g1.as_default():
with s1.as_default():
sec = time.time()
input_tensor = tf.convert_to_tensor(test_img, dtype=tf.float32)
detections = detect_fn_1(input_tensor)
detections = s1.run(detections)
curr = time.time()
print("Finished iterating in: " + str(curr - sec) + " seconds")
# the same for detection_model_2
For eager mode with only one model the mainProcess is:
def mainProcess():
print('Loading model...')
detection_model_1 = inference('config_1/pipeline.config', 'Checkpoint_1/ckpt-1')
detect_fn_1 = get_model_detection_function(detection_model_1)
for i, f in enumerate(listdir('images_dir/')):
...
... read the image
...
sec = time.time()
input_tensor = tf.convert_to_tensor(test_img, dtype=tf.float32)
detections = detect_fn_1(input_tensor)
print(detections['detection_boxes'][0].numpy())
print(detections['detection_scores'][0].numpy())
curr = time.time()
print("Finished iterating in: " + str(curr - sec) + " seconds")
I build an Inverse Compositional CNN but it reports errors as bellows:
ValueError: Variable left_src_tgt_warp/ICSTN/icnv1/weight already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
I find that using tf.reset_default_graph() can solve this problem. But I do not know where I should add it.
for l in range(opt.warpN):
with tf.variable_scope("ICSTN", reuse=l > 0) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with slim.arg_scope([slim.conv2d, slim.conv2d_transpose],
normalizer_fn=slim.batch_norm,
weights_regularizer=slim.l2_regularizer(0.05),
normalizer_params=batch_norm_params,
activation_fn=tf.nn.relu,
outputs_collections=end_points_collection):
imageWarp = inverse_warp(
inputImage,
depth,
pM,
intrinsics,
intrinsics_inv)
imageWarpAll.append(imageWarp)
feat = tf.reshape(imageWarp, [batch_size, H, W, C])
print('feat shape:', feat.get_shape())
print('pM_ini:', pM.get_shape())
with tf.variable_scope("icnv1"):
feat = conv2Layer(opt, feat, 4)
feat = tf.nn.relu(feat)
with tf.variable_scope("icnv2"):
feat = conv2Layer(opt, feat, 8)
feat = tf.nn.relu(feat)
feat = tf.nn.max_pool(feat, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
feat = tf.reshape(feat, [opt.batch_size, -1])
with tf.variable_scope("fc3"):
feat = linearLayer(opt, feat, 48)
feat = tf.nn.relu(feat)
with tf.variable_scope("fc4"):
feat = linearLayer(opt, feat, 6, final=True)
dp = tf.reshape(feat, [-1, 6])
print('dp: ', dp.get_shape())
dpM = pose_vec2mat(dp)
pM = tf.matmul(dpM, pM)
imageWarp = inverse_warp(
inputImage,
depth,
pM,
intrinsics,
intrinsics_inv)
imageWarpAll.append(imageWarp)
return imageWarpAll, pM
def build_train_graph():
with tf.name_scope("cnn1"):...
with tf.name_scope("cnn2"):...
with tf.name_scope("Inverse Compositional CNN"):...
def train(self, opt):
with tf.variable_scope(tf.get_variable_scope()):
for i in range(opt.num_gpus):
print('gpu:', i)
with tf.device('/gpu:%d' % i):
self.build_train_graph(L_img_splits[i], R_img_splits[i], L_cam2pix_splits[i], L_pix2cam_splits[i],
R_cam2pix_splits[i], R_pix2cam_splits[i], L_sca_splits[i], R_sca_splits[i],
reuse_variables)
self.collect_summaries(i)
tower_losses.append(self.total_loss)
reuse_variables = True
grads = opt_step.compute_gradients(self.total_loss)
tower_grads.append(grads)
grads = average_gradients(tower_grads)
apply_gradient_op = opt_step.apply_gr`enter code here`adients(grads, global_step=global_step)
incr_global_step = tf.assign(global_step, global_step + 1)
total_loss = tf.reduce_mean(tower_losses)
tf.summary.scalar('learning_rate', learning_rate, ['model_0'])
tf.summary.scalar('total_loss', total_loss, ['model_0'])
summary_op = tf.summary.merge_all('model_0')
# self.collect_summaries()
# SESSION
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# SAVER
summary_writer = tf.summary.FileWriter(
opt.checkpoint_dir + '/s%.1d_%.3d/' % (opt.seq_length, opt.img_height) + opt.model_name, sess.graph)
self.saver = tf.train.Saver()
# COUNT PARAM
total_num_parameters = 0
for variable in tf.trainable_variables():
total_num_parameters += np.array(variable.get_shape().as_list()).prod()
print("number of trainable parameters: {}".format(total_num_parameters))
# INIT
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
coordinator = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coordinator)
# LOAD CHECKPOINT IF SET
if opt.continue_train:
print("Resume training from previous checkpoint")
checkpoint = tf.train.latest_checkpoint(
os.path.join(opt.checkpoint_dir, 's%.1d_%.3d' % (opt.seq_length, opt.img_height), opt.model_name))
self.saver.restore(sess, checkpoint)
if opt.re_train:
sess.run(global_step.assign(0))
It's because of the for loop in the first part of the code that's possibly missing function name.
The loop tries to create the left_src_tgt_warp/ICSTN/icnv1/weight (same for icnv2 and so on):
def foo(num_layers):
opt = tf.placeholder(tf.float32, [None, 64])
for i in range(num_layers):
with tf.variable_scope("icnv1"):
feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)
foo(5)
ValueError: Variable icnv1/dense/kernel already exists, disallowed. Did you mean to set reuse=True
You need distinct names for Variables. One way to achieve this is like this:
def foo(num_layers):
opt = tf.placeholder(tf.float32, [None, 64])
for i in range(num_layers):
with tf.variable_scope("icnv1_layer_{}".format(i)):
feat = tf.layers.dense(opt, units=1, activation=tf.nn.relu)
we now have distinct names for each, icnv1_layer_1, icnv1_layer_2, etc. Depending on the depth.
Unless of course, you want a shared weights (e.g. it's the same layer, updates as one). In that case just set:
with tf.variable_scope("icnv1", reuse=tf.AUTO_REUSE):
I'm using Jupyter Notebooks to run my models and I recently realize that this error was occuring due to the fact that variables of my model was been saved on an "outter context". So when I restart kernel (and thus clean all my workspace variable) and run all cells the error gone away.
I have been using TensorFlow for a reasonable length of time now. and believed I had a thorough understanding of how a TensorFlow graph works and executes within a session. However, I have written all of my TensorFlow models in a script-like fashion as such:
import tensorflow as tf
import DataWorker
import Constants
x = tf.placeholder(tf.float32, [None, Constants.sequenceLength, DataWorker.numFeatures])
y = tf.placeholder(tf.float32, [None, 1])
xTensors = tf.unstack(x, axis=1) # [seqLength tensors of shape (batchSize, numFeatures)]
W = tf.Variable(tf.random_normal([Constants.numHidden, 1])) # Weighted matrix
b = tf.Variable(tf.random_normal([1])) # Bias
cell = tf.contrib.rnn.BasicLSTMCell(Constants.numHidden, forget_bias=Constants.forgetBias)
outputs, finalState = tf.nn.static_rnn(cell, xTensors, dtype=tf.float32)
# predictions = [tf.add(tf.matmul(output, W), b) for output in outputs] # List of predictions after each time step
prediction = tf.add(tf.matmul(outputs[-1], W), b) # Prediction after final time step
prediction = tf.tanh(prediction) # Activation
mse = tf.losses.mean_squared_error(predictions=prediction, labels=y) # Mean loss over entire batch
accuracy = tf.reduce_mean(1 - (tf.abs(y - prediction) / DataWorker.labelRange)) # Accuracy over entire batch
optimiser = tf.train.AdamOptimizer(Constants.learningRate).minimize(mse) # Backpropagation
with tf.Session() as session:
session.run(tf.global_variables_initializer())
# #############################################
# TRAINING
# #############################################
for epoch in range(Constants.numEpochs):
print("***** EPOCH:", epoch + 1, "*****\n")
IDPointer, TSPointer = 0, 0 # Pointers to current ID and timestamp
epochComplete = False
batchNum = 0
while not epochComplete:
batchNum += 1
batchX, batchY, IDPointer, TSPointer, epochComplete = DataWorker.generateBatch(IDPointer, TSPointer, isTraining=True)
dict = {x: batchX, y: batchY}
session.run(optimiser, dict)
if batchNum % 1000 == 0 or epochComplete:
batchLoss = session.run(mse, dict)
batchAccuracy = session.run(accuracy, dict)
print("Iteration:", batchNum)
print(batchLoss)
print(str("%.2f" % (batchAccuracy * 100) + "%\n"))
# #############################################
# TESTING
# #############################################
testX, testY, _, _, _ = DataWorker.generateBatch(0, 0, isTraining=False)
testAccuracy = session.run(accuracy, {x: testX, y: testY})
print("Testing Accuracy:", str("%.2f" % (testAccuracy * 100) + "%"))
But now, for practicality and readability, I want to implement my model as a class, but have encountered many problems with initializing my variables, etc.
This is the closest I have got to implementing the above example using my own LSTM class
Model.py
import tensorflow as tf
import Constants
import DataWorker # Remove this dependency
class LSTM():
"""docstring."""
def __init__(self,
inputDimensionList,
outputDimensionList,
numLayers=Constants.numLayers,
numHidden=Constants.numHidden,
learningRate=Constants.learningRate,
forgetBias=Constants.forgetBias
):
"""docstring."""
self.batchInputs = tf.placeholder(tf.float32, [None] + inputDimensionList)
self.batchLabels = tf.placeholder(tf.float32, [None] + outputDimensionList)
self.weightedMatrix = tf.Variable(tf.random_normal([numHidden] + outputDimensionList))
self.biasMatrix = tf.Variable(tf.random_normal(outputDimensionList))
self.cell = tf.contrib.rnn.BasicLSTMCell(numHidden, forget_bias=forgetBias)
self.numLayers = numLayers
self.numHidden = numHidden
self.learningRate = learningRate
self.forgetBias = forgetBias
self.batchDict = {}
self.batchInputTensors = None
self.batchOutputs = None # All needed as instance variables?
self.batchFinalStates = None
self.batchPredictions = None
self.batchLoss = None
self.batchAccuracy = None
self.initialised = False
self.session = tf.Session()
# Take in activation, loss and optimiser FUNCTIONS as args
def execute(self, command):
"""docstring."""
return self.session.run(command, self.batchDict)
def setBatchDict(self, inputs, labels):
"""docstring."""
self.batchDict = {self.batchInputs: inputs, self.batchLabels: labels}
self.batchInputTensors = tf.unstack(self.batchInputs, axis=1)
def processBatch(self):
"""docstring."""
self.batchOutputs, self.batchFinalState = tf.nn.static_rnn(self.cell, self.batchInputTensors, dtype=tf.float32)
pred = tf.tanh(tf.add(tf.matmul(self.batchOutputs[-1], self.weightedMatrix), self.biasMatrix))
mse = tf.losses.mean_squared_error(predictions=pred, labels=self.batchLabels)
optimiser = tf.train.AdamOptimizer(self.learningRate).minimize(mse)
if not self.initialised:
self.session.run(tf.global_variables_initializer())
self.initialised = True
with tf.variable_scope("model") as scope:
if self.initialised:
scope.reuse_variables()
self.execute(optimiser)
self.batchPredictions = self.execute(pred)
self.batchLoss = self.execute(tf.losses.mean_squared_error(predictions=self.batchPredictions, labels=self.batchLabels))
self.batchAccuracy = self.execute(tf.reduce_mean(1 - (tf.abs(self.batchLabels - self.batchPredictions) / DataWorker.labelRange)))
return self.batchPredictions, self.batchLabels, self.batchLoss, self.batchAccuracy
def kill(self):
"""docstring."""
self.session.close()
This class is quite messy, especially processBatch() as I have just been trying to get it to work before refining it.
I then run my model here:
Main.py
import DataWorker
import Constants
from Model import LSTM
inputDim = [Constants.sequenceLength, DataWorker.numFeatures]
outputDim = [1]
lstm = LSTM(inputDimensionList=inputDim, outputDimensionList=outputDim)
# #############################################
# TRAINING
# #############################################
for epoch in range(Constants.numEpochs):
print("***** EPOCH:", epoch + 1, "*****\n")
IDPointer, TSPointer = 0, 0 # Pointers to current ID and timestamp
epochComplete = False
batchNum = 0
while not epochComplete:
batchNum += 1
batchX, batchY, IDPointer, TSPointer, epochComplete = DataWorker.generateBatch(IDPointer, TSPointer, isTraining=True)
lstm.setBatchDict(batchX, batchY)
batchPredictions, batchLabels, batchLoss, batchAccuracy = lstm.runBatch()
if batchNum % 1000 == 0 or epochComplete:
print("Iteration:", batchNum)
print("Pred:", batchPredictions[-1], "\tLabel:", batchLabels[-1])
print("Loss:", batchLoss)
print("Accuracy:", str("%.2f" % (batchAccuracy * 100) + "%\n"))
# #############################################
# TESTING
# #############################################
testX, testY, _, _, _ = DataWorker.generateBatch(0, 0, isTraining=False)
lstm.setBatchDict(testX, testY)
_, _, _, testAccuracy = lstm.runBatch()
print("Testing Accuracy:", str("%.2f" % (testAccuracy * 100) + "%"))
lstm.kill()
A single passthrough of the graph is executed fine, when all the variables are initialized, but it is on the second iteration where I get the error
ValueError: Variable rnn/basic_lstm_cell/kernel/Adam/ already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
optimiser = tf.train.AdamOptimizer(self.learningRate).minimize(mse)
I Googled this problem and learned that using scope.reuse_variables() should stop it trying to initialize the AdamOptimizer a second time, but cleary this isn't working how I have implemented it. How can I fix this issue?
As a side note, is my method of creating the TensorFlow session as an instance variable within my LSTM class acceptable, or should I create the session in Main and then pass it into the LSTM instance?
In general I wrap anything that creates variables under the hood with tf.make_template when doing object oriented model building.
However, you should avoid adding ops to the graph in a training loop, which looks like it's happening here. They will build up and cause problems, and likely give you incorrect results. Instead, define the graph (with inputs from tf.data, placeholders, or queues) and only loop over a session.run call. Even better, structure your code as an Estimator and this will be enforced.
I can't enable epoch limits on my string_input_producer without getting a OutOfRange error (requested x, current size 0). It doesn't seem to matter how many elements I request, there is always 0 available.
Here is my FileQueue builder:
def get_queue(base_directory):
files = [f for f in os.listdir(base_directory) if f.endswith('.bin')]
shuffle(files)
file = [os.path.join(base_directory, files[0])]
fileQueue = tf.train.string_input_producer(file, shuffle=False, num_epochs=1)
return fileQueue
If I remove num_epochs=1 from the string_input_producer it can create samples fine.
My input pipeline:
def input_pipeline(instructions, fileQueue):
example, label, feature_name_list = read_binary_format(fileQueue, instructions)
num_preprocess_threads = 16
capacity = 20
example, label = tf.train.batch(
[example, label],
batch_size=50000, # set the batch size way bigger so we always return the full amount of samples from the file
allow_smaller_final_batch=True,
capacity=capacity,
num_threads=num_preprocess_threads)
return example, label
And lastly my session:
with tf.Session(graph=tf.Graph()) as sess:
train_inst_set = sf.DeserializationInstructions.from_filename(os.path.join(input_dir, "Train/config.json"))
fileQueue = sf.get_queue(os.path.join(input_dir, "Train"))
features_train, labels_train = sf.input_pipeline(train_inst_set, fileQueue)
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
train_feature_batch, train_label_batch = sess.run([features_train, labels_train])
The issue was caused by this: Issue #1045
For whatever reason, tf.global_variable_initialiser does not initialise all variables. You need to initialise the local variables too.
Add
sess.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))
to your session.
I need to pass an image during session Run in Tensorflow
I have tried
myimage="andrew/semaphore.jpg"
mydict={'ssignals':myimage}
output = sess.run(train_prediction, feed_dict=mydict)
but doesn't work
Returns me the error:
Cannot interpret feed_dict key as Tensor
train_prediction tensor has the shape=(3, 120), dtype=float32
(120 is the number of training classes in which images are grouped)
the program load and labels input images with this method
def write_records_file(dataset, record_location):
writer = None
global counter
global start_time
current_index = 0
for breed, images_filenames in dataset.items():
for image_filename in images_filenames:
with tf.Graph().as_default():
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
if current_index % 100 == 0:
if writer:
writer.close()
record_filename = "{record_location}-{current_index}.tfrecords".format(
record_location=record_location,
current_index=current_index)
writer = tf.python_io.TFRecordWriter(record_filename)
current_index += 1
image_file = tf.read_file(image_filename)
image = tf.image.decode_jpeg(image_file)
grayscale_image = tf.image.rgb_to_grayscale(image)
resized_image = tf.image.resize_images(
grayscale_image, [250, 251])
try:
image_bytes = sess.run(
tf.cast(resized_image, tf.uint8)).tobytes()
except:
continue
image_label = breed.encode("utf-8")
example = tf.train.Example(features=tf.train.Features(feature={
'label': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_label])),
'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_bytes]))
}))
writer.write(example.SerializeToString())
counter += 1
than a batch load the record and feed the CNN
EDIT
I don't use same symbol-key value please before flag as duplicate pay attention.