For better context, I have uploaded a pre-trained model on cloud ml. It's an inceptionV3 model converted from keras to acceptable format in tensorflow.
from keras.applications.inception_v3 import InceptionV3
model = InceptionV3(weights='imagenet')
from keras.models import Model
intermediate_layer_model = Model(inputs=model.input,outputs=model.layers[311].output)
with tf.Graph().as_default() as g_input:
input_b64 = tf.placeholder(shape=(1,),
dtype=tf.string,
name='input')
input_bytes = tf.decode_base64(input_b64[0])
image = tf.image.decode_image(input_bytes)
image_f = tf.image.convert_image_dtype(image, dtype=tf.float32)
input_image = tf.expand_dims(image_f, 0)
output = tf.identity(input_image, name='input_image')
g_input_def = g_input.as_graph_def()
K.set_learning_phase(0)
sess = K.get_session()
from tensorflow.python.framework import graph_util
g_trans = sess.graph
g_trans_def = graph_util.convert_variables_to_constants(sess,
g_trans.as_graph_def(),
[intermediate_layer_model.output.name.replace(':0','')])
with tf.Graph().as_default() as g_combined:
x = tf.placeholder(tf.string, name="input_b64")
im, = tf.import_graph_def(g_input_def,
input_map={'input:0': x},
return_elements=["input_image:0"])
pred, = tf.import_graph_def(g_trans_def,
input_map={intermediate_layer_model.input.name: im,
'batch_normalization_1/keras_learning_phase:0': False},
return_elements=[intermediate_layer_model.output.name])
with tf.Session() as sess2:
inputs = {"inputs": tf.saved_model.utils.build_tensor_info(x)}
outputs = {"outputs":tf.saved_model.utils.build_tensor_info(pred)}
signature =tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs,
outputs=outputs,
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
)
# save as SavedModel
b = tf.saved_model.builder.SavedModelBuilder('inceptionv4/')
b.add_meta_graph_and_variables(sess2,
[tf.saved_model.tag_constants.SERVING],
signature_def_map={'serving_default': signature})
b.save()
The generated pb file works fine when I use it locally. But when I deploy it on cloud ml I get the following error.
RuntimeError: Prediction failed: Error during model execution: AbortionError(code=StatusCode.INVALID_ARGUMENT, details="Invalid character found in base64.
[[Node: import/DecodeBase64 = DecodeBase64[_output_shapes=[<unknown>], _device="/job:localhost/replica:0/task:0/device:CPU:0"](import/strided_slice)]]")
Following is the code I use for getting local predictions.
import base64
import json
with open('MEL_BE_0.jpg', 'rb') as image_file:
encoded_string = str(base64.urlsafe_b64encode(image_file.read()),'ascii')
import tensorflow as tf
with tf.Session(graph=tf.Graph()) as sess:
MetaGraphDef=tf.saved_model.loader.load(
sess,
[tf.saved_model.tag_constants.SERVING],
'inceptionv4')
input_tensor = tf.get_default_graph().get_tensor_by_name('input_b64:0')
print(input_tensor)
avg_tensor = tf.get_default_graph().get_tensor_by_name('import_1/avg_pool/Mean:0')
print(avg_tensor)
predictions = sess.run(avg_tensor, {input_tensor: [encoded_string]})
And finally following is the code snippet that I use for wrapping the encoded string in the request that is sent to the cloud-ml engine.
request_body= json.dumps({"key":"0", "image_bytes": {"b64": [encoded_string]}})
It looks like you are trying to do the base64 decoding in TensorFlow and use the {"b64": ...} JSON format. You need to do one or the other; we typically recommend the latter.
As a side note, your input placeholder must have an outer dimension of None. That can make some things tricky, e.g., you'll either have to reshape the dimensions to be size 1 (which will prevent you from using the batch prediction service in its current state) or you'll have to us tf.map_fn to apply the same set of transformations to each element of the input "batch". You can find an example of that technique in this example.
Finally, I recommend the use of tf.saved_model.simple_save.
Putting it altogether, here is some modified code. Note that I'm inlining your input function (as opposed to serializing it to a graph def and reimporting):
HEIGHT = 299
WIDTH = 299
# Get Keras Model
from keras.applications.inception_v3 import InceptionV3
model = InceptionV3(weights='imagenet')
from keras.models import Model
intermediate_layer_model = Model(inputs=model.input,outputs=model.layers[311].output)
K.set_learning_phase(0)
sess = K.get_session()
from tensorflow.python.framework import graph_util
g_trans = sess.graph
g_trans_def = graph_util.convert_variables_to_constants(sess,
g_trans.as_graph_def(),
[intermediate_layer_model.output.name.replace(':0','')])
# Create inputs to model and export
with tf.Graph().as_default() as g_combined:
def decode_and_resize(image_bytes):
image = tf.image.decode_image(image_bytes)
# Note resize expects a batch_size, but tf_map supresses that index,
# thus we have to expand then squeeze. Resize returns float32 in the
# range [0, uint8_max]
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(
image, [HEIGHT, WIDTH], align_corners=False)
image = tf.squeeze(image, squeeze_dims=[0])
image = tf.cast(image, dtype=tf.uint8)
return image
input_byes = tf.placeholder(shape=(None,),
dtype=tf.string,
name='input')
images = tf.map_fn(
decode_and_resize, input_bytes, back_prop=False, dtype=tf.uint8)
images = tf.image.convert_image_dtype(images, dtype=tf.float32)
pred, = tf.import_graph_def(g_trans_def,
input_map={intermediate_layer_model.input.name: images,
'batch_normalization_1/keras_learning_phase:0': False},
return_elements=[intermediate_layer_model.output.name])
with tf.Session() as sess2:
tf.saved_model.simple_save(
sess2,
model_dir='inceptionv4/'
inputs={"inputs": input_bytes},
outputs={"outputs": pred})
Note: I'm not 100% certain that the shapes of intermediate_layer_model and images are compatible. The shape of images will be [None, height, width, num_channels].
Also note that your local prediction code will change a bit. You don't base64 encode the images and you need to send a "batch"/list of images rather than single images. Something like:
with open('MEL_BE_0.jpg', 'rb') as image_file:
encoded_string = image_file.read()
input_tensor = tf.get_default_graph().get_tensor_by_name('input:0')
print(input_tensor)
avg_tensor = tf.get_default_graph().get_tensor_by_name('import_1/avg_pool/Mean:0')
print(avg_tensor)
predictions = sess.run(avg_tensor, {input_tensor: [encoded_string]})
You didn't specify whether you're doing batch prediction or online prediction, which have similar but slightly different "formats" for the inputs. In either case, your model is not exporting a "key" field (did you mean to? It's probably helpful for batch prediction, but not for online).
For batch prediction, the file format is JSON lines; each line contains one example. Each line can be generated like so from Python:
example = json.dumps({"image_bytes": {"b64": ENCODED_STRING}})
(Note the omission of "key" for now). Since you only have one input, there is a shorthand:
example = json.dumps({"b64": ENCODED_STRING})
If you want to do online prediction, you'll note that if you are using gcloud to send requests, you actually use the same file format as for batch prediction.
In fact, we highly recommend using gcloud ml-engine local predict --json-instances=FILE --model-dir=... before deploying to the cloud to help debug.
If you intend to use some other client besides gcloud, e.g., in a web app, mobile app, frontend server, etc., then you won't be sending a file and you need to construct the full request yourself. It's very similar to the file format above. Basically, take each line of the JSON lines file and put them in an array calle "instances", i.e.,
request_body= json.dumps({"instances": [{"image_bytes": {"b64": [encoded_string]}}]})
You can use the same syntactic sugar if you'd like:
request_body= json.dumps({"instances": [{"b64": [encoded_string]}]})
I hope this helps!
Related
I have an issue implementing an input pipeline with the new tf.data tensorflow class.
Specifically, when I include a convolution operation to the preprocessing - which I add to the pipeline with the map method - I get the following error
tensorflow.python.framework.errors_impl.UnimplementedError: Generic conv implementation only supports NHWC tensor format for now.
[[{{node conv_debug}} = Conv2D[T=DT_FLOAT, data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true](conv_debug-0-TransposeNHWCToNCHW-LayoutOptimizer, ArithmeticOptimizer/FoldMultiplyIntoConv_scaled_conv_debug_Const)]]
When I exclude the convolution from the pipeline, everything works as expected.
I attach below the minimal code needed to reproduce the problem.
Tested with 3 configurations:
Tensorflow 1.12.0, CUDA 10.0, CUDnn 7.4.1, got the error.
Tensorflow 1.11.0, CUDA 9.0, CUDnn 7.3.1, got the error.
Tensorflow 1.8.0, CUDA 8.0, CUDnn 6.0, it works.
Am I doing it wrong or is it a CUDA/CUDnn related issue?
Thanks!
import numpy as np
import tensorflow as tf
image_height, image_width = 100, 200
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def serialize_to_record(record_name, label, image):
"""Create a data record and store it"""
writer = tf.python_io.TFRecordWriter(record_name)
image_raw = image.tostring()
label_raw = label
sample = tf.train.Example(features=tf.train.Features(feature={
'image_raw': _bytes_feature(image_raw),
'label_raw': _bytes_feature(label_raw)}))
writer.write(sample.SerializeToString())
writer.close()
return
def _dataset_parser(record):
"""Read and deserialize a tensorflow record"""
parsed = tf.parse_single_example(record,
features={'image_raw': tf.FixedLenFeature([], tf.string),
'label_raw': tf.FixedLenFeature([], tf.string)})
image_ = tf.decode_raw(parsed['image_raw'], tf.uint8)
image_.set_shape(image_height * image_width * 3)
image_ = tf.reshape(image_, (image_height, image_width, 3))
image = tf.cast(image_, tf.float32) / 255.0
label = parsed['label_raw']
return {'image': image, 'label': label}
def _dataset_preprocessor(datum):
"""dummy preprocessor consisting of a convolution with a random kernel"""
image = datum['image']
kernel = np.random.rand(5, 5, 3, 3)
kernel_tf = tf.constant(kernel, dtype=tf.float32)
image = tf.expand_dims(image, axis=0)
image = tf.nn.conv2d(image, kernel_tf, [1, 1, 1, 1], padding='SAME', name='conv_debug')
image = tf.squeeze(image, axis=0)
datum['image'] = image
return datum
def _dataset_operator(record):
"""define a sequence of operation to run on the dataset"""
datum = _dataset_parser(record)
datum = _dataset_preprocessor(datum)
return datum
def _dataset_operator_noconv(record):
"""define a sequence of operation to run on the dataset"""
datum = _dataset_parser(record)
return datum
if __name__ == '__main__':
# create a random tensor
image = (255.0 * np.random.rand(image_height, image_width, 3)).astype(np.uint8)
record_path = 'example.tfrecord'
# store a tf record to disk
serialize_to_record(record_path, label='example', image=image)
# build a dummy dataset of copies of the generated image
N = 32
dataset_filenames = [record_path for n in range(N)]
dataset = tf.data.TFRecordDataset(dataset_filenames)
# add parser and preprocessor to the pipeline
include_convolution_to_pipeline = True
if include_convolution_to_pipeline:
dataset = dataset.map(_dataset_operator)
else:
dataset = dataset.map(_dataset_operator_noconv)
# complete pipeline for iteratively visiting the dataset in batches of 8 samples
dataset = dataset.shuffle(buffer_size=100)
dataset = dataset.batch(8)
dataset = dataset.repeat()
iterator = dataset.make_initializable_iterator()
next_data = iterator.get_next()
# init session and go for the first batch
sess = tf.Session()
sess.run(iterator.initializer)
next_data_ = sess.run(next_data)
print('***')
As error message states, convolution operation requires NCHW data format. Regardless of what data format you want, it still needs batch_size as one of dimensions. But you're trying to apply map function prior to batching. It's usually not standard order but if you need convolution, you need to apply map function after batch.
dataset = dataset.map(_dataset_operator)
dataset = dataset.shuffle(buffer_size=100)
dataset = dataset.batch(8)
dataset = dataset.map(_dataset_operator)
dataset = dataset.repeat()
It is tensorflow's layout optimizer problem.
Tensorflow "map" function executes the graph in CPU and placing tensors in the map otherwise confuses the layout optimizer.
Placing tf.device("/cpu:0") when creating the tensors inside the map function solves the layout optimizer confusion. Another option is to disable the layout optimizer which may cost in extra training time( it may not be feasible not to optimize the whole graph layout to execute "map" phase ).
There is already an open issue regarding this problem :
https://github.com/tensorflow/tensorflow/issues/26411
As this is a workaround, I think more robust solutions(executing "map" tensors in GPU, fixes for layout optimizer etc.) may come in the next releases of TF. But for now, suggested workaround solves my problem without hassling any layout deoptimization issues.
when I learn a tensorflow project,find one line code:
cls_prob, box_pred = sess.run([output_cls_prob, output_box_pred], feed_dict={input_img: blob})
But, this line code It took a lot of time. (use CPU need 15 seconds...┭┮﹏┭┮)
By consulting information, I find use function 'dataset' could solve this problem which took a lot of time, How should I use it?
source of 'blob':
img = cv2.imread('./imgs/001.jpg')
img_scale = float(600) / min(img_data.shape[0], img_data.shape[1])
if np.round(img_scale * max(img_data.shape[0], img_data.shape[1])) > 1200:
img_scale = float(1200) / max(img_data.shape[0], img_data.shape[1])
img_data = cv2.resize(img_data, None, None, fx=img_scale, fy=img_scale, interpolation=cv2.INTER_LINEAR)
img_orig = img_data.astype(np.float32, copy=True)
blob = np.zeros((1, img_data.shape[0], img_data.shape[1], 3),dtype=np.float32)
blob[0, 0:img_data.shape[0], 0:img_data.shape[1], :] = img_orig
source of 'output_cls_prob'&'output_box_pred'&'input_img':
# Actually,read PB model...
input_img = sess.graph.get_tensor_by_name('Placeholder:0')
output_cls_prob = sess.graph.get_tensor_by_name('Reshape_2:0')
output_box_pred = sess.graph.get_tensor_by_name('rpn_bbox_pred/Reshape_1:0')
Parameter type：
blob：type 'numpy.ndarray'
output_cls_prob：class 'tensorflow.python.framework.ops.Tensor'
output_box_pred：class 'tensorflow.python.framework.ops.Tensor'
input_img：class 'tensorflow.python.framework.ops.Tensor'
tf.data is the recommended API for tensorflow input pipelines. Here is a tutorial on tensorflow.org. For your example, the section "Decoding image data and resizing it" could be most useful. For example, you could do something like:
# Reads an image from a file, decodes it into a dense tensor, and resizes it
# to a fixed shape.
def _parse_function(filename):
image_string = tf.read_file(filename)
image_decoded = tf.image.decode_jpeg(image_string)
image_resized = tf.image.resize_images(image_decoded, [new_width, new_height])
image_resized = tf.expand_dims(image_resized, 0) # Adds size 1 dimension
return image_resized
# A vector of filenames.
filenames = tf.constant(["./imgs/001.jpg", ...])
dataset = tf.data.Dataset.from_tensor_slices(filenames)
dataset = dataset.map(_parse_function)
And instead of having input_img be a placeholder, change:
input_img = tf.placeholder(tf.float32)
output_class_prob, output_class_pred = (... use input_img ...)
to:
iterator = dataset.make_one_shot_iterator()
input_img = iterator.get_next()
output_class_prob, output_class_pred = (... use input_img ...)
First of all you should know that the use of Dataset API has a great impact in performance when multiples GPUs are used... Otherwise is almost identical to feed_dict. I recommend you to read this other answer from a TF developer, it has almost everything one needs to know to create a mental image of the benefits of this new API.
When following the tensorflow image classification tutorial, at first it caches the bottleneck of each image:
def: cache_bottlenecks())
I have rewritten the training using tensorflow's Estimator. This really simplified all the code. However I want to cache the bottleneck features here.
Here is my model_fn. I want to cache the results of the dense layer so I can make changes to the actual training without having to compute the bottlenecks each time.
How can I accomplish that?
def model_fn(features, labels, mode, params):
is_training = mode == tf.estimator.ModeKeys.TRAIN
num_classes = len(params['label_vocab'])
module = hub.Module(params['module_spec'], trainable=is_training and params['train_module'])
bottleneck_tensor = module(features['image'])
with tf.name_scope('final_retrain_ops'):
logits = tf.layers.dense(bottleneck_tensor, units=num_classes, trainable=is_training) # save this?
def train_op_fn(loss):
optimizer = tf.train.AdamOptimizer()
return optimizer.minimize(loss, global_step=tf.train.get_global_step())
head = tf.contrib.estimator.multi_class_head(n_classes=num_classes, label_vocabulary=params['label_vocab'])
return head.create_estimator_spec(
features, mode, logits, labels, train_op_fn=train_op_fn
)
TF cannot work as you code. You should:
Export bottleneck to file from the raw net.
Use bottleneck result as input, use another net to train your data.
To expand on what #Feng said:
see TFRecords and TFExamples and Load Images
Something like this should work (untested):
# Serialize the data into two tfrecord files
tf.enable_eager_execution()
feature_extractor = ...
features_file = tf.python_io.TFRecordWriter('features.tfrec')
label_file = tf.python_io.TFRecordWriter('labels.tfrec')
for images, labels in dataset:
features = feature_extractor(images)
features_file.write(tf.serialize_tensor(features))
label_file.write(tf.serialize_tensor(labels))
# Parse the files and zip them together
def parse(type, shape):
_def parse(x):
result = tf.parse_tensor(x, out_type=shape)
result = tf.reshape(result, FEATURE_SHAPE)
return result
return parse
features_ds = tf.data.TFRecordDataset('features.tfrec')
features_ds = features_ds.map(parse(tf.float32, FEATURE_SHAPE), num_parallel_calls=AUTOTUNE)
labels_ds = tf.data.TFRecordDataset('labels.tfrec')
labels_ds = labels_ds.map(parse(tf.float32, FEATURE_SHAPE), num_parallel_calls=AUTOTUNE)
ds = tf.data.Dataset.zip(features_ds, labels_ds)
ds = ds.unbatch().shuffle().repeat().batch().prefetch()...
You might also be able to do it using Dataset.cache, but I'm not 100% sure of the details.
I have already converted a pre-trained .ckpt file to .pb file freezing the model and saving the weighs as well. What I am trying to do now is to make a simple inference using that .pb file and extract and save output image. The model is a (Fully Convolutional Network for Semantic Segmentation) downloaded from here : https://github.com/MarvinTeichmann/KittiSeg . So far I have managed to, load the image, set the default tf graph and import the graph defined by the model on that, read the input and the output tensors and run the session (error here).
import tensorflow as tf
import os
import numpy as np
from tensorflow.python.platform import gfile
from PIL import Image
# Read the image & get statstics
img=Image.open('/path-to-image/demoImage.png')
img.show()
width, height = img.size
print(width)
print(height)
#Plot the image
#image.show()
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
# Load the graph in graph_def
print("load graph")
# We load the protobuf file from the disk and parse it to retrive the unserialized graph_drf
with gfile.FastGFile("/path-to-FCN-model/FCN8.pb",'rb') as f:
#Set default graph as current graph
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
#sess.graph.as_default() #new line
# Import a graph_def into the current default Graph
tf.import_graph_def(graph_def, name='')
# Print the name of operations in the session
#for op in sess.graph.get_operations():
#print "Operation Name :",op.name # Operation name
#print "Tensor Stats :",str(op.values()) # Tensor name
# INFERENCE Here
l_input = graph.get_tensor_by_name('Placeholder:0')
l_output = graph.get_tensor_by_name('save/Assign_38:0')
print "l_input", l_input
print "l_output", l_output
print
print
# Acceptable feed values include Python scalars, strings, lists, numpy ndarrays, or TensorHandles.
result = sess.run(l_output, feed_dict={l_input : img})
print(results)
print("Inference done")
# Info
# First Tensor name : Placeholder:0
# Last tensor name : save/Assign_38:0"
Can the error come from the format of the image (e.g should I convert .png to another format?). Is it another fundamental error?
I managed to fix the error, below is the working script to inference a single image on Fully Convolutional Networks (for whoever is interesting in an alternative segmentation algorithm from SEGNET) . This model use billinear interpolation for scaling rather than an un-pooling layer. Anyway, because the model is available to download in a .chkpt format, you must first freeze the model and save it as a .pb file. Later on, you must pass the network from TF optimizer to set Dropout probabilities to 1. Afterwards, set the correct input and output tensor name in this script and the inference works correctly, extracting the segmented image.
import tensorflow as tf # Default graph is initialized when the library is imported
import os
from tensorflow.python.platform import gfile
from PIL import Image
import numpy as np
import scipy
from scipy import misc
import matplotlib.pyplot as plt
import cv2
with tf.Graph().as_default() as graph: # Set default graph as graph
with tf.Session() as sess:
# Load the graph in graph_def
print("load graph")
# We load the protobuf file from the disk and parse it to retrive the unserialized graph_drf
with gfile.FastGFile("/path-to-protobuf/FCN8_Freezed.pb",'rb') as f:
print("Load Image...")
# Read the image & get statstics
image = scipy.misc.imread('/Path-To-Image/uu_000010.png')
image = image.astype(float)
Input_image_shape=image.shape
height,width,channels = Input_image_shape
print("Plot image...")
#scipy.misc.imshow(image)
# Set FCN graph to the default graph
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
sess.graph.as_default()
# Import a graph_def into the current default Graph (In this case, the weights are (typically) embedded in the graph)
tf.import_graph_def(
graph_def,
input_map=None,
return_elements=None,
name="",
op_dict=None,
producer_op_list=None
)
# Print the name of operations in the session
for op in graph.get_operations():
print "Operation Name :",op.name # Operation name
print "Tensor Stats :",str(op.values()) # Tensor name
# INFERENCE Here
l_input = graph.get_tensor_by_name('Inputs/fifo_queue_Dequeue:0') # Input Tensor
l_output = graph.get_tensor_by_name('upscore32/conv2d_transpose:0') # Output Tensor
print "Shape of input : ", tf.shape(l_input)
#initialize_all_variables
tf.global_variables_initializer()
# Run Kitty model on single image
Session_out = sess.run( l_output, feed_dict = {l_input : image}
Have you already looked at the demo.py. There is shown at line 141 how they modify the input of the graph:
# Create placeholder for input
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
# build Tensorflow graph using the model from logdir
prediction = core.build_inference_graph(hypes, modules,
image=image)
And at line 164 how the image is opened:
image = scp.misc.imread(input_image)
Which is fed directly to image_pl. The only point is that core.build_inference_graph is a TensorVision call.
Note, it would be interesting to provide the exact error message as as input as well.
I trained an inception_resnet_v2 model for the flowers images following the README at https://github.com/tensorflow/models/tree/master/slim
I got my graph.pbtxt file out of this after training with which I converted to a graph.pb file with the following code:
import tensorflow as tf
from google.protobuf import text_format
def convert_pbtxt_to_graphdef(filename):
"""Returns a `tf.GraphDef` proto representing the data in the given pbtxt file.
Args:
filename: The name of a file containing a GraphDef pbtxt (text-formatted
`tf.GraphDef` protocol buffer data).
Returns:
A `tf.GraphDef` protocol buffer.
"""
with tf.gfile.FastGFile(filename, 'r') as f:
graph_def = tf.GraphDef()
file_content = f.read()
# Merges the human-readable string in `file_content` into `graph_def`.
text_format.Merge(file_content, graph_def)
return graph_def
with tf.gfile.FastGFile('/foo/bar/workspace/results/graph.pb', 'wb') as f:
f.write(convert_pbtxt_to_graphdef('/foo/bar/workspace/results/graph.pbtxt'))
After getting this file I tried feeding the trained model a random image using tensorflow's classify_image.py found here: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/image/imagenet/classify_image.py
using my .pb, .pbtxt, and my labels file, however, I get the following error:
Traceback (most recent call last):
File "classify_image.py", line 212, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "classify_image.py", line 208, in main
run_inference_on_image(image)
File "classify_image.py", line 170, in run_inference_on_image
softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2615, in get_tensor_by_name
return self.as_graph_element(name, allow_tensor=True, allow_operation=False)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2466, in as_graph_element
return self._as_graph_element_locked(obj, allow_tensor, allow_operation)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2508, in _as_graph_element_locked
"graph." % (repr(name), repr(op_name)))
KeyError: "The name 'softmax:0' refers to a Tensor which does not exist. The operation, 'softmax', does not exist in the graph."
The problem with slim, in fact tensorflow/models, is that the framework and so the produced models don't really fit the prediction use case:
TF-slim is a new lightweight high-level API of TensorFlow
(tensorflow.contrib.slim) for defining, training and evaluating
complex models.
(Source https://github.com/tensorflow/models/tree/master/slim)
The main problem at prediction time is, that it only works well with the checkpoint files created by the Saver class. When using checkpoint files the assign_from_checkpoint_fn() method can be used to initialize all the variables with the trained parameters contained in the checkpoint. On the other hand, in the situation when having the GraphDef file *.pb only, you kinda lost. There is a nice trick though.
The key idea is to inject a tf.placeholder variable for the input image(s) into the computation graph after you saved your trained model as a checkpoint. The following script (convert_checkpoint_to_pb.py) reads a checkpoint, inserts a placeholder, converts the graph variables to constants and dumps it to a *.pb file.
import tensorflow as tf
from tensorflow.contrib import slim
from nets import inception
from tensorflow.python.framework.graph_util import convert_variables_to_constants
from tensorflow.python.tools.optimize_for_inference_lib import optimize_for_inference
from preprocessing import inception_preprocessing
checkpoints_dir = '/path/to/your/checkpoint_dir/'
OUTPUT_PB_FILENAME = 'minimal_graph.proto'
NUM_CLASSES = 2
# We need default size of image for a particular network.
# The network was trained on images of that size -- so we
# resize input image later in the code.
image_size = inception.inception_resnet_v2.default_image_size
with tf.Graph().as_default():
# Inject placeholder into the graph
input_image_t = tf.placeholder(tf.string, name='input_image')
image = tf.image.decode_jpeg(input_image_t, channels=3)
# Resize the input image, preserving the aspect ratio
# and make a central crop of the resulted image.
# The crop will be of the size of the default image size of
# the network.
# I use the "preprocess_for_eval()" method instead of "inception_preprocessing()"
# because the latter crops all images to the center by 85% at
# prediction time (training=False).
processed_image = inception_preprocessing.preprocess_for_eval(image,
image_size,
image_size, central_fraction=None)
# Networks accept images in batches.
# The first dimension usually represents the batch size.
# In our case the batch size is one.
processed_images = tf.expand_dims(processed_image, 0)
# Load the inception network structure
with slim.arg_scope(inception.inception_resnet_v2_arg_scope()):
logits, _ = inception.inception_resnet_v2(processed_images,
num_classes=NUM_CLASSES,
is_training=False)
# Apply softmax function to the logits (output of the last layer of the network)
probabilities = tf.nn.softmax(logits)
model_path = tf.train.latest_checkpoint(checkpoints_dir)
# Get the function that initializes the network structure (its variables) with
# the trained values contained in the checkpoint
init_fn = slim.assign_from_checkpoint_fn(
model_path,
slim.get_model_variables())
with tf.Session() as sess:
# Now call the initialization function within the session
init_fn(sess)
# Convert variables to constants and make sure the placeholder input_image is included
# in the graph as well as the other neccesary tensors.
constant_graph = convert_variables_to_constants(sess, sess.graph_def, ["input_image", "DecodeJpeg",
"InceptionResnetV2/Logits/Predictions"])
# Define the input and output layer properly
optimized_constant_graph = optimize_for_inference(constant_graph, ["eval_image"],
["InceptionResnetV2/Logits/Predictions"],
tf.string.as_datatype_enum)
# Write the production ready graph to file.
tf.train.write_graph(optimized_constant_graph, '.', OUTPUT_PB_FILENAME, as_text=False)
(The models/slim code must be in your python path to execute this code)
To predict new images with the converted model (now present as a *.pb file) use the code from file minimal_predict.py:
import tensorflow as tf
import urllib2
def create_graph(model_file):
"""Creates a graph from saved GraphDef file and returns a saver."""
# Creates graph from saved graph_def.pb.
with tf.gfile.FastGFile(model_file, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(graph_def, name='')
model_file = "/your/path/to/minimal_graph.proto"
url = ("http://pictureparadise.net/funny-babies/funny-babies02/funny-babies-053.jpg")
# Open specified url and load image as a string
image_string = urllib2.urlopen(url).read()
with tf.Graph().as_default():
with tf.Session() as new_sess:
create_graph(model_file)
softmax = new_sess.graph.get_tensor_by_name("InceptionResnetV2/Logits/Predictions:0")
# Loading the injected placeholder
input_placeholder = new_sess.graph.get_tensor_by_name("input_image:0")
probabilities = new_sess.run(softmax, {input_placeholder: image_string})
print probabilities
To use these scripts simply run python
python convert_checkpoint_to_pb.py
python minimal_predicty.py
while having tensorflow and tensorflow/models/slim in your PYTHONPATH.
In the convert_checkpoint_to_pb.pyprovided by #Maximilian,["eval_image"] in
optimized_constant_graph = optimize_for_inference(constant_graph, ["eval_image"],["InceptionResnetV2/Logits/Predictions"],tf.string.as_datatype_enum)
should be replaced by ["input_image"] else you will get an error stating "The following input nodes were not found: {'eval_image'}\n".