I am using Tensorflow Estimator API but haven't figured out how to use the L-BFGS optimizer available at tf.contrib.opt.ScipyOptimizerInterface.
It seems the estimator API expects some optimizer from the tf.train module but no BFGS implementation is available there. The only one defined in contrib does not follow the same interface.
To be more specific, in the official tutorial to define custom estimators, it's shown how to use the AdagradOptimizer:
optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
However, the API of the ScipyOptimizerInterface is as follows:
optimizer = ScipyOptimizerInterface(loss, options={'maxiter': 100})
with tf.Session() as session:
optimizer.minimize(session)
Taking a full example:
from sklearn import datasets
import numpy as np
def _custom_model_fn(features, labels, mode, feature_columns):
predictions = tf.feature_column.linear_model(features, feature_columns)
predictions = tf.reshape(predictions, [-1])
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {'predictions': predictions}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.mean_squared_error(labels=labels, predictions=predictions,
reduction=tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
# train_op = tf.contrib.opt.ScipyOptimizerInterface(loss, options={'maxiter': 10})
optimizer = tf.train.FtrlOptimizer(learning_rate=1.)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode,
predictions=predictions,
loss=loss,
train_op=train_op)
class MyRegressor(tf.estimator.Estimator):
def __init__(self, feature_columns, model_dir=None, config=None):
def _model_fn(features, labels, mode, config):
return _custom_model_fn(features, labels, mode, feature_columns)
super(MyRegressor, self).__init__(model_fn=_model_fn)
# Load the diabetes dataset
diabetes = datasets.load_diabetes()
diabetes_X = diabetes.data[:, np.newaxis, 2]
diabetes_y = diabetes.target
# Create the custom estimator and train it
feature_columns = [tf.feature_column.numeric_column('x')]
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={'x': np.array(diabetes.data[:, 2])},
y=np.array(diabetes.target),
num_epochs=None,
shuffle=True)
myregressor = MyRegressor(feature_columns)
myregressor.train(train_input_fn, steps=10000)
If I un-comment the line to use the ScipyOptimizer instead, I obviously get an error as follows
TypeError: train_op must be Operation or Tensor, given: <tensorflow.contrib.opt.python.training.external_optimizer.ScipyOptimizerInterface object
Is there an easy way to use the Scipy optimizer?
Thanks in advance.
Related
I have created an Estimator from a TF Slim Resnet V2 checkpoint and tested it to make predictions. The main thing of what I did is basically similar to a normal Estimator together with assign_from_checkpoint_fn:
def model_fn(features, labels, mode, params):
...
slim.assign_from_checkpoint_fn(os.path.join(checkpoint_dir, 'resnet_v2_50.ckpt'), slim.get_model_variables('resnet_v2')
...
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes[:, tf.newaxis],
'probabilities': tf.nn.softmax(logits),
'logits': logits,
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
To export the estimator as a SavedModel, I made a serving_input_fn as follows:
def image_preprocess(image_buffer):
image = tf.image.decode_jpeg(image_buffer, channels=3)
image_preprocessing_fn = preprocessing_factory.get_preprocessing('inception', is_training=False)
image = image_preprocessing_fn(image, FLAGS.image_size, FLAGS.image_size)
return image
def serving_input_fn():
input_ph = tf.placeholder(tf.string, shape=[None], name='image_binary')
image_tensors = image_preprocess(input_ph)
return tf.estimator.export.ServingInputReceiver(image_tensors, input_ph)
In the main function, I use export_saved_model to try to export Estimator to SavedModel format:
def main():
...
classifier = tf.estimator.Estimator(model_fn=model_fn)
classifier.export_saved_model(dir_path, serving_input_fn)
However, when I try to run the codes, it says "Couldn't find trained model at /tmp/tmpn3spty2z". From what I understand, this export_saved_model tries to find a trained Estimator model to export to SavedModel. However, I would like to know if there are any ways I can restore the pretrained checkpoint into an Estimator and export the Estimator to a SavedModel without any further training?
I have solved my problem. To export TF Slim Resnet checkpoint with TF 1.14 to SavedModel, warm start can be used together with export_savedmodel as follows:
config = tf.estimator.RunConfig(save_summary_steps = None, save_checkpoints_secs = None)
warm_start = tf.estimator.WarmStartSettings(checkpoint_dir, checkpoint_name)
classifier = tf.estimator.Estimator(model_fn=model_fn, warm_start_from = warm_start, config = config)
classifier.export_savedmodel(export_dir_base = FLAGS.output_dir, serving_input_receiver_fn = serving_input_fn)
I'm trying to adapt the CNN of the tutorial "Build a Convolutional Neural Network using Estimators" to a dataset of mine, and don't know how to fix this error
...well the input files should be fine, as they are already tested and ok, as I am currently running them on another CNN but much much different (it's working fine but I'm willing to change it, adding some extra features like the "dropout")
The fact is that the error (I use Spyder as IDE) is quite meaningless. I have done some tries to see where the error it is but I'm slightly getting more and more confused, so let's try to ask you guys
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
#----- global variables Start ------
nb_of_neurons=1024
model_learning_rate=0.001
#----- global variables End ------
def run_cnn(mymode, last_date, names, mydata, mylabels, run_id):
def cnn_model_fn(cnndata, mylabels, mode):
input_layer = tf.reshape(cnndata, [-1, 4, 5, 1])
conv = tf.layers.conv2d(
inputs=input_layer,
filters=16,
kernel_size=[2, 3],
padding="same",
activation=tf.nn.relu)
print(conv.shape.dims)
pool = tf.layers.max_pooling2d(inputs=conv, pool_size=[2, 2], strides=2)
pool_dims=pool.shape.as_list()[1]*pool.shape.as_list()[2]*pool.shape.as_list()[3]
pool_flat = tf.reshape(pool, [-1, pool_dims])
dense = tf.layers.dense(inputs=pool_flat, units=nb_of_neurons, activation=tf.nn.relu)
dropout = tf.layers.dropout(
inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
logits = tf.layers.dense(inputs=dropout, units=2)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=mylabels, logits=logits)
print(loss)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=mylabels, predictions=predictions["classes"])
}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
if mymode == 'TRAIN':
mode= tf.estimator.ModeKeys.TRAIN
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
batch_size=100,
num_epochs=None,
shuffle=True)
cnn_classifier.train(
input_fn=train_input_fn,
steps=1,
hooks=[logging_hook])
cnn_classifier.train(input_fn=train_input_fn, steps=1000)
elif mymode == 'PREDICT':
mode= tf.estimator.ModeKeys.PREDICT
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
num_epochs=1,
shuffle=False)
eval_results = cnn_classifier.evaluate(input_fn=eval_input_fn)
else:
print('**** ->*** ???? ***')
This is called as a module from another python script which passes to all the input data, as follows:
mymode: in ['PREDICT', 'TRAIN']
last_date: not relevant
names: not relevant
mydata: np array of shape (3195,20), of values in [0., 1.] (float)
mylabels: np array of shape (3195,), of values in [0, 1] (int)
run_i: not relevant
Finally, the error appears after the train_op (i.e. in the tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)), as follows:
...
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\function_utils.py", line 56, in fn_args
args = tf_inspect.getfullargspec(fn).args
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\tf_inspect.py", line 216, in getfullargspec
if d.decorator_argspec is not None), _getfullargspec(target))
File "C:\Users\Fulviooo\Anaconda3\lib\inspect.py", line 1095, in getfullargspec
raise TypeError('unsupported callable') from ex
TypeError: unsupported callable
I hope that someone can enlighten me about where's the error and how to fix it.
Furthermore, I'd be pleased to receive any other suggests for improvements.
Thanks
In practice, the problem is that this estimator is quite rigid and expects variables with predefined names and formats.
ie setting up the expected names:
train_data=mydata
train_labels=mylabels
and formats (dict):
x={"x": train_data}
then it runs
I'm trying to implement a network for MNIST dataset using custom estimators.
Here is my input function:
def input_train_fn():
train, test = tf.keras.datasets.mnist.load_data()
mnist_x, mnist_y = train
mnist_y = tf.cast(mnist_y, tf.int32)
mnist_x = tf.cast(mnist_x, tf.int32)
features = {'image': mnist_x}
labels = mnist_y
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
return dataset
Here is how I define my model:
def my_model(features, labels, mode, params):
# create net
net = tf.feature_column.input_layer(features, params['feature_columns'])
# create hidden layers
for unit in params['hidden_units']:
net = tf.layers.dense(net, unit, tf.nn.relu)
# create output layer
legits = tf.layers.dense(net, params['n_classes'], activation=None)
# predict (if in predict mode)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# define loss function
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
# evaluation metrics
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
And this is how I call the train function:
feature_columns = [tf.feature_column.numeric_column('image', shape=[28, 28], dtype=tf.int32), ]
classifier = tf.estimator.Estimator(model_fn=my_model,
params={
'feature_columns': feature_columns,
'hidden_units': [10, 10],
'n_classes': 10,
}, model_dir='/model')
classifier.train(input_fn=input_train_fn, steps=10)
As far as I can see i'm doing everything by the book both for estimators and feature_columns but I get the error:
ValueError: Cannot reshape a tensor with 784 elements to shape [28,784] (21952 elements) for 'input_layer/image/Reshape' (op: 'Reshape') with input shapes: [28,28], 2 and with input tensors computed as partial shapes: input1 = [28,784].
Is there anything I'm missing?
thanks in advance and any help appreciated.
First, you need to produce batches. For more detail see https://www.tensorflow.org/guide/datasets
...
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
dataset = dataset.batch(size)
return dataset
Then reshape your image and cast to float. -1 is for batch_size, it will be substituted during training. Cast labels to float is optional depending on the datatype provided.
net = tf.cast(tf.reshape(features, [-1, 28*28]), tf.float32)
labels = tf.cast(labels, tf.int64)
net = tf.layers.dense(net, 10, tf.nn.relu)
legits = tf.layers.dense(net, 10, activation=None)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
classifier = tf.estimator.Estimator(model_fn=my_model)
classifier.train(input_fn=lambda: input_train_fn(), steps=10)
I am trying to feed serialized data to my saved model.
I suspect either the model is not exported correctly, or serialization of the data is not done correctly. Any hints or suggestions would be appreciated.
import tensorflow as tf
import numpy as np
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={'x': np.array([[1,2,3,4,3], [1,3,4,2,4], [10,2,4,1.3,4], [1,3,5.2,9, 0.3]]).astype(np.float32)},
y=np.array([0,0,1,1]).astype(np.float32),
batch_size=2,
shuffle=True
)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={'x': np.array([[1,2,3,1,4], [1,23,4,1,90]]).astype(np.float32)},
y=np.array([0,1]).astype(np.float32),
batch_size=2,
num_epochs=1,
shuffle=False
)
def my_model(features, labels, mode, params):
net = features['x']
net = tf.layers.dense(net, 32, activation=tf.nn.relu)
logits = tf.layers.dense(net, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=logits,
export_outputs=
{'logits':tf.estimator.export.PredictOutput(logits)}
)
loss = tf.reduce_sum(tf.square(labels-logits))
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=0.02)
train_op = optimizer.minimize(loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode=mode, loss=loss)
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=5)
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn)
my_estimator = tf.estimator.Estimator(model_fn=my_model)
tf.estimator.train_and_evaluate(my_estimator, train_spec, eval_spec)
saved_model = my_estimator.export_savedmodel('foo', tf.estimator.export.build_parsing_serving_input_receiver_fn({'x': tf.FixedLenFeature([5],tf.float32)}))
predict_fn = tf.contrib.predictor.from_saved_model(saved_model,)
features=tf.train.Features(feature={'x':tf.train.Feature(float_list=tf.train.FloatList(value=[1,2,3,4,1]))})
ex = tf.train.Example(features=features)
prediction = predict_fn({'examples':ex.SerializeToString()})
Here is the error message:
ValueError: Cannot feed value of shape () for Tensor 'input_example_tensor:0', which has shape '(?,)'
Turned out that the list of tf.train.Examples.SerializeToString() had to be provided.
In the last line, prediction = predict_fn({'examples': ex.SerializeToString()}) is changed to prediction = predict_fn({'examples': [ex.SerializeToString()]}).
I figured this out thanks to this nice tutorial:
http://shzhangji.com/blog/2018/05/14/serve-tensorflow-estimator-with-savedmodel/
I have this snippet, for my model
import pandas as pd
import tensorflow as tf
from tensorflow.contrib import learn
from tensorflow.contrib.learn.python import SKCompat
#Assume my dataset is using X['train'] as input and y['train'] as output
regressor = SKCompat(learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS),model_dir=LOG_DIR))
validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000)
regressor.fit(X['train'], y['train'],
monitors=[validation_monitor],
batch_size=BATCH_SIZE,
steps=TRAINING_STEPS)
#After training this model I want to save it in a folder, so I can use the trained model for implementing in my algorithm to predict the output
#What is the correct format to use here to save my model in a folder called 'saved_model'
regressor.export_savedmodel('/saved_model/')
#I want to import it later in some other code, How can I import it?
#is there any function like import model from file?
How can I save this estimator? I tried finding some examples for tf.contrib.learn.Estimator.export_savedmodel, I did not have a success? Help Appreciated.
The function export_savedmodel requires the argument serving_input_receiver_fn, that is a function without arguments, which defines the input from the model and the predictor. Therefore, you must create your own serving_input_receiver_fn, where the model input type match with the model input in the training script, and the predictor input type match with the predictor input in the testing script.
On the other hand, if you create a custom model, you must define the export_outputs, defined by the function tf.estimator.export.PredictOutput, which input is a dictionary that define the name that has to match with the name of the predictor output in the testing script.
For example:
TRAINING SCRIPT
def serving_input_receiver_fn():
serialized_tf_example = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensors')
receiver_tensors = {"predictor_inputs": serialized_tf_example}
feature_spec = {"words": tf.FixedLenFeature([25],tf.int64)}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
def estimator_spec_for_softmax_classification(logits, labels, mode):
predicted_classes = tf.argmax(logits, 1)
if (mode == tf.estimator.ModeKeys.PREDICT):
export_outputs = {'predict_output': tf.estimator.export.PredictOutput({"pred_output_classes": predicted_classes, 'probabilities': tf.nn.softmax(logits)})}
return tf.estimator.EstimatorSpec(mode=mode, predictions={'class': predicted_classes, 'prob': tf.nn.softmax(logits)}, export_outputs=export_outputs) # IMPORTANT!!!
onehot_labels = tf.one_hot(labels, 31, 1, 0)
loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels, logits=logits)
if (mode == tf.estimator.ModeKeys.TRAIN):
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
eval_metric_ops = {'accuracy': tf.metrics.accuracy(labels=labels, predictions=predicted_classes)}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def model_custom(features, labels, mode):
bow_column = tf.feature_column.categorical_column_with_identity("words", num_buckets=1000)
bow_embedding_column = tf.feature_column.embedding_column(bow_column, dimension=50)
bow = tf.feature_column.input_layer(features, feature_columns=[bow_embedding_column])
logits = tf.layers.dense(bow, 31, activation=None)
return estimator_spec_for_softmax_classification(logits=logits, labels=labels, mode=mode)
def main():
# ...
# preprocess-> features_train_set and labels_train_set
# ...
classifier = tf.estimator.Estimator(model_fn = model_custom)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"words": features_train_set}, y=labels_train_set, batch_size=batch_size_param, num_epochs=None, shuffle=True)
classifier.train(input_fn=train_input_fn, steps=100)
full_model_dir = classifier.export_savedmodel(export_dir_base="C:/models/directory_base", serving_input_receiver_fn=serving_input_receiver_fn)
TESTING SCRIPT
def main():
# ...
# preprocess-> features_test_set
# ...
with tf.Session() as sess:
tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], full_model_dir)
predictor = tf.contrib.predictor.from_saved_model(full_model_dir)
model_input = tf.train.Example(features=tf.train.Features( feature={"words": tf.train.Feature(int64_list=tf.train.Int64List(value=features_test_set)) }))
model_input = model_input.SerializeToString()
output_dict = predictor({"predictor_inputs":[model_input]})
y_predicted = output_dict["pred_output_classes"][0]
(Code tested in Python 3.6.3, Tensorflow 1.4.0)