Develop Reference

tensorflow:Error reported to Coordinator: Items of feature_columns must be a _FeatureColumn. Given (type <class 'collections.NumericColumn'>)

I tried to use tf estimator to build logistic regression model. I used iris dataset and it ran successfully in my computer. However, when I tried to apply this model in cluster(using train_and_evaluate instead of classfier.train), I came across this problem.
python version:3.6.8
tensorflow version:1.13.1
Here is the code running locally:
iris dataset only contains numeric data. So feature_columns is a list of NumericColumn.
FUTURES = ['SepalLength', 'SepalWidth','PetalLength', 'PetalWidth', 'Species']
feature_columns = []
for key in FUTURES:
feature_columns.append(tf.feature_column.numeric_column(key=key))
define estimator. pass feature_columns into params
classifier = tf.estimator.Estimator(
model_fn=my_model_fn,
model_dir=models_path,
params={
'feature_columns': feature_columns,
'n_classes': 3,
})
define model_fn.
def my_model_fn(features,labels,mode,params):
net = tf.feature_column.input_layer(features, params['feature_columns'])
logits = tf.layers.dense(net, params['n_classes'], activation=None)
predicted_classes = tf.argmax(logits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {'logits': logits}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss,global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
This code works well and generates some result.
-------------------------------------------------------------
Then I want to train it in cluster. my_model_fn is the same as previous one and self._feature_numeric_col is still a list of NumericColumn.
class LogisticReg():
def __init__(self):
self._feature_col = x.columns.tolist()
self._feature_numeric_col = []
for key in self._feature_col:
self._feature_numeric_col.append(tf.feature_column.numeric_column(key=key))
self.estimator = tf.estimator.Estimator(model_fn=self.my_model_fn,
model_dir=self.model_path,
config=self.config,
params={'feature_columns':self._feature_numeric_col})
def my_model_fn(self, features, labels, mode, params):
net = tf.feature_column.input_layer(features, params['feature_columns'])
logits = tf.layers.dense(net, self.n_class, activation=None)
predicted_classes = tf.argmax(logits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {'logits': logits}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step()) ！
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
accuracy = tf.metrics.accuracy(labels=labels,predictions=predicted_classes)
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
use train_and_evaluate function instead of train/eval/predict
# input_fn
def input_fn(self, X, y, mode, batch_size):
y = y.astype(np.int32)
X = X.astype(np.float32)
dataset = tf.data.Dataset.from_tensor_slices((dict(X), y)) # x,y:pandas
if mode == 'train':
dataset = dataset.shuffle(500)
dataset = dataset.repeat()
dataset = dataset.batch(batch_size)
return dataset
# train_spec
train_spec = tf.estimator.TrainSpec(input_fn=lambda: self.input_fn(x_train,y_train,'train',batch_size),
max_steps=n_epochs)
# eval_spec
eval_spec = tf.estimator.EvalSpec(input_fn=lambda: self.input_fn(x_valid, y_valid, 'valid', batch_size),
start_delay_secs=30, throttle_secs=30, steps=None)
tf.estimator.train_and_evaluate(self.estimator, train_spec, eval_spec)
I expect the cluster version can generate similar output as the local one. However, I get this error.
Traceback (most recent call last):
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow/python/training/coordinator.py", line 297, in stop_on_exception
yield
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow/python/distribute/mirrored_strategy.py", line 852, in run
self.main_result = self.main_fn(*self.main_args, **self.main_kwargs)
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1112, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/mnt/glusterfs/model-center/train/classify.py", line 51, in my_model_fn
net = tf.feature_column.input_layer(features, params['feature_columns'])
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow/python/feature_column/feature_column.py", line 302, in input_layer
cols_to_output_tensors=cols_to_output_tensors)
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow/python/feature_column/feature_column.py", line 181, in _internal_input_layer
feature_columns = _normalize_feature_columns(feature_columns)
File "/usr/local/bin/python3/lib/python3.6/site-packages/tensorflow/python/feature_column/feature_column.py", line 2263, in _normalize_feature_columns
'Given (type {}): {}.'.format(type(column), column))
ValueError: Items of feature_columns must be a _FeatureColumn. Given (type <class 'collections.NumericColumn'>): NumericColumn(key='sepal_length', shape=(1,), default_value=None, dtype=tf.float32, normalizer_fn=None).

Tensorflow error : unsupported callable - (from ex ???)

I'm trying to adapt the CNN of the tutorial "Build a Convolutional Neural Network using Estimators" to a dataset of mine, and don't know how to fix this error
...well the input files should be fine, as they are already tested and ok, as I am currently running them on another CNN but much much different (it's working fine but I'm willing to change it, adding some extra features like the "dropout")
The fact is that the error (I use Spyder as IDE) is quite meaningless. I have done some tries to see where the error it is but I'm slightly getting more and more confused, so let's try to ask you guys
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
#----- global variables Start ------
nb_of_neurons=1024
model_learning_rate=0.001
#----- global variables End ------
def run_cnn(mymode, last_date, names, mydata, mylabels, run_id):
def cnn_model_fn(cnndata, mylabels, mode):
input_layer = tf.reshape(cnndata, [-1, 4, 5, 1])
conv = tf.layers.conv2d(
inputs=input_layer,
filters=16,
kernel_size=[2, 3],
padding="same",
activation=tf.nn.relu)
print(conv.shape.dims)
pool = tf.layers.max_pooling2d(inputs=conv, pool_size=[2, 2], strides=2)
pool_dims=pool.shape.as_list()[1]*pool.shape.as_list()[2]*pool.shape.as_list()[3]
pool_flat = tf.reshape(pool, [-1, pool_dims])
dense = tf.layers.dense(inputs=pool_flat, units=nb_of_neurons, activation=tf.nn.relu)
dropout = tf.layers.dropout(
inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
logits = tf.layers.dense(inputs=dropout, units=2)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=mylabels, logits=logits)
print(loss)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=mylabels, predictions=predictions["classes"])
}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
if mymode == 'TRAIN':
mode= tf.estimator.ModeKeys.TRAIN
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
batch_size=100,
num_epochs=None,
shuffle=True)
cnn_classifier.train(
input_fn=train_input_fn,
steps=1,
hooks=[logging_hook])
cnn_classifier.train(input_fn=train_input_fn, steps=1000)
elif mymode == 'PREDICT':
mode= tf.estimator.ModeKeys.PREDICT
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
num_epochs=1,
shuffle=False)
eval_results = cnn_classifier.evaluate(input_fn=eval_input_fn)
else:
print('**** ->*** ???? ***')
This is called as a module from another python script which passes to all the input data, as follows:
mymode: in ['PREDICT', 'TRAIN']
last_date: not relevant
names: not relevant
mydata: np array of shape (3195,20), of values in [0., 1.] (float)
mylabels: np array of shape (3195,), of values in [0, 1] (int)
run_i: not relevant
Finally, the error appears after the train_op (i.e. in the tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)), as follows:
...
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\function_utils.py", line 56, in fn_args
args = tf_inspect.getfullargspec(fn).args
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\tf_inspect.py", line 216, in getfullargspec
if d.decorator_argspec is not None), _getfullargspec(target))
File "C:\Users\Fulviooo\Anaconda3\lib\inspect.py", line 1095, in getfullargspec
raise TypeError('unsupported callable') from ex
TypeError: unsupported callable
I hope that someone can enlighten me about where's the error and how to fix it.
Furthermore, I'd be pleased to receive any other suggests for improvements.
Thanks

In practice, the problem is that this estimator is quite rigid and expects variables with predefined names and formats.
ie setting up the expected names:
train_data=mydata
train_labels=mylabels
and formats (dict):
x={"x": train_data}
then it runs

tensorflow feature_column tries to reshape features

I'm trying to implement a network for MNIST dataset using custom estimators.
Here is my input function:
def input_train_fn():
train, test = tf.keras.datasets.mnist.load_data()
mnist_x, mnist_y = train
mnist_y = tf.cast(mnist_y, tf.int32)
mnist_x = tf.cast(mnist_x, tf.int32)
features = {'image': mnist_x}
labels = mnist_y
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
return dataset
Here is how I define my model:
def my_model(features, labels, mode, params):
# create net
net = tf.feature_column.input_layer(features, params['feature_columns'])
# create hidden layers
for unit in params['hidden_units']:
net = tf.layers.dense(net, unit, tf.nn.relu)
# create output layer
legits = tf.layers.dense(net, params['n_classes'], activation=None)
# predict (if in predict mode)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# define loss function
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
# evaluation metrics
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
And this is how I call the train function:
feature_columns = [tf.feature_column.numeric_column('image', shape=[28, 28], dtype=tf.int32), ]
classifier = tf.estimator.Estimator(model_fn=my_model,
params={
'feature_columns': feature_columns,
'hidden_units': [10, 10],
'n_classes': 10,
}, model_dir='/model')
classifier.train(input_fn=input_train_fn, steps=10)
As far as I can see i'm doing everything by the book both for estimators and feature_columns but I get the error:
ValueError: Cannot reshape a tensor with 784 elements to shape [28,784] (21952 elements) for 'input_layer/image/Reshape' (op: 'Reshape') with input shapes: [28,28], 2 and with input tensors computed as partial shapes: input1 = [28,784].
Is there anything I'm missing?
thanks in advance and any help appreciated.

First, you need to produce batches. For more detail see https://www.tensorflow.org/guide/datasets
...
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
dataset = dataset.batch(size)
return dataset
Then reshape your image and cast to float. -1 is for batch_size, it will be substituted during training. Cast labels to float is optional depending on the datatype provided.
net = tf.cast(tf.reshape(features, [-1, 28*28]), tf.float32)
labels = tf.cast(labels, tf.int64)
net = tf.layers.dense(net, 10, tf.nn.relu)
legits = tf.layers.dense(net, 10, activation=None)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
classifier = tf.estimator.Estimator(model_fn=my_model)
classifier.train(input_fn=lambda: input_train_fn(), steps=10)

Using L-BFGS optimizer with Tensorflow estimator API

I am using Tensorflow Estimator API but haven't figured out how to use the L-BFGS optimizer available at tf.contrib.opt.ScipyOptimizerInterface.
It seems the estimator API expects some optimizer from the tf.train module but no BFGS implementation is available there. The only one defined in contrib does not follow the same interface.
To be more specific, in the official tutorial to define custom estimators, it's shown how to use the AdagradOptimizer:
optimizer = tf.train.AdagradOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
However, the API of the ScipyOptimizerInterface is as follows:
optimizer = ScipyOptimizerInterface(loss, options={'maxiter': 100})
with tf.Session() as session:
optimizer.minimize(session)
Taking a full example:
from sklearn import datasets
import numpy as np
def _custom_model_fn(features, labels, mode, feature_columns):
predictions = tf.feature_column.linear_model(features, feature_columns)
predictions = tf.reshape(predictions, [-1])
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {'predictions': predictions}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.mean_squared_error(labels=labels, predictions=predictions,
reduction=tf.losses.Reduction.SUM_BY_NONZERO_WEIGHTS)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
# train_op = tf.contrib.opt.ScipyOptimizerInterface(loss, options={'maxiter': 10})
optimizer = tf.train.FtrlOptimizer(learning_rate=1.)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode,
predictions=predictions,
loss=loss,
train_op=train_op)
class MyRegressor(tf.estimator.Estimator):
def __init__(self, feature_columns, model_dir=None, config=None):
def _model_fn(features, labels, mode, config):
return _custom_model_fn(features, labels, mode, feature_columns)
super(MyRegressor, self).__init__(model_fn=_model_fn)
# Load the diabetes dataset
diabetes = datasets.load_diabetes()
diabetes_X = diabetes.data[:, np.newaxis, 2]
diabetes_y = diabetes.target
# Create the custom estimator and train it
feature_columns = [tf.feature_column.numeric_column('x')]
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={'x': np.array(diabetes.data[:, 2])},
y=np.array(diabetes.target),
num_epochs=None,
shuffle=True)
myregressor = MyRegressor(feature_columns)
myregressor.train(train_input_fn, steps=10000)
If I un-comment the line to use the ScipyOptimizer instead, I obviously get an error as follows
TypeError: train_op must be Operation or Tensor, given: <tensorflow.contrib.opt.python.training.external_optimizer.ScipyOptimizerInterface object
Is there an easy way to use the Scipy optimizer?
Thanks in advance.

How to save a trained model (Estimator) and Load it back to test it with data in Tensorflow?

I have this snippet, for my model
import pandas as pd
import tensorflow as tf
from tensorflow.contrib import learn
from tensorflow.contrib.learn.python import SKCompat
#Assume my dataset is using X['train'] as input and y['train'] as output
regressor = SKCompat(learn.Estimator(model_fn=lstm_model(TIMESTEPS, RNN_LAYERS, DENSE_LAYERS),model_dir=LOG_DIR))
validation_monitor = learn.monitors.ValidationMonitor(X['val'], y['val'], every_n_steps=PRINT_STEPS, early_stopping_rounds=1000)
regressor.fit(X['train'], y['train'],
monitors=[validation_monitor],
batch_size=BATCH_SIZE,
steps=TRAINING_STEPS)
#After training this model I want to save it in a folder, so I can use the trained model for implementing in my algorithm to predict the output
#What is the correct format to use here to save my model in a folder called 'saved_model'
regressor.export_savedmodel('/saved_model/')
#I want to import it later in some other code, How can I import it?
#is there any function like import model from file?
How can I save this estimator? I tried finding some examples for tf.contrib.learn.Estimator.export_savedmodel, I did not have a success? Help Appreciated.

The function export_savedmodel requires the argument serving_input_receiver_fn, that is a function without arguments, which defines the input from the model and the predictor. Therefore, you must create your own serving_input_receiver_fn, where the model input type match with the model input in the training script, and the predictor input type match with the predictor input in the testing script.
On the other hand, if you create a custom model, you must define the export_outputs, defined by the function tf.estimator.export.PredictOutput, which input is a dictionary that define the name that has to match with the name of the predictor output in the testing script.
For example:
TRAINING SCRIPT
def serving_input_receiver_fn():
serialized_tf_example = tf.placeholder(dtype=tf.string, shape=[None], name='input_tensors')
receiver_tensors = {"predictor_inputs": serialized_tf_example}
feature_spec = {"words": tf.FixedLenFeature([25],tf.int64)}
features = tf.parse_example(serialized_tf_example, feature_spec)
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
def estimator_spec_for_softmax_classification(logits, labels, mode):
predicted_classes = tf.argmax(logits, 1)
if (mode == tf.estimator.ModeKeys.PREDICT):
export_outputs = {'predict_output': tf.estimator.export.PredictOutput({"pred_output_classes": predicted_classes, 'probabilities': tf.nn.softmax(logits)})}
return tf.estimator.EstimatorSpec(mode=mode, predictions={'class': predicted_classes, 'prob': tf.nn.softmax(logits)}, export_outputs=export_outputs) # IMPORTANT!!!
onehot_labels = tf.one_hot(labels, 31, 1, 0)
loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels, logits=logits)
if (mode == tf.estimator.ModeKeys.TRAIN):
optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
eval_metric_ops = {'accuracy': tf.metrics.accuracy(labels=labels, predictions=predicted_classes)}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def model_custom(features, labels, mode):
bow_column = tf.feature_column.categorical_column_with_identity("words", num_buckets=1000)
bow_embedding_column = tf.feature_column.embedding_column(bow_column, dimension=50)
bow = tf.feature_column.input_layer(features, feature_columns=[bow_embedding_column])
logits = tf.layers.dense(bow, 31, activation=None)
return estimator_spec_for_softmax_classification(logits=logits, labels=labels, mode=mode)
def main():
# ...
# preprocess-> features_train_set and labels_train_set
# ...
classifier = tf.estimator.Estimator(model_fn = model_custom)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"words": features_train_set}, y=labels_train_set, batch_size=batch_size_param, num_epochs=None, shuffle=True)
classifier.train(input_fn=train_input_fn, steps=100)
full_model_dir = classifier.export_savedmodel(export_dir_base="C:/models/directory_base", serving_input_receiver_fn=serving_input_receiver_fn)
TESTING SCRIPT
def main():
# ...
# preprocess-> features_test_set
# ...
with tf.Session() as sess:
tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], full_model_dir)
predictor = tf.contrib.predictor.from_saved_model(full_model_dir)
model_input = tf.train.Example(features=tf.train.Features( feature={"words": tf.train.Feature(int64_list=tf.train.Int64List(value=features_test_set)) }))
model_input = model_input.SerializeToString()
output_dict = predictor({"predictor_inputs":[model_input]})
y_predicted = output_dict["pred_output_classes"][0]
(Code tested in Python 3.6.3, Tensorflow 1.4.0)