I have a model function which accepts Features, targets and mode but when I add tf.keras layers I'm currently getting Exception pred must be a Tensor, a Variable, or a Python bool.
But, When I run the same code with out using tf.keras but directly from keras(i.e. from keras.layers), It's working.
Code :
def model_fn(features, labels, mode):
if mode == tf.estimator.ModeKeys.TRAIN:
tf.keras.backend.set_learning_phase(1)
else:
tf.keras.backend.set_learning_phase(0)
input_feature = features['x']
table = lookup.index_table_from_file(vocabulary_file='vocab.txt', num_oov_buckets=1, default_value=-1)
text = tf.squeeze(input_feature, [1])
words = tf.string_split(text)
densewords = tf.sparse_tensor_to_dense(words, default_value=PADWORD)
numbers = table.lookup(densewords)
padding = tf.constant([[0, 0], [0, MAX_FEATURES]])
padded = tf.pad(numbers, padding)
sliced = tf.slice(padded, [0, 0], [-1, MAX_FEATURES])
print('words_sliced={}'.format(words))
#embeds = tf.keras.layers.Embedding(MAX_FEATURES, 50, input_length=MAX_FEATURES)(sliced)
embeds = tf.contrib.layers.embed_sequence(sliced, vocab_size=MAX_FEATURES, embed_dim=50)
print('words_embed={}'.format(embeds))
f1 = tf.keras.layers.Dropout(0.2)(embeds)
f1 = tf.keras.layers.Conv1D(filters, kernel_size, padding='valid', activation='relu', strides=1)(f1)
f1 = tf.keras.layers.GlobalAveragePooling1D()(f1)
# f1 = layers.BatchNormalization()(f1)
f1 = tf.keras.layers.Dense(hidden_dims)(f1)
f1 = tf.keras.layers.Dropout(0.5)(f1)
f1 = tf.keras.layers.Activation('relu')(f1)
logits = tf.keras.layers.Dense(11)(f1)
predictions_dict = {
'class': tf.argmax(logits, 1),
'prob': tf.nn.softmax(logits)
}
prediction_output = tf.estimator.export.PredictOutput({"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits,
name="softmax_tensor")})
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions_dict, export_outputs={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_output
})
# one_hot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=11)
loss = tf.losses.sparse_softmax_cross_entropy(labels, logits=logits)
if mode == tf.contrib.learn.ModeKeys.TRAIN:
train_op = tf.contrib.layers.optimize_loss(loss, tf.contrib.framework.get_global_step(), optimizer='Adam',
learning_rate=0.001)
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metrics_ops = {
'accuracy': tf.metrics.accuracy(labels=labels, predictions=predictions_dict['class'])
}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metrics_ops)
When I execute the above script I'm getting an exception
TypeError: pred must be a Tensor, a Variable, or a Python bool.
But also when I used keras(from keras) directly without tf.keras it's working. What is going wrong here ?
Code :
if mode == tf.estimator.ModeKeys.TRAIN:
tf.keras.backend.set_learning_phase(True)
else:
tf.keras.backend.set_learning_phase(False)
Setting learning_phase = True or False is solving the problem.
Related
I'm trying to adapt the CNN of the tutorial "Build a Convolutional Neural Network using Estimators" to a dataset of mine, and don't know how to fix this error
...well the input files should be fine, as they are already tested and ok, as I am currently running them on another CNN but much much different (it's working fine but I'm willing to change it, adding some extra features like the "dropout")
The fact is that the error (I use Spyder as IDE) is quite meaningless. I have done some tries to see where the error it is but I'm slightly getting more and more confused, so let's try to ask you guys
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
#----- global variables Start ------
nb_of_neurons=1024
model_learning_rate=0.001
#----- global variables End ------
def run_cnn(mymode, last_date, names, mydata, mylabels, run_id):
def cnn_model_fn(cnndata, mylabels, mode):
input_layer = tf.reshape(cnndata, [-1, 4, 5, 1])
conv = tf.layers.conv2d(
inputs=input_layer,
filters=16,
kernel_size=[2, 3],
padding="same",
activation=tf.nn.relu)
print(conv.shape.dims)
pool = tf.layers.max_pooling2d(inputs=conv, pool_size=[2, 2], strides=2)
pool_dims=pool.shape.as_list()[1]*pool.shape.as_list()[2]*pool.shape.as_list()[3]
pool_flat = tf.reshape(pool, [-1, pool_dims])
dense = tf.layers.dense(inputs=pool_flat, units=nb_of_neurons, activation=tf.nn.relu)
dropout = tf.layers.dropout(
inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
logits = tf.layers.dense(inputs=dropout, units=2)
predictions = {
"classes": tf.argmax(input=logits, axis=1),
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=mylabels, logits=logits)
print(loss)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=mylabels, predictions=predictions["classes"])
}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
if mymode == 'TRAIN':
mode= tf.estimator.ModeKeys.TRAIN
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
batch_size=100,
num_epochs=None,
shuffle=True)
cnn_classifier.train(
input_fn=train_input_fn,
steps=1,
hooks=[logging_hook])
cnn_classifier.train(input_fn=train_input_fn, steps=1000)
elif mymode == 'PREDICT':
mode= tf.estimator.ModeKeys.PREDICT
cnn_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn(mydata, mylabels, mode), model_dir="/sess")
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x=mydata,
y=mylabels,
num_epochs=1,
shuffle=False)
eval_results = cnn_classifier.evaluate(input_fn=eval_input_fn)
else:
print('**** ->*** ???? ***')
This is called as a module from another python script which passes to all the input data, as follows:
mymode: in ['PREDICT', 'TRAIN']
last_date: not relevant
names: not relevant
mydata: np array of shape (3195,20), of values in [0., 1.] (float)
mylabels: np array of shape (3195,), of values in [0, 1] (int)
run_i: not relevant
Finally, the error appears after the train_op (i.e. in the tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)), as follows:
...
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\function_utils.py", line 56, in fn_args
args = tf_inspect.getfullargspec(fn).args
File "C:\Users\Fulviooo\Anaconda3\lib\site-packages\tensorflow\python\util\tf_inspect.py", line 216, in getfullargspec
if d.decorator_argspec is not None), _getfullargspec(target))
File "C:\Users\Fulviooo\Anaconda3\lib\inspect.py", line 1095, in getfullargspec
raise TypeError('unsupported callable') from ex
TypeError: unsupported callable
I hope that someone can enlighten me about where's the error and how to fix it.
Furthermore, I'd be pleased to receive any other suggests for improvements.
Thanks
In practice, the problem is that this estimator is quite rigid and expects variables with predefined names and formats.
ie setting up the expected names:
train_data=mydata
train_labels=mylabels
and formats (dict):
x={"x": train_data}
then it runs
I'm trying to implement a network for MNIST dataset using custom estimators.
Here is my input function:
def input_train_fn():
train, test = tf.keras.datasets.mnist.load_data()
mnist_x, mnist_y = train
mnist_y = tf.cast(mnist_y, tf.int32)
mnist_x = tf.cast(mnist_x, tf.int32)
features = {'image': mnist_x}
labels = mnist_y
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
return dataset
Here is how I define my model:
def my_model(features, labels, mode, params):
# create net
net = tf.feature_column.input_layer(features, params['feature_columns'])
# create hidden layers
for unit in params['hidden_units']:
net = tf.layers.dense(net, unit, tf.nn.relu)
# create output layer
legits = tf.layers.dense(net, params['n_classes'], activation=None)
# predict (if in predict mode)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# define loss function
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
# evaluation metrics
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
tf.summary.scalar('accuracy', accuracy[1])
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
And this is how I call the train function:
feature_columns = [tf.feature_column.numeric_column('image', shape=[28, 28], dtype=tf.int32), ]
classifier = tf.estimator.Estimator(model_fn=my_model,
params={
'feature_columns': feature_columns,
'hidden_units': [10, 10],
'n_classes': 10,
}, model_dir='/model')
classifier.train(input_fn=input_train_fn, steps=10)
As far as I can see i'm doing everything by the book both for estimators and feature_columns but I get the error:
ValueError: Cannot reshape a tensor with 784 elements to shape [28,784] (21952 elements) for 'input_layer/image/Reshape' (op: 'Reshape') with input shapes: [28,28], 2 and with input tensors computed as partial shapes: input1 = [28,784].
Is there anything I'm missing?
thanks in advance and any help appreciated.
First, you need to produce batches. For more detail see https://www.tensorflow.org/guide/datasets
...
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
dataset = dataset.batch(size)
return dataset
Then reshape your image and cast to float. -1 is for batch_size, it will be substituted during training. Cast labels to float is optional depending on the datatype provided.
net = tf.cast(tf.reshape(features, [-1, 28*28]), tf.float32)
labels = tf.cast(labels, tf.int64)
net = tf.layers.dense(net, 10, tf.nn.relu)
legits = tf.layers.dense(net, 10, activation=None)
predicted_classes = tf.arg_max(legits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes,
'probabilities': tf.nn.softmax(legits),
'logits': legits
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=legits)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode, loss=loss)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
classifier = tf.estimator.Estimator(model_fn=my_model)
classifier.train(input_fn=lambda: input_train_fn(), steps=10)
As the title the error I'm getting is:
ValueError: Expected scalar shape for SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits:0, saw shape: (52692,).
The shapes I'm passing:
logits - [52692, 4] - rank 2
labels - [52692] - rank 1
There are 4 classes
Here is how I pass my input to the model basically a tuple of (features, labels)
def _input_fn(training_dir, training_filename):
def getFeatureLabelOH(file):
features, labels = csvGetColumns(file)
count = len(features)
# converting features and labels to integers
featuresVec, labelsVec = convCharToVec(features, alphabetDict, maxFeatureLen), \
[conventions[label] for label in labels]
featuresVec = tf.convert_to_tensor(featuresVec, dtype=tf.int32)
labelsVec = tf.convert_to_tensor(labelsVec, dtype=tf.int32)
labelsVec = tf.reshape(labelsVec, [-1])
return {"featuresVec": featuresVec, "labelsVec": labelsVec, "count": count}
data = getFeatureLabelOH(os.path.join(training_dir,
return (data["featuresVec"], data["labelsVec"])
And my actual model
def model_fn(features, labels, mode, params):
net = keras.layers.Embedding(alphabetLen + 1, 8, input_length=maxFeatureLen)(features)
net = keras.layers.LSTM(12)(net)
logits = keras.layers.Dense(len(conventions), activation=tf.nn.softmax)(net) #output
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)
train_op = tf.contrib.layers.optimize_loss(
loss=loss,
global_step=tf.train.get_global_step(),
learning_rate=0.001,
optimizer="AdamOptimizer")
eval_metric_ops = {}
return tf.estimator.EstimatorSpec(
mode=mode,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops)
Following the instructions on tf custom estimator
I have created a cnn estimator and tried to train it. While training, i initialized tensorboard and was hoping to see some visualizations about training steps. However, tensorboard only showed the graph of my custom estimator but none of the scalar values i have defined.
Here's roughly what I have in code
def model_fn(features, labels, mode, params=None):
tf.logging.set_verbosity(tf.logging.INFO)
n_classes = params['n_classes']
base_learning_rate = params['learning_rate']
decay_rate = params['decay_rate']
embedding_dim = params['embedding_dim']
x = VGG_block1(features, (3, 3), 64, name='block1_1')
x = VGG_block1(x, (3, 3), 128, name='block1_2')
x = VGG_block1(x, (3, 3), 256, name='block1_3', regularizer=tf.contrib.layers.l1_regularizer(.1))
x = VGG_block2(x, (3, 3), 512, name='block2_4')
x = VGG_block2(x, (3, 3), 1024, name='block2_5')
x = conv2d(x, 512, (5, 5), padding='valid', normalizer_fn=batch_norm, activation_fn=tf.nn.leaky_relu,
weights_initializer=he_uniform())
x = flatten(x)
embedding = fully_connected(x, embedding_dim)
logits = fully_connected(embedding, n_classes)
# make predictions
predictions = {
'classes': tf.argmax(logits, axis=1, name='classes'),
'probabilities': tf.nn.softmax(logits, name='softmax'),
'embeddings':embedding
}
# if we are in prediction mode
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# otherwise define losses for training
c_loss, center = center_loss(embedding, labels, .9, n_classes)
xent_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))
total_loss = xent_loss + 0.5 * c_loss
# evaluation methods
accuracy, update_op = tf.metrics.accuracy(labels=labels, predictions=predictions['classes'], name='accuracy')
batch_acc = tf.reduce_mean(tf.cast(tf.equal(tf.cast(labels, tf.int64), predictions['classes']), tf.float32))
tf.summary.scalar('batch_acc', batch_acc)
tf.summary.scalar('streaming_acc', update_op)
tf.summary.scalar('total_loss', total_loss)
tf.summary.scalar('center_loss', c_loss)
tf.summary.scalar('xent_loss', xent_loss)
# training mode
if mode == tf.estimator.ModeKeys.TRAIN:
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
global_step = tf.Variable(0, trainable=False)
global_step_op = tf.assign(global_step, global_step + 1)
learning_rate = tf.train.exponential_decay(base_learning_rate, global_step, 8000, decay_rate, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate)
with tf.control_dependencies(update_ops+[global_step_op]):
objective = optimizer.minimize(total_loss)
return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=objective)
eval_metric_ops = {
'accuracy': (accuracy, update_op)
}
return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, eval_metric_ops=eval_metric_ops)
X_train, X_test, y_train, y_test = load_data()
epochs = 10
batch_size = 64
n_classes = len(classes)
model_params = {'n_classes':n_classes,
'learning_rate':0.0001,
'decay_rate':0.5,
'embedding_dim':128}
model_dir = 'output'
face_classifier = tf.estimator.Estimator(model_fn=model_fn, params=model_params, model_dir=model_dir)
My Tensorflow version is 1.12.0
Edit
Forgot to mention I was using eager execution for this exercise, for unknown reasons that was the cause of this bug
as was mentioned in the edit, disabling eager execution solved the problem
I'm trying to build a model that uses sampled_softmax_loss and I can't seem to get the input tensors shaped properly for the function. Here is an example that as best I can tell should match the documentation, but throws this exception:
ValueError: Shape must be rank 2 but is rank 1 for
'sampled_softmax_loss/LogUniformCandidateSampler' (op:
'LogUniformCandidateSampler') with input shapes: [?].
Code:
import tensorflow as tf
import numpy as np
f1 = np.random.randint(low = 0, high = 4,size = 100)
labels = np.random.randint(low = 0, high = 5,size = 100)
f1_t = tf.feature_column.categorical_column_with_vocabulary_list('f1', vocabulary_list = [0,1,2,3])
base_columns = [f1_t]
feat_dict = {'f1' : f1}
def my_model_fn(
features,
labels,
mode,
params):
logits = tf.feature_column.linear_model(features, base_columns,units = params["n_classes"])
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'probabilities': tf.nn.softmax(logits),
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
weights = [v for v in tf.global_variables() if v.name == 'linear_model/f1/weights:0'][0]
biases = [v for v in tf.global_variables() if v.name == 'linear_model/bias_weights:0'][0]
if mode == "train":
loss = tf.nn.sampled_softmax_loss(
weights=tf.transpose(weights),
biases=biases,
labels=labels,
inputs=logits,
num_classes = 5,
num_sampled= 11,
num_true=1,
partition_strategy="div")
elif mode == "eval":
None
# implement later
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode,
loss=loss,
)
optimizer = tf.train.FtrlOptimizer(learning_rate=.1,l2_regularization_strength=0.1)#AdagradOptimizer(0.001)
train_op = optimizer.minimize(
loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(
mode,
loss=loss,
train_op=train_op)
classifier = tf.estimator.Estimator(
model_fn = my_model_fn,
params = {
"feature_columns" : base_columns,
"n_classes" : 5
})
classifier.train(
input_fn = tf.estimator.inputs.numpy_input_fn(feat_dict,
labels,
batch_size = 3,
num_epochs=2,
shuffle=True))
If anyone could give me some pointers, I'll owe you virtual beer eternally.
Labels should be in the shape [batch, one_hot], e.g., [100, 6]