Develop Reference

softmax_cross_entropy_with_logits nan

I have extracted CNN features from a pretrain vgg19 with size 4096. Then I am using a shallower architecture to train a classifier with softmax and center losses. Unfortunately, the softmax loss function returns nan. There is detailed discussion available here, however I am not able to resolve the problem with clip because labels and logits are in two different data format (int64, float32). Furthermore, I also changed the learning rate but still got the same error.
Can some please let me know, how to resolve this situation.
from __future__ import division
from __future__ import print_function
import csv
import numpy as np
import tensorflow as tf
from retrieval_model import setup_train_model
FLAGS = None
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
def get_name(read_file):
feat_lst = []
identifier_lst = []
with open(read_file, 'r') as csvfile:
read_file = csv.reader(csvfile, delimiter=',')
for row in read_file:
feat = row[:-1]
s_feat = [float(i) for i in feat]
identifier = row[-1]
feat_lst.append(s_feat)
identifier_lst.append(identifier)
return feat_lst, identifier_lst
def get_batch(batch_index, batch_size, labels, f_lst):
start_ind = batch_index * batch_size
end_ind = start_ind + batch_size
return f_lst[start_ind:end_ind], labels[start_ind:end_ind]
def creat_dict(orig_labels):
dict = {}
count = 0
for x in orig_labels:
n_label = dict.get(x, None)
if n_label is None:
dict[x] = count
count += 1
return dict
def main(_):
save_dir = 'model/one-branch-ckpt'
train_file = 'gtrain.csv'
img_feat, img_labels = get_name(train_file)
map_dict = creat_dict(img_labels)
img_labels = [map_dict.get(x) for x in img_labels]
im_feat_dim = 4096
batch_size = 50
max_num_epoch = 10
steps_per_epoch = len(img_feat) // batch_size
num_steps = steps_per_epoch * max_num_epoch
# Setup placeholders for input variables.
im_feat_plh = tf.placeholder(tf.float32, shape=[batch_size, im_feat_dim])
label_plh = tf.placeholder(tf.int64, shape=(batch_size), name='labels')
train_phase_plh = tf.placeholder(tf.bool)
# Setup training operation.
t_l = setup_train_model(im_feat_plh, train_phase_plh, label_plh, classes)
# Setup optimizer.
global_step = tf.Variable(0, trainable=False)
init_learning_rate = 0.0001
learning_rate = tf.train.exponential_decay(init_learning_rate, global_step,
steps_per_epoch, 0.794, staircase=True)
optim = tf.train.AdamOptimizer(init_learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = optim.minimize(t_l, global_step=global_step)
# Setup model saver.
saver = tf.train.Saver(save_relative_paths=True,max_to_keep=1)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(num_steps):
im_feats, labels = get_batch(
i % steps_per_epoch, batch_size, img_labels, img_feat)
feed_dict = {
im_feat_plh: im_feats,
label_plh: labels,
train_phase_plh: True,
}
[_, loss_val] = sess.run([train_step, t_l], feed_dict=feed_dict)
if i % 100 == 0:
print('Epoch: %d Step: %d Loss: %f' % (i // steps_per_epoch, i, loss_val))
if i % steps_per_epoch == 0 and i > 0:
print('Saving checkpoint at step %d' % i)
saver.save(sess, save_dir, global_step=global_step)
if __name__ == '__main__':
np.random.seed(0)
tf.set_random_seed(0)
tf.app.run(main=main)
**************************retrieval_model********************************
def setup_train_model(im_feats, train_phase, im_labels, nrof_classes):
alfa = 0.9
# nrof_classes = 28783
i_embed = embedding_model(im_feats, train_phase, im_labels)
c_l = embedding_loss(i_embed, im_labels, alfa, nrof_classes)
loss = softmax_loss(i_embed, im_labels)
total_loss = loss + c_l
return total_loss
def add_fc(inputs, outdim, train_phase, scope_in):
fc = fully_connected(inputs, outdim, activation_fn=None, scope=scope_in + '/fc')
fc_bnorm = tf.layers.batch_normalization(fc, momentum=0.1, epsilon=1e-5,
training=train_phase, name=scope_in + '/bnorm')
fc_relu = tf.nn.relu(fc_bnorm, name=scope_in + '/relu')
fc_out = tf.layers.dropout(fc_relu, seed=0, training=train_phase, name=scope_in + '/dropout')
return fc_out
def embedding_loss(features, label, alfa, nrof_classes):
nrof_features = features.get_shape()[1]
centers = tf.get_variable('centers', [nrof_classes, nrof_features], dtype=tf.float32,
initializer=tf.constant_initializer(0), trainable=False)
label = tf.reshape(label, [-1])
centers_batch = tf.gather(centers, label)
diff = (1 - alfa) * (centers_batch - features)
#centers = tf.scatter_sub(centers, label, diff)
center_loss = tf.reduce_mean(tf.square(features - centers_batch))
#softmax_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=features))
#total_loss = softmax_loss + center_loss
return center_loss
def embedding_model(im_feats, train_phase, im_labels,
fc_dim=2048, embed_dim=512):
# Image branch.
im_fc1 = add_fc(im_feats, fc_dim, train_phase, 'im_embed_1')
im_fc2 = fully_connected(im_fc1, embed_dim, activation_fn=None,
scope='im_embed_2')
return tf.nn.l2_normalize(im_fc2, 1, epsilon=1e-10)
def softmax_loss(feat, im_labels):
label = tf.reshape(im_labels, [-1])
softmax = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=feat))
return softmax

why get train accuracy not test accuracy in tensorboard

I want to see test accuracy in tensorboard, but it seems I get accuracy with training data. I print test accuracy on console, and it is showing about 70%, but in tensorboard, the curve showed accuracy is growing and finally almost 100%.
This is my code:
def train_crack_captcha_cnn(is_train, checkpoint_dir):
global max_acc
X = tf.placeholder(tf.float32, [None, dr.ROWS, dr.COLS, dr.CHANNELS])
Y = tf.placeholder(tf.float32, [None, 1, 1, 2])
output, end_points = resnet_v2_50(X, num_classes = 2)
global_steps = tf.Variable(1, trainable=False)
learning_rate = tf.train.exponential_decay(0.001, global_steps, 100, 0.9)
with tf.device('/device:GPU:0'):
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=output))
# optimizer 为了加快训练 learning_rate应该开始大，然后慢慢衰
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss, global_step=global_steps)
predict = tf.argmax(output, axis = 3)
l = tf.argmax(Y, axis = 3)
correct_pred = tf.equal(predict, l)
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
## tensorboard
tf.summary.scalar('test_accuracy', accuracy)
tf.summary.scalar("loss", loss)
tf.summary.scalar("learning_rate", learning_rate)
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(allow_soft_placement = True)) as sess:
if is_train:
writer = tf.summary.FileWriter("/tmp/cnn_log/log", graph = sess.graph)
sess.run(tf.global_variables_initializer())
step_value = sess.run(global_steps)
while step_value < 100000:
step_value = sess.run(global_steps)
merged = tf.summary.merge_all()
batch_x, batch_y = get_next_batch()
result, _, _loss= sess.run([merged, optimizer, loss], feed_dict={X: batch_x, Y: batch_y})
writer.add_summary(result, step_value)
print('step : {} loss : {}'.format(step_value, _loss))
# 每100 step计算一次准确率
if step_value % 20 == 0:
acc = sess.run(accuracy, feed_dict={X: validation, Y: validation_labels})
print('accuracy : {}'.format(acc))
# 如果准确率大于max_acc,保存模型,完成训练
if acc > max_acc:
max_acc = float(acc) #转换类型防止变为同一个引用
saver.save(sess, checkpoint_dir + "/" + str(step_value) + '-' + str(acc) + "/model.ckpt", global_step=global_steps)
##### predict #####
# predict_y = sess.run(output, feed_dict={X: test})
# data = pd.DataFrame([i for i in range(1, len(predict_y) + 1)], columns = ['id'])
# predict_y = np.argmax(predict_y, axis = 3)
# predict_y = np.reshape(predict_y,(-1))
# print(predict_y)
# predict_y = pd.Series(predict_y, name='label')
# data['label'] = predict_y
# data.to_csv("gender_submission.csv" + str(step), index=False)
##### end #####
writer.close()
else:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
acc = sess.run(accuracy, feed_dict={X: validation, Y: validation_labels})
print('accuracy : {}'.format(acc))
I add accuracy into tensorboard like this:
tf.summary.scalar('test_accuracy', accuracy)
and every 20 step, I get one accuracy about test data, and print the result to console, which is not the same with data shown on tensorboard.
Why?

With Tensorflow, 2 class classification using Neural Network

I'm trying 2 class classification of images by neural network using Tensorflow.
I want to extract 1000 pixels randomly.
However, I am in trouble with error:
"logits = inference(images_placeholder, keep_prob)
File "train5.py", line 83, in inference
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
TypeError: 'Tensor' object is not callable"
Please tell me what should I do.
I will attach a code below.
import sys
sys.path.append('/usr/local/opt/opencv3/lib/python3.5.4/site-packages')
import cv2
import numpy as np
import tensorflow as tf
import tensorflow.python.platform
import tensorboard as tb
import os
import math
import time
import random
start_time = time.time()
# TensorBoard output information directory
log_dir = '/tmp/data1' #tensorboard --logdir=/tmp/data1
#directory delete and reconstruction
if tf.gfile.Exists(log_dir):
tf.gfile.DeleteRecursively(log_dir)
tf.gfile.MakeDirs(log_dir)
# Reserve memory
config = tf.ConfigProto(
gpu_options=tf.GPUOptions(allow_growth=True))
sess = sess = tf.Session(config=config)
NUM_CLASSES = 2
IMAGE_SIZE_x = 1024
IMAGE_SIZE_y = 768
IMAGE_CHANNELS = 1
IMAGE_PIXELS = IMAGE_SIZE_x*IMAGE_SIZE_y*IMAGE_CHANNELS
SAMPLE_PIXELS = 1000
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_string('train', 'train.txt', 'File name of train data')
flags.DEFINE_string('test', 'test.txt', 'File name of train data')
flags.DEFINE_string('image_dir', 'trdata', 'Directory of images')
flags.DEFINE_string('train_dir', '/tmp/data', 'Directory to put the training data.')
flags.DEFINE_integer('max_steps', 20000, 'Number of steps to run trainer.')
flags.DEFINE_integer('batch_size', 10, 'Batch size'
'Must divide evenly into the dataset sizes.')
flags.DEFINE_float('learning_rate', 1e-5, 'Initial learning rate.')
def inference(images_placeholder, keep_prob):
""" Function to create predictive model
argument:
images_placeholder: image placeholder
keep_prob: dropout rate placeholder
Return:
y_conv:
"""
# Initialie with normal distribution with weight of 0.1
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
# Initialized with normal distribution with bias of 0.1
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
# Reshape input
x_image = images_placeholder
# ramdom sumpling pixels
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
x_list = [samples[i] for i in list]
# Input
with tf.name_scope('fc1') as scope:
W_fc1 = weight_variable([x_list,10])
b_fc1 = bias_variable([10])
h_fc1 = tf.nn.relu(tf.matmul(x_image,W_fc1) + b_fc1)
# Affine1
with tf.name_scope('fc2') as scope:
W_fc2 = weight_variable([10,10])
b_fc2 = bias_variable([10])
h_fc2 = tf.nn.relu(tf.matmul(h_fc1,W_fc2) + b_fc2)
# Affine2
with tf.name_scope('fc3') as scope:
W_fc3 = weight_variable([10,10])
b_fc3 = bias_variable([10])
h_fc3 = tf.nn.relu(tf.matmul(h_fc2,W_fc3) + b_fc3)
# Affine3
with tf.name_scope('fc4') as scope:
W_fc4 = weight_variable([10,10])
b_fc4 = bias_variable([10])
h_fc4 = tf.nn.relu(tf.matmul(h_fc3,W_fc4) + b_fc4)
# Affine4
with tf.name_scope('fc5') as scope:
W_fc5 = weight_variable([10,2])
b_fc5 = bias_variable([2])
# softmax regression
with tf.name_scope('softmax') as scope:
y_out=tf.nn.softmax(tf.matmul(h_fc4, W_fc5) + b_fc5)
# return
return y_out
def loss(logits, labels):
""" loss function
引数:
logits: logit tensor, float - [batch_size, NUM_CLASSES]
labels: labrl tensor, int32 - [batch_size, NUM_CLASSES]
返り値:
cross_entropy:tensor, float
"""
# cross entropy
cross_entropy = -tf.reduce_sum(labels*tf.log(tf.clip_by_value(logits,1e-10,1.0)))
# TensorBoard
tf.summary.scalar("cross_entropy", cross_entropy)
return cross_entropy
def training(loss, learning_rate):
#Adam
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
return train_step
def accuracy(logits, labels):
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
tf.summary.scalar("accuracy", accuracy)
return accuracy
if __name__ == '__main__':
f = open(FLAGS.train, 'r')
# array data
train_image = []
train_label = []
for line in f:
# Separate space and remove newlines
line = line.rstrip()
l = line.split()
# Load data and resize
img = cv2.imread(FLAGS.image_dir + '/' + l[0])
img = cv2.resize(img, (IMAGE_SIZE_x, IMAGE_SIZE_y))
#transrate grayscale
img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# transrate one row and 0-1 float
train_image.append(img_gry.flatten().astype(np.float32)/255.0)
# Prepare with label 1-of-k method
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
train_label.append(tmp)
# transrate numpy
train_image = np.asarray(train_image)
train_label = np.asarray(train_label)
f.close()
f = open(FLAGS.test, 'r')
test_image = []
test_label = []
for line in f:
line = line.rstrip()
l = line.split()
img = cv2.imread(FLAGS.image_dir + '/' + l[0])
img = cv2.resize(img, (IMAGE_SIZE_x, IMAGE_SIZE_y))
#transrate grayscale
img_gry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# transrate one row and 0-1 float
test_image.append(img_gry.flatten().astype(np.float32)/255.0)
tmp = np.zeros(NUM_CLASSES)
tmp[int(l[1])] = 1
test_label.append(tmp)
test_image = np.asarray(test_image)
test_label = np.asarray(test_label)
f.close()
with tf.Graph().as_default():
# Put the image Tensor
images_placeholder = tf.placeholder("float", shape=(None, IMAGE_PIXELS))
# Put the label Tensor
labels_placeholder = tf.placeholder("float", shape=(None, NUM_CLASSES))
# Put dropout rate Tensor
keep_prob = tf.placeholder("float")
# Load inference() and make model
logits = inference(images_placeholder, keep_prob)
# Load loss() and calculate loss
loss_value = loss(logits, labels_placeholder)
# Load training() and train
train_op = training(loss_value, FLAGS.learning_rate)
# calculate accuracy
acc = accuracy(logits, labels_placeholder)
# save
saver = tf.train.Saver()
# Make Session
sess = tf.Session()
# Initialize variable
sess.run(tf.global_variables_initializer())
# TensorBoard
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
# Start training
for step in range(FLAGS.max_steps):
for i in range(int(len(train_image)/FLAGS.batch_size)):
batch = FLAGS.batch_size*i
sess.run(train_op, feed_dict={
images_placeholder: train_image[batch:batch+FLAGS.batch_size],
labels_placeholder: train_label[batch:batch+FLAGS.batch_size],
keep_prob: 0.5})
# Accuracy calculation for every steps
train_accuracy = sess.run(acc, feed_dict={
images_placeholder: train_image,
labels_placeholder: train_label,
keep_prob: 1.0})
print("step %d, training accuracy %g" %(step, train_accuracy))
# Added value to be displayed in Tensorflow every 1step
summary_str = sess.run(summary_op, feed_dict={
images_placeholder: train_image,
labels_placeholder: train_label,
keep_prob: 1.0})
summary_writer.add_summary(summary_str, step)
# Display accuracy on test data after training
print(" test accuracy %g"%sess.run(acc, feed_dict={
images_placeholder: test_image,
labels_placeholder: test_label,
keep_prob: 1.0}))
duration = time.time() - start_time
print('%.3f sec' %duration)
# Save model
save_path = saver.save(sess, os.getcwd() + "\\model.ckpt")

The error is this:
images_placeholder = tf.placeholder("float", shape=(None, IMAGE_PIXELS))
...
x_image = images_placeholder
list = random.sample(x_image(IMAGE_PIXELS),SAMPLE_PIXELS)
x_image, just like images_placeholder is a variable node, so x_image(...) doesn't make sense and obviously leads to an error "TypeError: 'Tensor' object is not callable".
I assume you're trying to sample SAMPLE_PIXELS from each image in a batch. Note that random.sample won't work here, because x_image is a symbolic variable, its value is only known during the session. You have to use tf.boolean_mask with a random mask in order to select random pixels from the image.

tensorflow multi-gpu mnist example, loss does not decrease

I'm trying to write my own mnist example which could use all the two gpu of one machine.
It is a simple multi-layer perceptron.
Here is my code. You can run it directly.
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
import tensorflow as tf
learning_rate = 0.001
training_steps = 100000
batch_size = 100
display_step = 100
n_hidden_1 = 256
n_hidden_2 = 256
n_input = 784
n_classes = 10
def _variable_on_cpu(name, shape, initializer):
with tf.device('/cpu:0'):
dtype = tf.float32
var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
return var
def build_model():
def multilayer_perceptron(x, weights, biases):
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
layer_2 = tf.nn.relu(layer_2)
out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
return out_layer
with tf.variable_scope('aaa'):
weights = {
'h1': _variable_on_cpu('h1',[n_input, n_hidden_1],tf.constant_initializer(0.0)),
'h2': _variable_on_cpu('h2',[n_hidden_1, n_hidden_2],tf.constant_initializer(0.0)),
'out': _variable_on_cpu('out_w',[n_hidden_2, n_classes],tf.constant_initializer(0.0))
}
biases = {
'b1': _variable_on_cpu('b1',[n_hidden_1],tf.constant_initializer(0.0)),
'b2': _variable_on_cpu('b2',[n_hidden_2],tf.constant_initializer(0.0)),
'out': _variable_on_cpu('out_b',[n_classes],tf.constant_initializer(0.0))
}
pred = multilayer_perceptron(x, weights, biases)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
return cost
def average_gradients(tower_grads):
average_grads = []
for grad_and_vars in zip(*tower_grads):
grads = []
for g,_ in grad_and_vars:
expanded_g = tf.expand_dims(g, 0)
grads.append(expanded_g)
grad = tf.concat(axis=0, values=grads)
grad = tf.reduce_mean(grad, 0)
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
with tf.Graph().as_default(), tf.device('/cpu:0'):
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])
tower_grads = []
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
with tf.variable_scope(tf.get_variable_scope()):
for i in xrange(2):
with tf.device('/gpu:%d' % i):
cost = build_model()
tf.get_variable_scope().reuse_variables()
grads = optimizer.compute_gradients(cost)
tower_grads.append(grads)
grads = average_gradients(tower_grads)
apply_gradient_op = optimizer.apply_gradients(grads)
train_op = apply_gradient_op
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
for step in range(training_steps):
image_batch, label_batch = mnist.train.next_batch(batch_size)
_, cost_print = sess.run([train_op, cost],
{x:image_batch,
y:label_batch})
if step % display_step == 0:
print("step=%04d" % (step+1)+ " cost=" + str(cost_print))
print("Optimization Finished!")
sess.close()
The print info looks like:
step=0001 cost=2.30258
step=0101 cost=2.30246
step=0201 cost=2.30128
step=0301 cost=2.30376
step=0401 cost=2.29817
step=0501 cost=2.2992
step=0601 cost=2.3104
step=0701 cost=2.29995
step=0801 cost=2.29802
step=0901 cost=2.30524
step=1001 cost=2.29673
step=1101 cost=2.30016
step=1201 cost=2.31057
step=1301 cost=2.29815
step=1401 cost=2.29669
step=1501 cost=2.30345
step=1601 cost=2.29811
step=1701 cost=2.30867
step=1801 cost=2.30757
step=1901 cost=2.29716
step=2001 cost=2.30394
The loss doesn't decrease. I don't know how to fix it.
By the way, GPU-Util is about 26% and 26%. How to increase the GPU-Util?

The problem is that,
I should use tf.constant_initializer(0.1) for the weights instead of tf.constant_initializer(0)

Why does Tensorflow network predict different results each run?

I have a tensorflow network that is currently predicting a value 1-5 from an input of 14 numbers. Right now it is set up with the input test_X = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1]) and dispite seeding it, the network returns different classifications every time. I am also unsure of how to predict the confidence of the prediction (I'm assuming this should be a float value in between 0 and 1). My code can be found below:
import tensorflow as tf
import pandas as pd
import numpy as np
df = pd.read_csv('/Users/zach/desktop/export.csv')
data_ = df.drop(['ID','Species'], axis=1)
n_classes = data_["Phylum"].nunique()
dim = 14
learning_rate = 0.0001
display_step = 10
n_hidden_1 = 2000
n_hidden_2 = 1500
n_hidden_3 = 1000
n_hidden_4 = 500
X = tf.placeholder(tf.float32, [None, dim])
train_set = data_.sample(frac=0.7)
test_set = data_.loc[~data_.index.isin(train_set.index)]
train_size = train_set.size
inputY_test = pd.get_dummies(test_set['Phylum'])
inputY_train = pd.get_dummies(train_set['Phylum'])
train_X = train_set.iloc[:train_size, 5:].as_matrix()
train_X = pd.DataFrame(data=train_X)
train_X = train_X.fillna(value=0).as_matrix()
train_Y = inputY_train.as_matrix()
train_Y = pd.DataFrame(data=train_Y)
train_Y = train_Y.fillna(value=0).as_matrix()
#test_X = test_set.iloc[:, 5:].as_matrix()
#test_X = pd.DataFrame(data=test_X)
#test_X = test_X.fillna(value=0).as_matrix()
test_X = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1])
test_X = pd.DataFrame(data=test_X)
test_X = test_X.fillna(value=0).as_matrix()
test_X.resize(1,14)
test_Y = inputY_test.as_matrix()
test_Y = pd.DataFrame(data=test_Y)
test_Y = test_Y.fillna(value=0).as_matrix()
n_samples = train_Y.size
total_len = train_X.shape[0]
n_input = train_X.shape[1]
batch_size = 5
W = tf.Variable(tf.zeros([dim, n_classes]))
b = tf.Variable(tf.zeros([n_classes]))
def multilayer_perceptron(x, weights, biases):
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
# Hidden layer with RELU activation
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
layer_2 = tf.nn.relu(layer_2)
# Hidden layer with RELU activation
layer_3 = tf.add(tf.matmul(layer_2, weights['h3']), biases['b3'])
layer_3 = tf.nn.relu(layer_3)
# Hidden layer with RELU activation
layer_4 = tf.add(tf.matmul(layer_3, weights['h4']), biases['b4'])
layer_4 = tf.nn.relu(layer_4)
# Output layer with linear activation
out_layer = tf.matmul(layer_4, weights['out']) + biases['out']
return out_layer
# Store layers weight & bias
weights = {
'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], 0, 0.1)),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], 0, 0.1)),
'h3': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_3], 0, 0.1)),
'h4': tf.Variable(tf.random_normal([n_hidden_3, n_hidden_4], 0, 0.1)),
'out': tf.Variable(tf.random_normal([n_hidden_4, n_classes], 0, 0.1))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1], 0, 0.1)),
'b2': tf.Variable(tf.random_normal([n_hidden_2], 0, 0.1)),
'b3': tf.Variable(tf.random_normal([n_hidden_3], 0, 0.1)),
'b4': tf.Variable(tf.random_normal([n_hidden_4], 0, 0.1)),
'out': tf.Variable(tf.random_normal([n_classes], 0, 0.1))
}
# Construct model
pred = multilayer_perceptron(X, weights, biases)
pred1 = tf.argmax(pred,1)
y = tf.placeholder(tf.float32, [None, n_classes])
#cost = -tf.reduce_sum(y*tf.log(tf.clip_by_value(pred,1e-10,1.0)))
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)
hm_epochs = 20
tf.set_random_seed(1234)
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
for epoch in range(hm_epochs):
avg_cost = 0
total_batch = int(total_len/batch_size)
for i in range(total_batch-1):
batch_x = train_X[i*batch_size:(i+1)*batch_size]
batch_y = train_Y[i*batch_size:(i+1)*batch_size]
_, c, p = sess.run([optimizer, cost, pred], feed_dict={X: batch_x,
y: batch_y})
avg_cost += c / total_batch
label_value = batch_y
estimate = p
err = label_value-estimate
if epoch % display_step == 0:
print "Epoch:", '%04d' % (epoch+1), "cost=", \
"{:.9f}".format(avg_cost)
print "[*]----------------------------------------------------"
for i in xrange(3):
print "label value:", label_value[i], \
"estimated value:", estimate[i]
print "======================================================="
print "Optimization Finished!"
#Test model
#correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
#Calculate accuracy
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
#print ("Accuracy:", accuracy.eval({X: test_X, y: test_Y}))
feed_dict = {X: test_X}
classification = pred.eval(feed_dict)
print "Network Prediction:", classification
print "Classification: ", np.argmax(classification) + 1
The network returns something like this (with the "Network Prediction" and "classification" values differing significantly with each run)
Epoch: 0001 cost= 18.784451194
[*]----------------------------------------------------
label value: [1 0 0 0 0] estimated value: [ 7.22390413 14.61596966 -0.08044712 41.44412231 -2.90992975]
label value: [1 0 0 0 0] estimated value: [-18.6686306 51.33540726 65.4961853 67.72460938 -3.51442194]
label value: [0 0 0 1 0] estimated value: [-32.95540619 22.96526909 -18.13385201 70.66561127 -28.7767086 ]
=======================================================
Epoch: 0011 cost= 0.798838628
[*]----------------------------------------------------
label value: [1 0 0 0 0] estimated value: [ 11.22842598 -3.67433786 -14.22806835 1.45541549 -7.12817001]
label value: [1 0 0 0 0] estimated value: [ 24.90147591 16.51822853 23.89280701 24.66276169 21.37914276]
label value: [0 0 0 1 0] estimated value: [ 0.13968639 7.37386036 -21.62825203 21.48152542 -23.18427277]
=======================================================
Optimization Finished!
Network Prediction: [[ 3.85959864 0.32653514 0.26948914 -0.70163095 -2.03481865]]
Classification: 1
How can I stabilized the predictions of the network so it predicts the same outcome each time when the input remains constant? Also, how would I display the confidence of this prediction?