Problem Summary:
The issue is that even after running this code for multiple epochs, the cost isn't reducing much ( I have tried this for a variety of starting_learning_rates ). The equation that I am trying to optimize is ((m * pow(length, u) * pow(start_y, t) + c) where length and start_y are the inputs and u,t,m and c are learn-able parameters. I was able to observe (my dataset is quite small) that length * sqrt(start_y) is almost a constant and thought that tensorflow would be able to better help me find the value of the variables
This is my tensorflow code, combined_vehicles is an array with 129 rows and 2 columns( 2 features ), combined_labels is an array corresponding to labels for each of the examples in combined_vehicles
u = tf.Variable(0.0,dtype = "float32")
t = tf.Variable(0.0,dtype = "float32")
c = tf.Variable(0.0,dtype = "float32")
m = tf.Variable(0.0,dtype = "float32")
length = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name="length")
start_y = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name="start_y")
labels = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name = "labels")
output = tf.add(tf.multiply(tf.multiply(tf.pow(length, u), tf.pow(start_y, t)), m), c)
cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = output, labels = labels))
global_step = tf.Variable(0, trainable=False, name = 'global_step')
start_learning_rate = 0.0001
decay_steps = 100
learning_rate = tf.train.exponential_decay(start_learning_rate, global_step, decay_steps, 0.1, staircase=True )
result_output = output > 0.5
result_label = combined_labels > 0.5
correct_prediction = tf.equal( result_output, result_label )
accuracy = tf.reduce_mean( tf.cast( correct_prediction, "float" ) )
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost, global_step=global_step)
init = tf.global_variables_initializer()
with tf.Session() as sess:
epochs = 100
sess.run(init)
for i in range(epochs):
_,cost_estimate = sess.run([optimizer, cost], feed_dict = {length: combined_vehicles[:,0].reshape([combined_vehicles.shape[0],1]), start_y:combined_vehicles[:,1].reshape([combined_vehicles.shape[0],1]), labels: combined_labels})
total_accuracy = accuracy.eval({length: combined_vehicles[:,0].reshape([combined_vehicles.shape[0],1]), start_y:combined_vehicles[:,1].reshape([combined_vehicles.shape[0],1]), labels: combined_labels})
Related
I am trying to use Tensorflow to calculate the linear regression of some data.
I do not understand why cannot predict a decent line.
Below the result I am getting:
This is my code, I have tried to change different parameters but nothing to do.
Any suggestion is welcome.
# Prepare the data
x = df["Attainment8_float"]
y = df["Progress8_float"]
# Check the data
plt.scatter(x, y)
plt.show()
# TensorFlow Model
# Config
num_epochs = 1000
learning_rate = 0.0001
# /Config
# Creating the graph
ops.reset_default_graph()
tf.disable_v2_behavior()
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
a = tf.get_variable('a', initializer=0.)
b = tf.get_variable('b', initializer=0.)
h = a * X + b
cost = tf.reduce_mean( (h - Y)**2 )
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate
).minimize(cost)
init = tf.global_variables_initializer()
# Running the Model
found_a = 0
found_b = 0
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
_, costValue = sess.run(
[optimizer, cost],
feed_dict={
X: x,
Y: y,
}
)
found_a = a.eval()
found_b = b.eval()
if epoch % (num_epochs/10) == 0: # Every 10 percent
print("... epoch: " + str(epoch))
print(f"cost[{str(costValue)}] / a[{str(a.eval())}] / b[{str(b.eval())}]")
# Seing the obtained values in a plot
xrange = np.linspace(x.min(), x.max(), 2)
# Plot points
plt.plot(x, y, 'ro')
# Plot resulting function
plt.plot(xrange, xrange * found_a + found_b, 'b')
plt.show()
When I run it with
a = tf.get_variable('a', initializer= 0.05)
b = tf.get_variable('b', initializer=-2.0)
I get
However, I did some data preprocessing. I removed entries with "." as you did as far as I can see. Furthermore I removed entries with "x", so code looks like:
df = df[df.Attainment8 != "."]
df = df[df.Progress8 != "."]
df = df[df.Attainment8 != "x"]
df = df[df.Progress8 != "x"]
#convert object in float
df["Attainment8_float"] = df["Attainment8"].astype(float)
df["Progress8_float"]= df["Progress8"].astype(float)
When I additionally use (together with initializer set to 0.05 and -2.0)
num_epochs = 2000
learning_rate = 0.000001
I get
How important is to choose the right value for the initializer in tensorflow?
With this code:
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=-3.0)
with:
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=0.0)
Why in the second example tensorflow doesn`t manage to fit the data properly? There is anything that can be done changing number_epochs or learning_rate?
This is my code:
# TensorFlow Model
# Config
num_epochs = 2000
learning_rate = 0.0001
# /Config
# Creating the graph
ops.reset_default_graph()
tf.disable_v2_behavior()
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=-3.0)
h = a * X + b
cost = tf.reduce_mean( (h - Y)**2 )
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate
).minimize(cost)
init = tf.global_variables_initializer()
# Running the Model
found_a = 0
found_b = 0
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
_, costValue = sess.run(
[optimizer, cost],
feed_dict={
X: x,
Y: y,
}
)
found_a = a.eval()
found_b = b.eval()
if epoch % (num_epochs/10) == 0: # Every 10 percent
print("... epoch: " + str(epoch))
print(f"cost[{str(costValue)}] / a[{str(a.eval())}] / b[{str(b.eval())}]")
# Seing the obtained values in a plot
xrange = np.linspace(x.min(), x.max(), 2)
# Plot points
plt.plot(x, y, 'ro')
# Plot resulting function
plt.plot(xrange, xrange * found_a + found_b, 'b')
plt.show()
I have a problem with a tensorflow NN. I got an error "You must feed a value for placeholder tensor 'palceholders_5/Placeholder' with dtype float and shape [?,1]"
I have tried different shapes of the placeholder as well as the shape of the numpy array in feed dictionary. The code is given below.
import numpy as np
import tensorflow as tf
###################################
# data
###################################
N = 100
w_true = 5
b_true = 2
noise_scale = 0.1
x_np = np.random.rand(N,1)
noise = np.random.normal(scale=noise_scale, size = (N,1))
y_np = np.reshape(w_true*x_np+b_true+noise,(-1))
###########################
# Model
###########################
n_hidden = 10
with tf.name_scope("palceholders"):
x = tf.placeholder(tf.float32, (None,1))
y = tf.placeholder(tf.float32, (None,))
keep_prob = tf.placeholder(tf.float32)
with tf.name_scope("hidden-layer"):
W = tf.Variable(tf.random_normal((1, n_hidden)))
b = tf.Variable(tf.random_normal((n_hidden,)))
x_hidden = tf.nn.relu(tf.matmul(x,W) + b)
#add droput
x_hidden = tf.nn.dropout(x_hidden, keep_prob)
with tf.name_scope("output"):
W = tf.Variable(tf.random_normal((n_hidden,1)))
b = tf.Variable(tf.random_normal((1,)))
y_logit = tf.matmul(x_hidden, W) + b
y_one_prob = tf.sigmoid(y_logit)
y_pred = tf.round(y_one_prob)
with tf.name_scope("loss"):
y_expand = tf.expand_dims(y,1)
entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=y_logit, labels=y_expand)
l = tf.reduce_sum(entropy)
with tf.name_scope("optim"):
train_op = tf.train.AdamOptimizer(.001).minimize(l)
with tf.name_scope("summaries"):
tf.summary.scalar("loss",l)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter("tmp/full-train", tf.get_default_graph())
#########################
# train model
#########################
step = 0
n_epochs = 15
batch_size = 10
dropout_prob = 0.5
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(n_epochs):
pos = 0
while pos < N:
batch_x = x_np[pos:pos+batch_size]
batch_x = batch_x.reshape((-1,1))
print(batch_x.shape)
batch_y = y_np[pos:pos+batch_size]
feed_dict = {x:batch_x, y:batch_y, keep_prob:dropout_prob}
_, summary, loss = sess.run([train_op, merged, l], feed_dict=feed_dict)
print("epoch %d, step %d, loss %f" % (epoch, step, loss))
train_writer.add_summary(summary, step)
step += 1
pos += batch_size
I get an error:
"InvalidArgumentError: You must feed a value for placeholder tensor 'palceholders_5/Placeholder' with dtype float and shape [?,1]"
I'm experimenting with TensorFlow (which seems amazing so far!) and I'm playing around with a toy example a 1 class classification problem. I'm generating some features and if the first feature is above a threshold then the example is "positive"
Full code here:
https://gist.github.com/tnbredillet/f136c2bc40815517e0aa1139bd2060ee
The problem is that it seems that the model is unable to capture that simple relationship.
Of course I'm missing a lot of stuff (CV, regularization, batch normalization, hyperparameter tuning) to name a few.
But still I would expect the model to manage to figure that one out right ?
Maybe there's simply a bug in my code?
Would welcome any insights :-)
EDIT:
Data generating code:
num_examples = 100000
split = 0.2
num_features = 1
def generate_input_data(num_examples, num_features):
features = []
labels = []
for i in xrange(num_examples):
features.append(np.random.rand(num_features) * np.random.randint(1, 10) + np.random.rand(num_features))
if np.random.randint(101) > 90:
features[i-1][np.random.randint(num_features)] = 0
hard = ceil(np.sum(features[i-1])) % 2
easy = 0
if features[i-1][0] > 3:
easy = 1
labels.append(easy)
df = pd.concat(
[
pd.DataFrame(features),
pd.Series(labels).rename('labels')
],
axis=1,
)
return df
def one_hot_encoding(train_df):
#TODO: handle categorical feature one hot encoding.
return 0, 0
def scale_data(train_df, test_df):
categorical_columns, encoding = one_hot_encoding(train_df)
scaler = MinMaxScaler(feature_range=(0,1))
scaler.fit(train_df.drop(['labels'], axis=1))
train_df = pd.concat(
[
pd.DataFrame(scaler.transform(train_df.drop('labels', axis=1))),
train_df['labels']
],
axis=1,
)
test_df = pd.concat(
[
pd.DataFrame(scaler.transform(test_df.drop('labels', axis=1))),
test_df['labels']
],
axis=1,
)
return train_df, test_df
def preprocess_data(train_df, test_df):
all_dfs = [train_df, test_df]
features = set()
for df in all_dfs:
features |= set(df.columns)
for df in all_dfs:
for f in features:
if f not in df.columns:
df[f] = 0.0
for df in all_dfs:
df.sort_index(axis=1, inplace=True)
train_df, test_df = scale_data(train_df, test_df)
train_df = shuffle(train_df).reset_index(drop=True)
return train_df, test_df
def get_data(num_examples, split):
train_df = generate_input_data(num_examples, num_features)
test_df = generate_input_data(int(ceil(num_examples*split)), num_features)
return preprocess_data(train_df, test_df)
def get_batch(df, batch_size, epoch):
start = batch_size*epoch-batch_size
end = batch_size*epoch
if end > len(df):
end = len(df)
size = end - start
batch_x = df.drop('labels', axis=1)[start:end].as_matrix()
batch_y = df['labels'][start:end].as_matrix().reshape(size, 1)
return batch_x, batch_y
And the network definition/training and evaluation:
train_df, test_df = get_data(num_examples, split)
n_hidden_1 = 8
n_hidden_2 = 4
learning_rate = 0.01
batch_size = 500
num_epochs = 200
display_epoch = 50
def neural_net(x):
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
return out_layer
weights = {
'h1': tf.Variable(tf.random_normal([num_features, n_hidden_1])),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_hidden_2, 1]))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1])),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
'out': tf.Variable(tf.random_normal([1]))
}
X = tf.placeholder(tf.float32, shape=(None, num_features))
Y = tf.placeholder(tf.float32, shape=(None, 1))
logits = neural_net(X)
loss_op = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
predictions = tf.sigmoid(logits)
predicted_class = tf.greater(predictions, 0.5)
correct = tf.equal(predicted_class, tf.equal(Y,1.0))
accuracy = tf.reduce_mean( tf.cast(correct, 'float') )
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
for epoch in range(1, num_epochs + 1):
batch_x, batch_y = get_batch(train_df, batch_size, epoch)
sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})
if epoch % display_epoch == 0 or epoch == 1:
loss, acc , pred, fff= sess.run([loss_op, accuracy, predictions, logits],
feed_dict={X: batch_x,
Y: batch_y})
c = ', '.join('{}={}'.format(*t) for t in zip(pred, batch_y))
print("[{}] Batch loss={:.4f}, Accuracy={:.5f}, Logits vs labels= {}".format(epoch, loss, acc, c))
print("Optimization Finished!")
batch_x, batch_y = get_batch(test_df, batch_size, 1)
print("Testing Accuracy:", \
sess.run(accuracy, feed_dict={X: batch_x,
Y: batch_y}))
final output:
[1] Batch loss=3.2160, Accuracy=0.41000
[50] Batch loss=0.6661, Accuracy=0.61800
[100] Batch loss=0.6472, Accuracy=0.65200
[150] Batch loss=0.6538, Accuracy=0.64000
[200] Batch loss=0.6508, Accuracy=0.64400
Optimization Finished!
('Testing Accuracy:', 0.63999999)
In this case it is not a machine learning algorithm problem, but a bug in your data generation which is scrambling the relationship that you intend. In this function:
def generate_input_data(num_examples, num_features):
features = []
labels = []
for i in xrange(num_examples):
features.append(np.random.rand(num_features) * np.random.randint(1, 10) + np.random.rand(num_features))
if np.random.randint(101) > 90:
features[i-1][np.random.randint(num_features)] = 0
hard = ceil(np.sum(features[i-1])) % 2
easy = 0
if features[i-1][0] > 3:
easy = 1
labels.append(easy)
df = pd.concat(
[
pd.DataFrame(features),
pd.Series(labels).rename('labels')
],
axis=1,
)
return df
You are indexing features by i-1 to determine the label. However, xrange will generate numbers starting from 0, so you don't need to subtract the 1. In fact, when you do, the relationship becomes close to random, and essentially unpredictable, so even though the rest of your model is OK, it won't be able to score well.
So you need to index by i instead e.g. if features[i][0] > 3.
This question already has answers here:
Save Tensorflow graph for viewing in Tensorboard without summary operations
(5 answers)
Closed 5 years ago.
I am trying to use tensorboard to analyse a graph in tensorflow with summaryWriter. However, TensorFlow is not outputting a 'graph' folder with information. Perhaps I am missing a command or it is not in the right place?
writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph());
Is what I used. I think this may not work for TensorFlow 1.0 anymore (just the summarywriter command)
import numpy as np
import tensorflow as tf
# %matplotlib inline
import matplotlib.pyplot as plt
# Global config variables
num_steps = 5 # number of truncated backprop steps ('n' in the discussion above)
batch_size = 200
num_classes = 2
state_size = 4
learning_rate = 0.1
logs_path = "./graph"
def gen_data(size=1000000):
X = np.array(np.random.choice(2, size=(size,)))
Y = []
for i in range(size):
threshold = 0.5
if X[i-3] == 1:
threshold += 0.5
if X[i-8] == 1:
threshold -= 0.25
if np.random.rand() > threshold:
Y.append(0)
else:
Y.append(1)
return X, np.array(Y)
# adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/reader.py
def gen_batch(raw_data, batch_size, num_steps):
raw_x, raw_y = raw_data
data_length = len(raw_x)
# partition raw data into batches and stack them vertically in a data matrix
batch_partition_length = data_length // batch_size
data_x = np.zeros([batch_size, batch_partition_length], dtype=np.int32)
data_y = np.zeros([batch_size, batch_partition_length], dtype=np.int32)
for i in range(batch_size):
data_x[i] = raw_x[batch_partition_length * i:batch_partition_length * (i + 1)]
data_y[i] = raw_y[batch_partition_length * i:batch_partition_length * (i + 1)]
# further divide batch partitions into num_steps for truncated backprop
epoch_size = batch_partition_length // num_steps
for i in range(epoch_size):
x = data_x[:, i * num_steps:(i + 1) * num_steps]
y = data_y[:, i * num_steps:(i + 1) * num_steps]
yield (x, y)
def gen_epochs(n, num_steps):
for i in range(n):
yield gen_batch(gen_data(), batch_size, num_steps)
"""
Placeholders
"""
x = tf.placeholder(tf.int32, [batch_size, num_steps], name='input_placeholder')
y = tf.placeholder(tf.int32, [batch_size, num_steps], name='labels_placeholder')
init_state = tf.zeros([batch_size, state_size])
"""
Inputs
"""
x_one_hot = tf.one_hot(x, num_classes)
rnn_inputs = tf.unstack(x_one_hot, axis=1)
"""
RNN
"""
cell = tf.contrib.rnn.BasicRNNCell(state_size)
rnn_outputs, final_state = tf.contrib.rnn.static_rnn(cell, rnn_inputs, initial_state=init_state)
"""
Predictions, loss, training step
"""
with tf.variable_scope('softmax'):
W = tf.get_variable('W', [state_size, num_classes])
b = tf.get_variable('b', [num_classes], initializer=tf.constant_initializer(0.0))
logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]
predictions = [tf.nn.softmax(logit) for logit in logits]
y_as_list = [tf.squeeze(i, axis=[1]) for i in tf.split(axis=1, num_or_size_splits=num_steps, value=y)]
loss_weights = [tf.ones([batch_size]) for i in range(num_steps)]
losses = tf.contrib.legacy_seq2seq.sequence_loss_by_example(logits, y_as_list, loss_weights)
tf.scalar_summary("losses", losses)
total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(learning_rate).minimize(total_loss)
# Not sure why this is not outputting a graph for tensorboard
writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph());
"""
Function to train the network
"""
def train_network(num_epochs, num_steps, state_size=4, verbose=True):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
training_losses = []
saved = gen_epochs(num_epochs, num_steps);
for idx, epoch in enumerate(gen_epochs(num_epochs, num_steps)):
training_loss = 0
training_state = np.zeros((batch_size, state_size))
if verbose:
print("\nEPOCH", idx)
for step, (X, Y) in enumerate(epoch):
tr_losses, training_loss_, training_state, _ = \
sess.run([losses,
total_loss,
final_state,
train_step],
feed_dict={x:X, y:Y, init_state:training_state})
training_loss += training_loss_
if step % 100 == 0 and step > 0:
if verbose:
print("Average loss at step", step,
"for last 250 steps:", training_loss/100)
training_losses.append(training_loss/100)
training_loss = 0
return training_losses
training_losses = train_network(1,num_steps)
plt.plot(training_losses)
# tensorboard --logdir="my_graph"
This worked for me:
writer = tf.summary.FileWriter(logdir='logdir', graph=tf.get_default_graph())
writer.flush()