Develop Reference

tf.get_variable initializer Tensorflow

How important is to choose the right value for the initializer in tensorflow?
With this code:
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=-3.0)
with:
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=0.0)
Why in the second example tensorflow doesn`t manage to fit the data properly? There is anything that can be done changing number_epochs or learning_rate?
This is my code:
# TensorFlow Model
# Config
num_epochs = 2000
learning_rate = 0.0001
# /Config
# Creating the graph
ops.reset_default_graph()
tf.disable_v2_behavior()
X = tf.placeholder(tf.float32, name='X')
Y = tf.placeholder(tf.float32, name='Y')
a = tf.get_variable('a', initializer=0.1)
b = tf.get_variable('b', initializer=-3.0)
h = a * X + b
cost = tf.reduce_mean( (h - Y)**2 )
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate
).minimize(cost)
init = tf.global_variables_initializer()
# Running the Model
found_a = 0
found_b = 0
with tf.Session() as sess:
sess.run(init)
for epoch in range(num_epochs):
_, costValue = sess.run(
[optimizer, cost],
feed_dict={
X: x,
Y: y,
}
)
found_a = a.eval()
found_b = b.eval()
if epoch % (num_epochs/10) == 0: # Every 10 percent
print("... epoch: " + str(epoch))
print(f"cost[{str(costValue)}] / a[{str(a.eval())}] / b[{str(b.eval())}]")
# Seing the obtained values in a plot
xrange = np.linspace(x.min(), x.max(), 2)
# Plot points
plt.plot(x, y, 'ro')
# Plot resulting function
plt.plot(xrange, xrange * found_a + found_b, 'b')
plt.show()

Computing weights in linear regression problem

I have written the script that demonstrates the linear regression algorithm as follows:
training_epochs = 100
learning_rate = 0.01
# the training set
x_train = np.linspace(0, 10, 100)
y_train = x_train + np.random.normal(0,1,100)
# set up placeholders for input and output
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)
# set up variables for weights
w0 = tf.Variable(0.0, name="w0")
w1 = tf.Variable(0.0, name="w1")
y_predicted = X*w1 + w0
# Define the cost function
costF = 0.5*tf.square(Y-y_predicted)
# Define the operation that will be called on each iteration
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(costF)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
# Loop through the data training
for epoch in range(training_epochs):
for (x, y) in zip(x_train, y_train):
sess.run(train_op, feed_dict={X: x, Y: y})
# get values of the final weights
w_val_0,w_val_1 = sess.run([w0,w1])
sess.close()
With this script above, I could compute w_val_1 and w_val_0 easily. But if I changed something with the y_predicted:
w0 = tf.Variable(0.0, name="w0")
w1 = tf.Variable(0.0, name="w1")
w2 = tf.Variable(0.0, name="w2")
y_predicted = X*X*w2 + X*w1 + w0
...
w_val_0,w_val_1,w_val_2 = sess.run([w0,w1,w2])
then I couldn't compute w_val_0, w_val_1, w_val_2. Please help me!

When you are doing X*X the weight (w2, w1 and w0) increase rapidly reaching inf which results in nan values in the loss and no training happens. As a rule of thumb always normalize the data to 0 mean and unit variance.
Fixed code
training_epochs = 100
learning_rate = 0.01
# the training set
x_train = np.linspace(0, 10, 100)
y_train = x_train + np.random.normal(0,1,100)
# # Normalize the data
x_mean = np.mean(x_train)
x_std = np.std(x_train)
x_train_ = (x_train - x_mean)/x_std
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)
# set up variables for weights
w0 = tf.Variable(0.0, name="w0")
w1 = tf.Variable(0.0, name="w1")
w2 = tf.Variable(0.0, name="w3")
y_predicted = X*X*w1 + X*w2 + w0
# Define the cost function
costF = 0.5*tf.square(Y-y_predicted)
# Define the operation that will be called on each iteration
train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(costF)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
# Loop through the data training
for epoch in range(training_epochs):
for (x, y) in zip(x_train_, y_train):
sess.run(train_op, feed_dict={X: x, Y: y})
y_hat = sess.run(y_predicted, feed_dict={X: x_train_})
print (sess.run([w0,w1,w2]))
sess.close()
plt.plot(x_train, y_train)
plt.plot(x_train, y_hat)
plt.show()
output:
[4.9228806, -0.08735728, 3.029659]

Having trouble getting TensorFlow to do something trivial

I have a vector x and want to compute a vector y such that y[j] = x[j]**2 using a neural network specified by TensorFlow, below. It doesn't work so well, the error is high.
Am I doing something wrong?
Any help will be appreciated
The way it works is it first generates data in Xtrain, Ytrain, Xtest, and Ytest and then creates placeholder variables to get TensorFlow going.
Then it specifies three hidden layers and one output layer. Then it trains, and Ypred, the prediction for Ytest, is created using a feed dictionary.
import numpy as np
import tensorflow as tf
n = 10
k = 1000
n_hidden = 10
learning_rate = .01
training_epochs = 100000
Xtrain = []
Ytrain = []
Xtest = []
Ytest = []
for i in range(0,k,1):
X = np.random.randn(1,n)[0]
Xtrain += [X]
Ytrain += [Xtrain[-1]**2]
X = np.random.randn(1,n)[0]
Xtest += [X]
Ytest += [Xtest[-1]**2]
x = tf.placeholder(tf.float64,shape = (k,n))
y = tf.placeholder(tf.float64,shape = (k,n))
W1 = tf.Variable(tf.random_normal((n,n_hidden),dtype = tf.float64))
b1 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
x_hidden1 = tf.nn.sigmoid(tf.matmul(x,W1) + b1)
W2 = tf.Variable(tf.random_normal((n,n_hidden),dtype = tf.float64))
b2 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
x_hidden2 = tf.nn.sigmoid(tf.matmul(x_hidden1,W2) + b2)
W3 = tf.Variable(tf.random_normal((n,n_hidden),dtype = tf.float64))
b3 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
x_hidden3 = tf.nn.sigmoid(tf.matmul(x_hidden1,W3) + b3)
W4 = tf.Variable(tf.random_normal((n,n_hidden),dtype = tf.float64))
b4 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
y_pred = tf.matmul(x_hidden3,W4) + b4
penalty = tf.reduce_sum(tf.abs((y - y_pred)))
train_op = tf.train.AdamOptimizer(learning_rate).minimize(penalty)
model = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(model)
for i in range(0,training_epochs):
sess.run(train_op,{x: Xtrain,y: Ytrain})
Ypred = y_pred.eval(feed_dict = {x: Xtest})

Here is just some simple modification of your code:
import numpy as np
import tensorflow as tf
n = 10
k = 1000
learning_rate = 1e-3
training_epochs = 100000
# It will be better for you to use PEP8 style
# None here will allow you to feed data with ANY k size
x = tf.placeholder(tf.float64, shape=(None, n))
y = tf.placeholder(tf.float64, shape=(None, n))
# Use default layer constructors
# from your implementation it uses another random initializer
out = tf.layers.dense(x, 100)
out = tf.layers.batch_normalization(out)
# ReLU is better than sigmoid, there are a lot of articles about it
out = tf.nn.relu(out)
out = tf.layers.dense(out, 200)
out = tf.layers.batch_normalization(out)
out = tf.nn.relu(out)
out = tf.layers.dense(out, n)
# total loss = mean L1 for samples
# each sample is a vector of 10 values, so you need to calculate
# sum along first axis, and them calculate mean of sums
l1 = tf.reduce_mean(tf.reduce_sum(tf.abs(y - out), axis=1))
train_op = tf.train.AdamOptimizer(learning_rate).minimize(l1)
model = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(model)
for i in range(training_epochs):
xs = np.random.randn(k, n)
ys = xs ** 2
_, l1_value = sess.run(
[train_op, l1],
feed_dict={x: xs, y: ys})
if (i + 1) % 10000 == 0 or i == 0:
print('Current l1({}/{}) = {}'.format(
i + 1, training_epochs, l1_value))
xs = np.random.randn(k, n)
ys = xs ** 2
test_l1 = sess.run(l1, feed_dict={x: xs, y: ys})
print ('Total l1 at test = {}'.format(test_l1))
Output:
Current l1(1/100000) = 11.0853215657
Current l1(10000/100000) = 0.126037403282
Current l1(20000/100000) = 0.096445475666
Current l1(30000/100000) = 0.0719392853473
Current l1(40000/100000) = 0.0690671103719
Current l1(50000/100000) = 0.07661241544
Current l1(60000/100000) = 0.0743827124406
Current l1(70000/100000) = 0.0656016587469
Current l1(80000/100000) = 0.0675546809828
Current l1(90000/100000) = 0.0649035400487
Current l1(100000/100000) = 0.0583308788607
Total l1 at test = 0.0613149096968
Total penalty may be enhanced using some other architecture, learning rate, batch size, count of epochs, loss function, e.t.c.
It looks that architecture may be increased, then you will be able to run training for a long period of time to get 1e-3.
More information about how it works and how to do it you can find at CS231 course.
P.S. here is some assumption about data feeding: some data on which I tested might was at the training process. Because task is simple it's OK, but it's better to guarantee, that not any train sample will be in test set.

This code does a lot better. Anyone want to make any further improvements?
import numpy as np
import tensorflow as tf
n = 10
k = 1000
n_hidden = 50
learning_rate = .001
training_epochs = 100000
Xtrain = []
Ytrain = []
Xtest = []
Ytest = []
for i in range(0,k,1):
X = np.random.randn(1,n)[0]
Xtrain += [X]
Ytrain += [Xtrain[-1]**2]
X = np.random.randn(1,n)[0]
Xtest += [X]
Ytest += [Xtest[-1]**2]
x = tf.placeholder(tf.float64,shape = (k,n))
y = tf.placeholder(tf.float64,shape = (k,n))
W1 = tf.Variable(tf.random_normal((n,n_hidden),dtype = tf.float64))
b1 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
x_hidden1 = tf.nn.sigmoid(tf.matmul(x,W1) + b1)
W2 = tf.Variable(tf.random_normal((n_hidden,n_hidden),dtype = tf.float64))
b2 = tf.Variable(tf.random_normal((n_hidden,),dtype = tf.float64))
x_hidden2 = tf.nn.sigmoid(tf.matmul(x_hidden1,W2) + b2)
W3 = tf.Variable(tf.random_normal((n_hidden,n),dtype = tf.float64))
b3 = tf.Variable(tf.random_normal((n,),dtype = tf.float64))
y_pred = tf.matmul(x_hidden2,W3) + b3
penalty = tf.reduce_sum((y - y_pred)**2)
train_op = tf.train.AdamOptimizer(learning_rate).minimize(penalty)
model = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(model)
for i in range(0,training_epochs):
sess.run(train_op,{x: Xtrain,y: Ytrain})
Ypred = y_pred.eval(feed_dict = {x: Xtest})

Tensorflow model not updating the variables

Problem Summary:
The issue is that even after running this code for multiple epochs, the cost isn't reducing much ( I have tried this for a variety of starting_learning_rates ). The equation that I am trying to optimize is ((m * pow(length, u) * pow(start_y, t) + c) where length and start_y are the inputs and u,t,m and c are learn-able parameters. I was able to observe (my dataset is quite small) that length * sqrt(start_y) is almost a constant and thought that tensorflow would be able to better help me find the value of the variables
This is my tensorflow code, combined_vehicles is an array with 129 rows and 2 columns( 2 features ), combined_labels is an array corresponding to labels for each of the examples in combined_vehicles
u = tf.Variable(0.0,dtype = "float32")
t = tf.Variable(0.0,dtype = "float32")
c = tf.Variable(0.0,dtype = "float32")
m = tf.Variable(0.0,dtype = "float32")
length = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name="length")
start_y = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name="start_y")
labels = tf.placeholder(dtype = "float32", shape = [combined_vehicles.shape[0],1], name = "labels")
output = tf.add(tf.multiply(tf.multiply(tf.pow(length, u), tf.pow(start_y, t)), m), c)
cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = output, labels = labels))
global_step = tf.Variable(0, trainable=False, name = 'global_step')
start_learning_rate = 0.0001
decay_steps = 100
learning_rate = tf.train.exponential_decay(start_learning_rate, global_step, decay_steps, 0.1, staircase=True )
result_output = output > 0.5
result_label = combined_labels > 0.5
correct_prediction = tf.equal( result_output, result_label )
accuracy = tf.reduce_mean( tf.cast( correct_prediction, "float" ) )
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost, global_step=global_step)
init = tf.global_variables_initializer()
with tf.Session() as sess:
epochs = 100
sess.run(init)
for i in range(epochs):
_,cost_estimate = sess.run([optimizer, cost], feed_dict = {length: combined_vehicles[:,0].reshape([combined_vehicles.shape[0],1]), start_y:combined_vehicles[:,1].reshape([combined_vehicles.shape[0],1]), labels: combined_labels})
total_accuracy = accuracy.eval({length: combined_vehicles[:,0].reshape([combined_vehicles.shape[0],1]), start_y:combined_vehicles[:,1].reshape([combined_vehicles.shape[0],1]), labels: combined_labels})

SummaryWriter not outputting graph in TensorFlow [duplicate]

This question already has answers here:
Save Tensorflow graph for viewing in Tensorboard without summary operations
(5 answers)
Closed 5 years ago.
I am trying to use tensorboard to analyse a graph in tensorflow with summaryWriter. However, TensorFlow is not outputting a 'graph' folder with information. Perhaps I am missing a command or it is not in the right place?
writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph());
Is what I used. I think this may not work for TensorFlow 1.0 anymore (just the summarywriter command)
import numpy as np
import tensorflow as tf
# %matplotlib inline
import matplotlib.pyplot as plt
# Global config variables
num_steps = 5 # number of truncated backprop steps ('n' in the discussion above)
batch_size = 200
num_classes = 2
state_size = 4
learning_rate = 0.1
logs_path = "./graph"
def gen_data(size=1000000):
X = np.array(np.random.choice(2, size=(size,)))
Y = []
for i in range(size):
threshold = 0.5
if X[i-3] == 1:
threshold += 0.5
if X[i-8] == 1:
threshold -= 0.25
if np.random.rand() > threshold:
Y.append(0)
else:
Y.append(1)
return X, np.array(Y)
# adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/reader.py
def gen_batch(raw_data, batch_size, num_steps):
raw_x, raw_y = raw_data
data_length = len(raw_x)
# partition raw data into batches and stack them vertically in a data matrix
batch_partition_length = data_length // batch_size
data_x = np.zeros([batch_size, batch_partition_length], dtype=np.int32)
data_y = np.zeros([batch_size, batch_partition_length], dtype=np.int32)
for i in range(batch_size):
data_x[i] = raw_x[batch_partition_length * i:batch_partition_length * (i + 1)]
data_y[i] = raw_y[batch_partition_length * i:batch_partition_length * (i + 1)]
# further divide batch partitions into num_steps for truncated backprop
epoch_size = batch_partition_length // num_steps
for i in range(epoch_size):
x = data_x[:, i * num_steps:(i + 1) * num_steps]
y = data_y[:, i * num_steps:(i + 1) * num_steps]
yield (x, y)
def gen_epochs(n, num_steps):
for i in range(n):
yield gen_batch(gen_data(), batch_size, num_steps)
"""
Placeholders
"""
x = tf.placeholder(tf.int32, [batch_size, num_steps], name='input_placeholder')
y = tf.placeholder(tf.int32, [batch_size, num_steps], name='labels_placeholder')
init_state = tf.zeros([batch_size, state_size])
"""
Inputs
"""
x_one_hot = tf.one_hot(x, num_classes)
rnn_inputs = tf.unstack(x_one_hot, axis=1)
"""
RNN
"""
cell = tf.contrib.rnn.BasicRNNCell(state_size)
rnn_outputs, final_state = tf.contrib.rnn.static_rnn(cell, rnn_inputs, initial_state=init_state)
"""
Predictions, loss, training step
"""
with tf.variable_scope('softmax'):
W = tf.get_variable('W', [state_size, num_classes])
b = tf.get_variable('b', [num_classes], initializer=tf.constant_initializer(0.0))
logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]
predictions = [tf.nn.softmax(logit) for logit in logits]
y_as_list = [tf.squeeze(i, axis=[1]) for i in tf.split(axis=1, num_or_size_splits=num_steps, value=y)]
loss_weights = [tf.ones([batch_size]) for i in range(num_steps)]
losses = tf.contrib.legacy_seq2seq.sequence_loss_by_example(logits, y_as_list, loss_weights)
tf.scalar_summary("losses", losses)
total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(learning_rate).minimize(total_loss)
# Not sure why this is not outputting a graph for tensorboard
writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph());
"""
Function to train the network
"""
def train_network(num_epochs, num_steps, state_size=4, verbose=True):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
training_losses = []
saved = gen_epochs(num_epochs, num_steps);
for idx, epoch in enumerate(gen_epochs(num_epochs, num_steps)):
training_loss = 0
training_state = np.zeros((batch_size, state_size))
if verbose:
print("\nEPOCH", idx)
for step, (X, Y) in enumerate(epoch):
tr_losses, training_loss_, training_state, _ = \
sess.run([losses,
total_loss,
final_state,
train_step],
feed_dict={x:X, y:Y, init_state:training_state})
training_loss += training_loss_
if step % 100 == 0 and step > 0:
if verbose:
print("Average loss at step", step,
"for last 250 steps:", training_loss/100)
training_losses.append(training_loss/100)
training_loss = 0
return training_losses
training_losses = train_network(1,num_steps)
plt.plot(training_losses)
# tensorboard --logdir="my_graph"

This worked for me:
writer = tf.summary.FileWriter(logdir='logdir', graph=tf.get_default_graph())
writer.flush()