The task is to compute f(2) + f(10) in tensorflow. One of the ways is
x = tf.placeholder(tf.float32)
f = x ** 2
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
a = sess.run(f, feed_dict={x: 2})
b = sess.run(f, feed_dict={x: 10})
c = a + b
print(c)
But a + b is Python operation, not tensorflow. The question is how to define that operation in tf? I can't understand how to define two nodes in computational grph which correspond to values of the same function in different points.
Since for f(2) + f(10), you need to feed two parameters, you'll have to define two placeholders as well:
# define two placeholders
a = tf.placeholder(tf.float32)
b = tf.placeholder(tf.float32)
def f(x):
return x ** 2
c = f(a) + f(b) # this is the tf operation
sess = tf.Session()
c = sess.run(c, feed_dict={a: 2, b: 10})
print(c)
# 104.0
Related
I have written 2 functions that initialize tensorflow's variables in different ways. I don't know why the results are different. Here is the first function using placeholder for initialization:
First function
import tensorflow as tf
import numpy as np
def linear_function():
np.random.seed(1)
X = tf.placeholder(dtype = tf.float64, name='X')
W = tf.placeholder(dtype = tf.float64, name='W')
b = tf.placeholder(dtype = tf.float64, name='b')
Y = tf.add(tf.matmul(W, X), b)
sess = tf.Session()
result = sess.run(Y, feed_dict={W:np.random.randn(4,3), X:np.random.randn(3,1), b:np.random.randn(4,1)})
sess.close()
return result
print( "result = " + str(linear_function()))
And the result is:
result = [[-1.98748544]
[-2.76826248]
[-0.78635415]
[-2.77463846]]
Second function
Second function uses tf.constant to initialize variables:
def linear_function():
np.random.seed(1)
X = tf.constant(np.random.randn(3,1), name ="X")
W = tf.constant(np.random.randn(4,3), name ="X")
b = tf.constant(np.random.randn(4,1), name ="X")
Y = tf.add(tf.matmul(W,X), b)
sess = tf.Session()
result = sess.run(Y)
sess.close()
return result
print( "result = " + str(linear_function()))
Result:
result = [[-2.15657382]
[ 2.95891446]
[-1.08926781]
[-0.84538042]]
What is the problem? Is it related to np.random.seed(1) ?
Thanks.
In the first snippet, the feed_dict is:
{W:np.random.randn(4,3), X:np.random.randn(3,1), b:np.random.randn(4,1)}
So first a random value for W is produced, then for X and then for b. However, in the second snippet the random values are given in the order X, W and b. Since the order in which the random numbers are generated is not the same, the values differ. If for example you change the order adequately in the feed_dict in the first snippet you will get the same result as the second one:
import tensorflow as tf
import numpy as np
def linear_function():
np.random.seed(1)
X = tf.placeholder(dtype = tf.float64, name='X')
W = tf.placeholder(dtype = tf.float64, name='W')
b = tf.placeholder(dtype = tf.float64, name='b')
Y = tf.add(tf.matmul(W, X), b)
sess = tf.Session()
result = sess.run(Y, feed_dict={X:np.random.randn(3,1), W:np.random.randn(4,3), b:np.random.randn(4,1)})
sess.close()
return result
print( "result = " + str(linear_function()))
Output:
result = [[-2.15657382]
[ 2.95891446]
[-1.08926781]
[-0.84538042]]
I try to build a model that will identify by data and try to see the LOSS function
loss =tf.reduce_mean(-(y_ * tf.log(y)+(1- y_)* tf.log (1-y)))
But as of now I only get NAN on the prediction and printing NAN in the LOSS function
np_labels = np.array(labels)
np_labels = np_labels.reshape([np_labels.shape[0], 1])
features = 910
hidden_layer_nodes = 100
x = tf.placeholder(tf.float32, [None, features])
y_ = tf.placeholder(tf.float32, [None, 1])
W1 = tf.Variable(tf.truncated_normal([features,hidden_layer_nodes], stddev=0.1))
b1 = tf.Variable(tf.constant(0.1, shape=[hidden_layer_nodes]))
z1 = tf.add(tf.matmul(x,W1),b1)
a1 = tf.nn.relu(z1)
W2 = tf.Variable(tf.truncated_normal([hidden_layer_nodes,1], stddev=0.1))
b2 = tf.Variable(0.)
z2 = tf.matmul(a1,W2) + b2
y = 1 / (1.0 + tf.exp(-z2))
loss =tf.reduce_mean(-(y_ * tf.log(y)+(1- y_)* tf.log (1-y)))
update = tf.train.AdamOptimizer(0.01).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in range(0,50):
sess.run(update, feed_dict = {x:fvecs, y_:np_labels})
print(sess.run(loss, feed_dict={x: fvecs, y_: np_labels}))
# sess.run(update, feed_dict = {x:data_x, y_:data_y})
# print(sess.run(loss, feed_dict={x: data_x, y_: data_y}))
print('prediction: ', y.eval(session=sess, feed_dict = {x:[[493.9, 702.6, .....
i want to print the loss
Thanks
This is not a TensorFlow-Issue. This results from the very bad idea of implementing the loss-function yourself.
import tensorflow as tf
z2 = tf.random_normal([8, 10]) * 20
y_ = tf.random_uniform([8, 1], minval=0, maxval=10, dtype=tf.float32)
y = 1 / (1.0 + tf.exp(-z2))
loss = tf.reduce_mean(-(y_ * tf.log(y)+(1- y_)* tf.log (1-y)))
with tf.Session() as sess:
print sess.run(loss) # will always fail with high prob
Will give Inf just because of missing the log-sum-exp trick which then causes your implementation to fail due to numerical instabilities (a folklore example which produces overflows). Just run this code several times and you get either NaN or Inf.
Solution would be:
replace y = tf.sigmoid(-z2) by y = tf.identity(z2) to just get the untransformed logits
replace loss = .. by loss = tf.nn.sigmoid_cross_entropy_with_logits(...) to use the numerical stable way
See the docs of sigmoid_cross_entropy_with_logits which explicitly describes this issue.
I wanna implement a function like this:if x == k, f(x) = 1, else f(x) = 0(k is a parameter). So I used tf.equal and tf.cast and my code was like this:
import tensorflow as tf
a = range(12)
a = tf.Variable(a)
b = 6
b = tf.Variable(b)
a = tf.reshape(a, [3, 4])
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
c = tf.equal(a, b)
d = tf.cast(c, tf.int32)
print sess.run(c)
print sess.run(d)
It seems fine, but the problem is tf.gradients(d, a) and tf.gradients(d, b) are None. I tried tf.gradients(c, a) and got TypedError. Are there any decent way to implement this function?
I'm not sure the gradient is even defined here.
The indicator function is f(a,b) = 1 if a=b, 0 otherwise. Away from a=b, this function is constant (zero) and so has zero derivative. At any point where a=b the function is discontinuous, so it doesn't have a derivative there.
More intuitively: derivatives don't exist where you have a 'jump' in your function.
It would be possible to have the PDF of the normal distribution to approximate the indicator function. I am also new to TensorFlow, so feel free to point out any issue.
##I am using tensorflow2
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
import tensorflow_probability as tfp
a = tf.range(12)
a = tf.Variable(a)
b = 6
b = tf.Variable(b)
a = tf.reshape(a, [3, 4])
## Define the PDF of a normal distribution to approximate the indicator function
dist = tfp.distributions.Normal(0., 0.1)
scalar = dist.prob(0) # a normalization constant
#since the pdf at data zero is not one
## Implement the approximazed indicator function
a = tf.cast(a, dtype= tf.float32)
b = tf.cast(b, dtype= tf.float32)
c = dist.prob(a-b)/scalar
#d = tf.cast(c, tf.int32)
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
print(sess.run(c))
## calcualte the gradient
c_a = tf.gradients(c, a)
print(sess.run(c_a))
This is Faster R-CNN implement in tensorflow.
The proposal_layer is implement by python
i am curious about if gradient can pass by tf.py_func
the weights and biases are keep changing
so I think the gradient deliver back successful
Then I do a small test
import tensorflow as tf
import numpy as np
def addone(x):
# print type(x)
return x + 1
def pyfunc_test():
# create data
x_data = tf.placeholder(dtype=tf.float32, shape=[None])
y_data = tf.placeholder(dtype=tf.float32, shape=[None])
w = tf.Variable(tf.constant([0.5]))
b = tf.Variable(tf.zeros([1]))
y1 = tf.mul(w, x_data, name='y1')
y2 = tf.py_func(addone, [y1], tf.float32)
y = tf.add(y2, b)
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in xrange(201):
ran = np.random.rand(115).astype(np.float32)
ans = ran * 1.5 + 3
dic = {x_data: ran, y_data: ans}
tt, yy, yy1= sess.run([train, y1, y2], feed_dict=dic)
if step % 20 == 0:
print 'step {}'.format(step)
print '{}, {}'.format(w.eval(), b.eval())
test = sess.run(y, feed_dict={x_data:[1]})
print 'test = {}'.format(test)
if __name__ == '__main__':
pyfunc_test()
Variable b keep changing, but w keep the value after initialize and never change
sess.run(tf.gradients(loss, b), feed_dict=dic) get value
sess.run(tf.gradients(loss, w), feed_dict=dic) get {TypeError}Fetch argument None has invalid type <type 'NoneType'>
I know some questions suggest use tf.RegisterGradient and gradient_override_map
but I can't find these in the faster rcnn repo(link on top of post)
am I do something wrong or missing something so that w is freeze
Gradient of py_func is None (just check ops.get_gradient_function(y2.op)). There's this gist by #harpone which shows how to use gradient override map for py_func.
Here's your example modified to use that recipe
import numpy as np
import tensorflow as tf
def addone(x):
# print(type(x)
return x + 1
def addone_grad(op, grad):
x = op.inputs[0]
return x
from tensorflow.python.framework import ops
import numpy as np
# Define custom py_func which takes also a grad op as argument:
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
# Need to generate a unique name to avoid duplicates:
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
tf.RegisterGradient(rnd_name)(grad) # see _MySquareGrad for grad example
g = tf.get_default_graph()
with g.gradient_override_map({"PyFunc": rnd_name}):
return tf.py_func(func, inp, Tout, stateful=stateful, name=name)
def pyfunc_test():
# create data
x_data = tf.placeholder(dtype=tf.float32, shape=[None])
y_data = tf.placeholder(dtype=tf.float32, shape=[None])
w = tf.Variable(tf.constant([0.5]))
b = tf.Variable(tf.zeros([1]))
y1 = tf.mul(w, x_data, name='y1')
y2 = py_func(addone, [y1], [tf.float32], grad=addone_grad)[0]
y = tf.add(y2, b)
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
print("Pyfunc grad", ops.get_gradient_function(y2.op))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in range(10):
# ran = np.random.rand(115).astype(np.float32)
ran = np.ones((115)).astype(np.float32)
ans = ran * 1.5 + 3
dic = {x_data: ran, y_data: ans}
tt, yy, yy1= sess.run([train, y1, y2], feed_dict=dic)
if step % 1 == 0:
print('step {}'.format(step))
print('{}, {}'.format(w.eval(), b.eval()))
test = sess.run(y, feed_dict={x_data:[1]})
print('test = {}'.format(test))
if __name__ == '__main__':
pyfunc_test()
I have following simple placeholders:
x = tf.placeholder(tf.float32, shape=[1])
y = tf.placeholder(tf.float32, shape=[1])
z = tf.placeholder(tf.float32, shape=[1])
There are two functions fn1 and fn2 defined as:
def fn1(a, b):
return tf.mul(a, b)
def fn2(a, b):
return tf.add(a, b)
Now I want to calculate result based on pred condition:
pred = tf.placeholder(tf.bool, shape=[1])
result = tf.cond(pred, fn1(x,y), fn2(y,z))
But it gives me an error saying fn1 and fn2 must be callable.
How can I write fn1 and fn2 so that they can receive parameters at runtime?
I want to call the following:
sess.run(result, feed_dict={x:1,y:2,z:3,pred:True})
You can pass parameters to the functions using lambda and the code is as bellows.
x = tf.placeholder(tf.float32)
y = tf.placeholder(tf.float32)
z = tf.placeholder(tf.float32)
def fn1(a, b):
return tf.mul(a, b)
def fn2(a, b):
return tf.add(a, b)
pred = tf.placeholder(tf.bool)
result = tf.cond(pred, lambda: fn1(x, y), lambda: fn2(y, z))
Then you can call it as bellowing:
with tf.Session() as sess:
print sess.run(result, feed_dict={x: 1, y: 2, z: 3, pred: True})
# The result is 2.0
The easiest would be to define your functions in the call:
result = tf.cond(pred, lambda: tf.mul(a, b), lambda: tf.add(a, b))