I have following simple placeholders:
x = tf.placeholder(tf.float32, shape=[1])
y = tf.placeholder(tf.float32, shape=[1])
z = tf.placeholder(tf.float32, shape=[1])
There are two functions fn1 and fn2 defined as:
def fn1(a, b):
return tf.mul(a, b)
def fn2(a, b):
return tf.add(a, b)
Now I want to calculate result based on pred condition:
pred = tf.placeholder(tf.bool, shape=[1])
result = tf.cond(pred, fn1(x,y), fn2(y,z))
But it gives me an error saying fn1 and fn2 must be callable.
How can I write fn1 and fn2 so that they can receive parameters at runtime?
I want to call the following:
sess.run(result, feed_dict={x:1,y:2,z:3,pred:True})
You can pass parameters to the functions using lambda and the code is as bellows.
x = tf.placeholder(tf.float32)
y = tf.placeholder(tf.float32)
z = tf.placeholder(tf.float32)
def fn1(a, b):
return tf.mul(a, b)
def fn2(a, b):
return tf.add(a, b)
pred = tf.placeholder(tf.bool)
result = tf.cond(pred, lambda: fn1(x, y), lambda: fn2(y, z))
Then you can call it as bellowing:
with tf.Session() as sess:
print sess.run(result, feed_dict={x: 1, y: 2, z: 3, pred: True})
# The result is 2.0
The easiest would be to define your functions in the call:
result = tf.cond(pred, lambda: tf.mul(a, b), lambda: tf.add(a, b))
Related
Suppose I have the following simple example of a function of several variables
#tf.function
def f(A, Y, X):
AX = tf.matmul(A, X)
norm = tf.norm(Y - AX)
return norm
N = 2
A = tf.Variable(np.array([[1, 2], [3, 4]]))
Y = tf.Variable(np.identity(N))
X = tf.Variable(np.zeros((N, N)))
How do I find X that minimizes f with Tensorflow ?
I would be interested in a generic solution that works with a function declared as above and when there are more than one variable to optimize.
avanwyk is essentially right, although note that: 1) you can directly use the minimize method of the optimizer for simplicity 2) if you only want to optimize X you should make sure that is the only variable you are updating.
import tensorflow as tf
#tf.function
def f(A, Y, X):
AX = tf.matmul(A, X)
norm = tf.norm(Y - AX)
return norm
# Input data
N = 2
A = tf.Variable([[1., 2.], [3., 4.]], tf.float32)
Y = tf.Variable(tf.eye(N, dtype=tf.float32))
X = tf.Variable(tf.zeros((N, N), tf.float32))
# Initial function value
print(f(A, Y, X).numpy())
# 1.4142135
# Setup a stochastic gradient descent optimizer
opt = tf.keras.optimizers.SGD(learning_rate=0.01)
# Define loss function and variables to optimize
loss_fn = lambda: f(A, Y, X)
var_list = [X]
# Optimize for a fixed number of steps
for _ in range(1000):
opt.minimize(loss_fn, var_list)
# Optimized function value
print(f(A, Y, X).numpy())
# 0.14933111
# Optimized variable
print(X.numpy())
# [[-2.0012102 0.98504114]
# [ 1.4754106 -0.5111093 ]]
Assuming Tensorflow 2, you can use a Keras optimizer:
#tf.function
def f(A, Y, X):
AX = tf.matmul(A, X)
norm = tf.norm(Y - AX)
return norm
N = 2
A = tf.Variable(np.array([[1., 2.], [3., 4.]]))
Y = tf.Variable(np.identity(N))
X = tf.Variable(np.zeros((N, N)))
optimizer = tf.keras.optimizers.SGD()
for iteration in range(0, 100):
with tf.GradientTape() as tape:
loss = f(X, Y, X)
print(loss)
grads = tape.gradient(loss, [A, Y, X])
optimizer.apply_gradients(zip(grads, [A, Y, X]))
print(A, Y, X)
That will work for any differentiable function. For non-differentiable functions you could look at other optimization techniques (such as Genetic Algorithms, or Swarm Optimization. NEAT has implementations of these https://neat-python.readthedocs.io/en/latest/).
I have written 2 functions that initialize tensorflow's variables in different ways. I don't know why the results are different. Here is the first function using placeholder for initialization:
First function
import tensorflow as tf
import numpy as np
def linear_function():
np.random.seed(1)
X = tf.placeholder(dtype = tf.float64, name='X')
W = tf.placeholder(dtype = tf.float64, name='W')
b = tf.placeholder(dtype = tf.float64, name='b')
Y = tf.add(tf.matmul(W, X), b)
sess = tf.Session()
result = sess.run(Y, feed_dict={W:np.random.randn(4,3), X:np.random.randn(3,1), b:np.random.randn(4,1)})
sess.close()
return result
print( "result = " + str(linear_function()))
And the result is:
result = [[-1.98748544]
[-2.76826248]
[-0.78635415]
[-2.77463846]]
Second function
Second function uses tf.constant to initialize variables:
def linear_function():
np.random.seed(1)
X = tf.constant(np.random.randn(3,1), name ="X")
W = tf.constant(np.random.randn(4,3), name ="X")
b = tf.constant(np.random.randn(4,1), name ="X")
Y = tf.add(tf.matmul(W,X), b)
sess = tf.Session()
result = sess.run(Y)
sess.close()
return result
print( "result = " + str(linear_function()))
Result:
result = [[-2.15657382]
[ 2.95891446]
[-1.08926781]
[-0.84538042]]
What is the problem? Is it related to np.random.seed(1) ?
Thanks.
In the first snippet, the feed_dict is:
{W:np.random.randn(4,3), X:np.random.randn(3,1), b:np.random.randn(4,1)}
So first a random value for W is produced, then for X and then for b. However, in the second snippet the random values are given in the order X, W and b. Since the order in which the random numbers are generated is not the same, the values differ. If for example you change the order adequately in the feed_dict in the first snippet you will get the same result as the second one:
import tensorflow as tf
import numpy as np
def linear_function():
np.random.seed(1)
X = tf.placeholder(dtype = tf.float64, name='X')
W = tf.placeholder(dtype = tf.float64, name='W')
b = tf.placeholder(dtype = tf.float64, name='b')
Y = tf.add(tf.matmul(W, X), b)
sess = tf.Session()
result = sess.run(Y, feed_dict={X:np.random.randn(3,1), W:np.random.randn(4,3), b:np.random.randn(4,1)})
sess.close()
return result
print( "result = " + str(linear_function()))
Output:
result = [[-2.15657382]
[ 2.95891446]
[-1.08926781]
[-0.84538042]]
The task is to compute f(2) + f(10) in tensorflow. One of the ways is
x = tf.placeholder(tf.float32)
f = x ** 2
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
a = sess.run(f, feed_dict={x: 2})
b = sess.run(f, feed_dict={x: 10})
c = a + b
print(c)
But a + b is Python operation, not tensorflow. The question is how to define that operation in tf? I can't understand how to define two nodes in computational grph which correspond to values of the same function in different points.
Since for f(2) + f(10), you need to feed two parameters, you'll have to define two placeholders as well:
# define two placeholders
a = tf.placeholder(tf.float32)
b = tf.placeholder(tf.float32)
def f(x):
return x ** 2
c = f(a) + f(b) # this is the tf operation
sess = tf.Session()
c = sess.run(c, feed_dict={a: 2, b: 10})
print(c)
# 104.0
This is Faster R-CNN implement in tensorflow.
The proposal_layer is implement by python
i am curious about if gradient can pass by tf.py_func
the weights and biases are keep changing
so I think the gradient deliver back successful
Then I do a small test
import tensorflow as tf
import numpy as np
def addone(x):
# print type(x)
return x + 1
def pyfunc_test():
# create data
x_data = tf.placeholder(dtype=tf.float32, shape=[None])
y_data = tf.placeholder(dtype=tf.float32, shape=[None])
w = tf.Variable(tf.constant([0.5]))
b = tf.Variable(tf.zeros([1]))
y1 = tf.mul(w, x_data, name='y1')
y2 = tf.py_func(addone, [y1], tf.float32)
y = tf.add(y2, b)
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.5)
train = optimizer.minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in xrange(201):
ran = np.random.rand(115).astype(np.float32)
ans = ran * 1.5 + 3
dic = {x_data: ran, y_data: ans}
tt, yy, yy1= sess.run([train, y1, y2], feed_dict=dic)
if step % 20 == 0:
print 'step {}'.format(step)
print '{}, {}'.format(w.eval(), b.eval())
test = sess.run(y, feed_dict={x_data:[1]})
print 'test = {}'.format(test)
if __name__ == '__main__':
pyfunc_test()
Variable b keep changing, but w keep the value after initialize and never change
sess.run(tf.gradients(loss, b), feed_dict=dic) get value
sess.run(tf.gradients(loss, w), feed_dict=dic) get {TypeError}Fetch argument None has invalid type <type 'NoneType'>
I know some questions suggest use tf.RegisterGradient and gradient_override_map
but I can't find these in the faster rcnn repo(link on top of post)
am I do something wrong or missing something so that w is freeze
Gradient of py_func is None (just check ops.get_gradient_function(y2.op)). There's this gist by #harpone which shows how to use gradient override map for py_func.
Here's your example modified to use that recipe
import numpy as np
import tensorflow as tf
def addone(x):
# print(type(x)
return x + 1
def addone_grad(op, grad):
x = op.inputs[0]
return x
from tensorflow.python.framework import ops
import numpy as np
# Define custom py_func which takes also a grad op as argument:
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
# Need to generate a unique name to avoid duplicates:
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
tf.RegisterGradient(rnd_name)(grad) # see _MySquareGrad for grad example
g = tf.get_default_graph()
with g.gradient_override_map({"PyFunc": rnd_name}):
return tf.py_func(func, inp, Tout, stateful=stateful, name=name)
def pyfunc_test():
# create data
x_data = tf.placeholder(dtype=tf.float32, shape=[None])
y_data = tf.placeholder(dtype=tf.float32, shape=[None])
w = tf.Variable(tf.constant([0.5]))
b = tf.Variable(tf.zeros([1]))
y1 = tf.mul(w, x_data, name='y1')
y2 = py_func(addone, [y1], [tf.float32], grad=addone_grad)[0]
y = tf.add(y2, b)
loss = tf.reduce_mean(tf.square(y - y_data))
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
print("Pyfunc grad", ops.get_gradient_function(y2.op))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for step in range(10):
# ran = np.random.rand(115).astype(np.float32)
ran = np.ones((115)).astype(np.float32)
ans = ran * 1.5 + 3
dic = {x_data: ran, y_data: ans}
tt, yy, yy1= sess.run([train, y1, y2], feed_dict=dic)
if step % 1 == 0:
print('step {}'.format(step))
print('{}, {}'.format(w.eval(), b.eval()))
test = sess.run(y, feed_dict={x_data:[1]})
print('test = {}'.format(test))
if __name__ == '__main__':
pyfunc_test()
I get the error:
TypeError: Input 'ref' of 'AssignAdd' Op requires l-value input
on the line apply_gradient_op = opt.apply_gradients(grads, global_step=stepNum) of the function train below.
def x1_x2_diff_net_v0():
x = tf.placeholder(tf.float32, [None, 4])
lb = tf.placeholder(tf.float32, [None, 2])
#First fc layer
with tf.variable_scope('fc1') as scope:
w = tfu.get_weights([4,100], name='fc1_w')
b = tfu.get_bias([1,100], name='fc1_b')
fc1 = tf.nn.relu(tf.matmul(x, w) + b)
#Prediction layer
with tf.variable_scope('pred') as scope:
w = tfu.get_weights([100,2], name='pred_w')
b = tfu.get_bias([1, 2], name='pred_b')
pred = tf.nn.relu(tf.matmul(fc1, w) + b)
#Define the loss
loss = tf.nn.l2_loss(pred - lb, name='loss')
return loss
def train(stepNum, initLr=0.01):
g = tf.Graph()
with g.as_default():
loss = x1_x2_diff_net_v0()
lr = tf.train.exponential_decay(initLr, stepNum, 100,
0.1, staircase=True)
for tv in tf.trainable_variables():
print (tv.name)
# Compute gradients.
opt = tf.train.GradientDescentOptimizer(lr)
grads = opt.compute_gradients(loss)
# Apply gradients.
apply_gradient_op = opt.apply_gradients(grads, global_step=stepNum)
Any pointers on what might be going wrong? I took snippets of code from te method train in cifar10.py example file.
Oops! I was passing an integer into stepNum instead of tf.Variable. Its resolved now. It would be great if the error messages would be more intuitive.