I need to convert a KerasTensor to a Tensor because when I try to use a contional (tf.cond()) it reports an error:
def custon_loss(self, input_tensor): # input type = <class 'tensorflow.python.keras.engine.keras_tensor.KerasTensor'>
def loss(y_actual, y_predicted):
mse = K.mean(K.sum(K.square(y_actual - y_predicted)))
mse = tf.reshape(mse, [1, 1])
y_actual = keras.layers.core.Reshape([1, 1])(y_actual)[0]
ax_input = tf.reshape(input_tensor[0][-1:][0][:1], [1, 1])
# convert here ax_input to Tensor
greater_equal = tf.reshape(tf.math.logical_and(tf.math.greater_equal(ax_input, y_actual), tf.math.greater_equal(ax_input, y_predicted))[0], [1, 1])
less_equal = tf.reshape(tf.math.logical_and(tf.math.less_equal(ax_input, y_actual), tf.math.less_equal(ax_input, y_predicted))[0], [1, 1])
logical_or = tf.reshape(tf.math.logical_or(greater_equal, less_equal)[0], [1, 1])
return tf.cond(logical_or, lambda: mse, lambda: tf.math.multiply(mse, 10))
return loss
Error caused in tf.cond:
TypeError: Cannot convert a symbolic Keras input/output to a numpy array. This error may indicate that you're trying to pass a symbolic value to a NumPy call, which is not supported. Or, you may be trying to pass Keras symbolic inputs/outputs to a TF API that does not register dispatching, preventing Keras from automatically converting the API call to a lambda layer in the Functional Model.
I believe that converting the tensor will not make the error.
Looks like Issue is with numpy =1.20 version. Downgrade your numpy version to 1.19.5.
W.R.T Keras tensor you can use below sample code
import tensorflow as tf
import numpy as np
np_var = np.array([1])
keras_var = tf.keras.backend.variable(np_var)
def f1(): return tf.multiply(np_var, 17)
def f2(): return tf.add(np_var, 23)
r = tf.cond(tf.less(np_var, np_var), f1, f2)
Related
I have a custom tensorflow layer which works fine by generating an output but it throws an error when used with the Keras functional model API. Here is the code:
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input
# ------ Custom Layer -----------
class CustomLayer(tf.keras.layers.Layer):
def __init__(self):
super(CustomLayer, self).__init__()
def split_heads(self, x):
batch_size = x.shape[0]
split_inputs = tf.reshape(x, (batch_size, -1, 3, 1))
return split_inputs
def call(self, q):
qs = self.split_heads(q)
return qs
# ------ Testing Layer with sample data --------
x = np.random.rand(1,2,3)
values_emb = CustomLayer()(x)
print(values_emb)
This generates the following output:
tf.Tensor(
[[[[0.7148978 ]
[0.3997009 ]
[0.11451813]]
[[0.69927174]
[0.71329576]
[0.6588452 ]]]], shape=(1, 2, 3, 1), dtype=float32)
But when I use it in the Keras functional API it doesn't work. Here is the code:
x = Input(shape=(2,3))
values_emb = CustomLayer()(x)
model = Model(x, values_emb)
model.summary()
It gives this error:
TypeError: Failed to convert elements of (None, -1, 3, 1) to Tensor. Consider casting elements to a supported type. See https://www.tensorflow.org/api_docs/python/tf/dtypes for supported TF dtypes.
Does anyone know why this happens and how it can be fixed?
I think you should maybe try using tf.shape in your custom layer, since it will give you the dynamic shape of a tensor:
batch_size = tf.shape(x)[0]
When a tensorflow model contains tf.function decorated function with for loop in it, the tf->onnx conversion yields warnings:
WARNING:tensorflow:From /Users/amit/Programs/lammps/kim/kliff/venv/lib/python3.7/site-packages/tf2onnx/tf_loader.py:706: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.
Instructions for updating:
Use `tf.compat.v1.graph_util.extract_sub_graph`
Cannot infer shape for model/ex_layer/PartitionedCall/while: model/ex_layer/PartitionedCall/while:3
Cannot infer shape for model/ex_layer/PartitionedCall/Identity: model/ex_layer/PartitionedCall/Identity:0
Cannot infer shape for Func/model/ex_layer/PartitionedCall/output/_3: Func/model/ex_layer/PartitionedCall/output/_3:0
Cannot infer shape for Identity: Identity:0
missing output shape for while/Identity_3:0
missing output shape for while/Identity_3:0
missing output shape for while/Identity_3:0
missing output shape for while/Identity_3:0
...
And as the obtained model is run through onnxruntime it runs fine, but model checker gives the following error
Traceback (most recent call last):
File "failed_example.py", line 85, in <module>
onnx.checker.check_model(onnx.load("tmp.onnx"))
File "venv/lib/python3.7/site-packages/onnx/checker.py", line 106, in check_model
C.check_model(protobuf_string)
onnx.onnx_cpp2py_export.checker.ValidationError: Field 'shape' of type is required but missing.
Netron does not show any appreciable difference between the model with decorated function and without decorated function. I guess error comes from the fact that the for loop is converted to separate while-loop graph, whose input shape is not defined. But it does work perfectly without tf.function decorator. I am putting a minimal replication code below.
I think it is related to following issues:
https://github.com/onnx/onnx/issues/2932
https://github.com/onnx/onnx/issues/2492
https://github.com/onnx/onnx/pull/2937
Code to replicate:
import tensorflow as tf
import numpy as np
import sys
import onnx
import onnxruntime
import tf2onnx
# =============================================================================
# Layer and its herler functions
# COMMENT IT OUT TO PASS ONNX CHECK
#tf.function(
input_signature=[
tf.TensorSpec(shape=[None,None], dtype=tf.int32),
tf.TensorSpec(shape=[None,None], dtype=tf.float32),
tf.TensorSpec(shape=None, dtype=tf.float32),
])
def extra_function(
list1,
list2,
accum_var
):
some_num = 4
num_iter = tf.size(list1)//some_num
for i in range(num_iter):
xyz_i = list2[0, i * 3 : (i + 1) * 3]
accum_var += tf.reduce_sum(xyz_i)
return accum_var
class ExLayer(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
# Doesnt tf.function also create graphs out of called functions?
# however it does not seem to do that if `call` function is decorated
# #tf.function(
# input_signature=[
# tf.TensorSpec(shape=[None,None], dtype=tf.float32),
# tf.TensorSpec(shape=[None,None], dtype=tf.int32),
# ])
def call(self, list2,list1):
accum_var = tf.constant(0.0)
accum_var = extra_function( list1, list2, accum_var)
return accum_var
# =============================================================================
# =============================================================================
# Example implementation
layer1 = tf.keras.layers.Input(shape=(1,))
layer2 = tf.keras.layers.Input(shape=(1,), dtype=tf.int32)
EL = ExLayer()(layer1,layer2)
model = tf.keras.models.Model(inputs=[layer1, layer2], outputs=EL)
# Define input data
list2_tf = tf.constant([[0.,0.,0.,1.,1.,1.,2.,2.,2.,3.,3.,3.]],dtype=tf.float32)
list1_tf = tf.constant([[0,1,2,-1,1,0,2,-1,2,0,1,-1]],dtype=tf.int32)
list2_np = np.array([[0.,0.,0.,1.,1.,1.,2.,2.,2.,3.,3.,3.]],dtype=np.float32)
list1_np = np.array([[0,1,2,-1,1,0,2,-1,2,0,1,-1]],dtype=np.int32)
# Save to onnx
model_proto, external_tensor_storage = tf2onnx.convert.from_keras(model,
input_signature=[
tf.TensorSpec(shape=[None,None], dtype=tf.float32, name="list2"),
tf.TensorSpec(shape=[None,None], dtype=tf.int32, name="list1")
],
opset=11,
output_path="tmp.onnx")
# Load onnx runtime session
ort_session = onnxruntime.InferenceSession("tmp.onnx")
inputs = {"list2":list2_np, "list1":list1_np}
print("===================================================")
print("Original model evaluation:")
print(model([list2_tf,list1_tf]))
print("ORT session evaluation")
print(ort_session.run(None, inputs))
print("===================================================")
# Check with model checker
onnx.checker.check_model(onnx.load("tmp.onnx"))
ONNX version: 1.10.2
Python version: 3.7.7
TF version: 2.7.0
Related github issues I submitted:
https://github.com/onnx/onnx/issues/3909
https://github.com/onnx/tensorflow-onnx/issues/1812
The problem is in the way you specified the shape of accumm_var.
In the input signature you have tf.TensorSpec(shape=None, dtype=tf.float32). Reading the code I see that you are passing a scalar tensor. A scalar tensor is a 0-Dimension tensor, so you should use shape=[] instead of shape=None.
I run here without warnings after annotating extra_function with
tf.function(
input_signature=[
tf.TensorSpec(shape=[None,None], dtype=tf.int32),
tf.TensorSpec(shape=[None,None], dtype=tf.float32),
tf.TensorSpec(shape=[], dtype=tf.float32),
])
I am trying to create a model graph where my input is tensorflow variable which I am inputting from my java program
In my code, I am using numpy methods where I need to convert my tensorflow variable input to numpy array input
Here, is my code snippet
import tensorflow as tf
import numpy as np
eps = np.finfo(float).eps
EXPORT_DIR = './model'
def standardize(x):
med0 = np.median(x)
mad0 = np.median(np.abs(x - med0))
x1 = (x - med0) / (mad0 + eps)
return x1
#tensorflow input variable
a = tf.placeholder(tf.float32, name="input")
with tf.Session() as session:
session.run(tf.global_variables_initializer())
#Converting the input variable to numpy array
tensor = a.eval()
#calling standardize method
numpyArray = standardize(tensor)
#converting numpy array to tf
tf.convert_to_tensor(numpyArray)
#creating graph
graph = tf.get_default_graph()
tf.train.write_graph(graph, EXPORT_DIR, 'model_graph.pb', as_text=False)
I am getting error: InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'input' with dtype float in line tensor = a.eval()
When I am giving constant value in place of placeholder then it's working and generating the graph. But I want to input from my java code.
Is there any way to do that or do I need to convert all my numpy methods to tensorflow methods
placeholder is just an empty variable in tensorflow, to which you can feed numpy values. Now, what you are trying to do does not make sense. You can not get value out of an empty variable.
If you want to standardize your tensor, why convert it to numpy var first? You can directly do this using tensorflow.
The following taken from this stackoverflow ans
def get_median(v):
v = tf.reshape(v, [-1])
m = v.get_shape()[0]//2
return tf.nn.top_k(v, m).values[m-1]
Now, you can implement your function as
def standardize(x):
med0 = get_median(x)
mad0 = get_median(tf.abs(x - med0))
x1 = (x - med0)/(mad0 + eps)
return x1
i'm trying to use the tensorflow unique function (https://www.tensorflow.org/api_docs/python/tf/unique) in a keras lambda layer.
Code below:
def unique_idx(x):
output = tf.unique(x)
return output[1]
then
inp1 = Input(batch_shape(None, 1))
idx = Lambda(unique_idx)(inp1)
model = Model(inputs=inp1, outputs=idx)
when I now use **model.compile(optimizer='Adam', loss='mean_squared_error')**
I get the error:
ValueError: Tensor conversion requested dtype int32 for Tensor with
dtype float32: 'Tensor("lambda_9_sample_weights_1:0", shape=(?,),
dtype=float32)'
Does anybody know whats the error here or a different way of using the tensorflow function?
A keras model expects a float32 as output, but the indices returned from tf.unique is a int32. A casting fixes your problem.
Another issue is that unique expects a flatten array. reshape fixes this one.
import tensorflow as tf
from keras import Input
from keras.layers import Lambda
from keras.engine import Model
def unique_idx(x):
x = tf.reshape(x, [-1])
u, indices = tf.unique(x)
return tf.cast(indices, tf.float32)
x = Input(shape=(1,))
y = Lambda(unique_idx)(x)
model = Model(inputs=x, outputs=y)
model.compile(optimizer='adam', loss='mse')
I am trying to build a neural network using custom activation functions. I followed the solution given here, and it works when the input and output vectors have the same size, but not when using different sizes (like in a pooling function). Here is my problem so far:
I am trying to generalize this to the case when the input and the output have different sizes. In my code the input 'x' is of size (2,4), the output 'y' is of size (1,2), and the activation function MEX(.) does the mapping y = MEX(x). I have computed the gradient of MEX() as d_MEX(), where d_MEX(x) has the same size as 'x', that is (2,4). Nevertheless, I get this error
InvalidArgumentError (see above for traceback): Incompatible shapes: [1,2] vs. [2,4]
Shouldn't the gradient of MEX(x) be of the same size as x? Here is my complete code:
import tensorflow as tf
import numpy as np
# This is our target function
def MEX(x):
'''
:param x: is a row vector which is the concatenation of [input, beta]
:return MEX_{beta}(x): scalar output
'''
# lenx = np.size(x) # Number of columns (ROW vector)
lenx = x.shape[1]
N = x.shape[0]
out = np.zeros((1,N))
for ii in range(N):
c = x[ii,0:lenx-1]
beta = x[ii,lenx-1]
out[0,ii] = 1./beta * np.log( np.mean( np.exp(beta*c) ))
return np.array(out)
# Now we should write its derivative.
def d_MEX(x):
# lenx = np.size(x) # Number of
lenx = x.shape[1]
N = x.shape[0]
out = np.zeros((N,lenx))
for ii in range(N):
c = x[ii,0:lenx-1]
beta = x[ii,lenx-1]
d_beta = np.array([0.])
d_beta[0] = -1./beta*( MEX(np.array([x[ii,:]])) - np.mean( np.multiply( c, np.exp(beta*c)))/np.mean( np.exp(beta*c)) )
d_c = 1./lenx*np.exp(beta*c) /np.mean( np.exp(beta*c))
out[ii,:] = np.concatenate((d_c,d_beta), axis=0)
return out
# The first step is making it into a numpy function, this is easy:
np_MEX = np.vectorize(MEX, excluded=['x']) # IMPORTANT!! Otherwise np.vectorize() doesnt work
np_d_MEX = np.vectorize(d_MEX, excluded=['x']) # IMPORTANT!! Otherwise np.vectorize() doesnt work
# Now we make a tensforflow function
'''
Making a numpy fct to a tensorflow fct: We will start by making np_d_MEX_32 into a tensorflow function.
There is a function in tensorflow tf.py_func(func, inp, Tout, stateful=stateful, name=name) [doc]
which transforms any numpy function to a tensorflow function, so we can use it:
'''
np_d_MEX_32 = lambda x: np_d_MEX(x=x).astype(np.float32)
def tf_d_MEX(x,name=None):
with tf.name_scope(name, "d_MEX", [x]) as name:
y = tf.py_func(np_d_MEX_32,
[x],
[tf.float32],
name=name,
stateful=False)
return y[0]
'''
tf.py_func acts on lists of tensors (and returns a list of tensors), that is why we have [x] (and return y[0]).
The stateful option is to tell tensorflow whether the function always gives the same output for the same input (stateful = False)
in which case tensorflow can simply the tensorflow graph, this is our case and will probably be the case in most situations.
One thing to be careful of at this point is that numpy used float64 but tensorflow uses float32 so you need to convert
your function to use float32 before you can convert it to a tensorflow function otherwise tensorflow will complain.
This is why we need to make np_d_MEX_32 first.
What about the Gradients? The problem with only doing the above is that even though we now have tf_d_MEX which is the
tensorflow version of np_d_MEX, we couldn't use it as an activation function if we wanted to because tensorflow doesn't
know how to calculate the gradients of that function.
Hack to get Gradients: As explained in the sources mentioned above, there is a hack to define gradients of a function
using tf.RegisterGradient [doc] and tf.Graph.gradient_override_map [doc]. Copying the code from harpone we can modify
the tf.py_func function to make it define the gradient at the same time:
'''
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
# Need to generate a unique name to avoid duplicates:
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
tf.RegisterGradient(rnd_name)(grad) # see _MySquareGrad for grad example
g = tf.get_default_graph()
with g.gradient_override_map({"PyFunc": rnd_name}):
return tf.py_func(func, inp, Tout, stateful=stateful, name=name)
'''
Now we are almost done, the only thing is that the grad function we need to pass to the above py_func function needs to
take a special form. It needs to take in an operation, and the previous gradients before the operation and propagate
the gradients backward after the operation.
Gradient Function: So for our MEX activation function that is how we would do it:
'''
def MEXgrad(op, grad):
x = op.inputs[0]
# x = op
n_gr = tf_d_MEX(x)
return grad * n_gr
'''
The activation function has only one input, that is why x = op.inputs[0]. If the operation had many inputs, we would
need to return a tuple, one gradient for each input. For example if the operation was a-bthe gradient with respect to a
is +1 and with respect to b is -1 so we would have return +1*grad,-1*grad. Notice that we need to return tensorflow
functions of the input, that is why need tf_d_MEX, np_d_MEX would not have worked because it cannot act on
tensorflow tensors. Alternatively we could have written the derivative using tensorflow functions:
'''
# Combining it all together: Now that we have all the pieces, we can combine them all together:
np_MEX_32 = lambda x: np_MEX(x=x).astype(np.float32)
def tf_MEX(x, name=None):
with tf.name_scope(name, "MEX",[x]) as name:
y = py_func(np_MEX_32,
[x],
[tf.float32],
name=name,
grad=MEXgrad) # <-- here's the call to the gradient
return y[0]
with tf.Session() as sess:
x = tf.constant([[0.2,0.7,1.2,1.7],[0.2,0.7,1.2,1.7]])
y = tf_MEX(x)
tf.global_variables_initializer().run()
print(x.eval(), y.eval(), tf.gradients(y, [x])[0].eval())
In the console, I have checked that the variables have the "correct" shapes:
x.eval()
Out[9]:
array([[ 0.2 , 0.69999999, 1.20000005, 1.70000005],
[ 0.2 , 0.69999999, 1.20000005, 1.70000005]], dtype=float32)
y.eval()
Out[10]: array([[ 0.83393127, 0.83393127]], dtype=float32)
tf_d_MEX(x).eval()
Out[11]:
array([[ 0.0850958 , 0.19909413, 0.46581003, 0.07051659],
[ 0.0850958 , 0.19909413, 0.46581003, 0.07051659]], dtype=float32)
My bad, I just found the mistake.
Its here:
def MEXgrad(op, grad):
x = op.inputs[0]
# x = op
n_gr = tf_d_MEX(x)
return n_gr
I wonder if there is a typo here, where this mistake is also there.