Develop Reference

Tensorflow: Shape must be at least rank 3 but is rank 2

I am training a bilstm model for text classification and running into the following error
ValueError: Shape must be at least rank 3 but is rank 2 for '{{node
bidirectional_5/forward_lstm_6/lstm_cell_17/BiasAdd}} =
BiasAdd[T=DT_HALF,
data_format="NCHW"](bidirectional_5/forward_lstm_6/lstm_cell_17/MatMul,
bidirectional_5/forward_lstm_6/lstm_cell_17/split_1)' with input
shapes: [?,200], [200].
This is the code I am using
output_dim = 200
dropout = 0.2
recurrent_dropout = 0.2
lstm_units = 128
n_tags = 26
num_dense_neurons = 20
classifier_activation = time_activation = 'tanh'
learning_rate = 0.003
merge_mode = 'concat'
activation = 'softmax'
max_length = 20
model = Sequential()
# Add Embedding layer
e = Embedding(input_dim= 340862, output_dim=output_dim,
weights=[weight_matrix],
input_length=max_length, trainable=False)
model.add(e)
# Add bidirectional LSTM
b = Bidirectional(LSTM(units=output_dim, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout), merge_mode = merge_mode)
model.add(b)
# Add LSTM
model.add(LSTM(units=lstm_units, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
# Add timeDistributed Layer
model.add(TimeDistributed(Dense(n_tags, activation=time_activation)))
model.add(Flatten())
model.add(Dense(num_dense_neurons, activation=classifier_activation))
model.add(Dense(n_tags,activation='softmax'))
#Optimiser
adam = Adam(learning_rate=learning_rate, beta_1=0.9, beta_2=0.999)
# Compile model
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[f1_score])
model.summary()
The weight_matrix I am passing to the embedding layer is of shape (340862, 200)
Both keras and tensorflow versions are 2.6.0. I am running this on AWS Sagemaker. Not sure why this is happening.
Full error
InvalidArgumentError Traceback (most recent call last)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs, op_def)
1879 try:
-> 1880 c_op = pywrap_tf_session.TF_FinishOperation(op_desc)
1881 except errors.InvalidArgumentError as e:
InvalidArgumentError: Shape must be at least rank 3 but is rank 2 for '{{node bidirectional_6/forward_lstm_7/lstm_cell_20/BiasAdd}} = BiasAdd[T=DT_HALF, data_format="NCHW"](bidirectional_6/forward_lstm_7/lstm_cell_20/MatMul, bidirectional_6/forward_lstm_7/lstm_cell_20/split_1)' with input shapes: [?,200], [200].
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
<ipython-input-88-ddcbc617161c> in <module>
20 # Add bidirectional LSTM
21 b = Bidirectional(LSTM(units=output_dim, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout), merge_mode = merge_mode)
---> 22 model.add(b)
23 # Add LSTM
24 model.add(LSTM(units=lstm_units, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
528 self._self_setattr_tracking = False # pylint: disable=protected-access
529 try:
--> 530 result = method(self, *args, **kwargs)
531 finally:
532 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/sequential.py in add(self, layer)
215 # If the model is being built continuously on top of an input layer:
216 # refresh its output.
--> 217 output_tensor = layer(self.outputs[0])
218 if len(tf.nest.flatten(output_tensor)) != 1:
219 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/wrappers.py in __call__(self, inputs, initial_state, constants, **kwargs)
581
582 if initial_state is None and constants is None:
--> 583 return super(Bidirectional, self).__call__(inputs, **kwargs)
584
585 # Applies the same workaround as in `RNN.__call__`
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
975 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
976 return self._functional_construction_call(inputs, args, kwargs,
--> 977 input_list)
978
979 # Maintains info about the `Layer.call` stack.
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1113 # Check input assumptions set after layer building, e.g. input shape.
1114 outputs = self._keras_tensor_symbolic_call(
-> 1115 inputs, input_masks, args, kwargs)
1116
1117 if outputs is None:
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs)
846 return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature)
847 else:
--> 848 return self._infer_output_signature(inputs, args, kwargs, input_masks)
849
850 def _infer_output_signature(self, inputs, args, kwargs, input_masks):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks)
886 self._maybe_build(inputs)
887 inputs = self._maybe_cast_inputs(inputs)
--> 888 outputs = call_fn(inputs, *args, **kwargs)
889
890 self._handle_activity_regularization(inputs, outputs)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/wrappers.py in call(self, inputs, training, mask, initial_state, constants)
697
698 y = self.forward_layer(forward_inputs,
--> 699 initial_state=forward_state, **kwargs)
700 y_rev = self.backward_layer(backward_inputs,
701 initial_state=backward_state, **kwargs)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/recurrent.py in __call__(self, inputs, initial_state, constants, **kwargs)
657
658 if initial_state is None and constants is None:
--> 659 return super(RNN, self).__call__(inputs, **kwargs)
660
661 # If any of `initial_state` or `constants` are specified and are Keras
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
1035 with autocast_variable.enable_auto_cast_variables(
1036 self._compute_dtype_object):
-> 1037 outputs = call_fn(inputs, *args, **kwargs)
1038
1039 if self._activity_regularizer:
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/recurrent_v2.py in call(self, inputs, mask, training, initial_state)
1165 input_length=row_lengths if row_lengths is not None else timesteps,
1166 time_major=self.time_major,
-> 1167 zero_output_for_mask=self.zero_output_for_mask)
1168 runtime = _runtime(_RUNTIME_UNKNOWN)
1169 else:
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
204 """Call target, and fall back on dispatchers if there is a TypeError."""
205 try:
--> 206 return target(*args, **kwargs)
207 except (TypeError, ValueError):
208 # Note: convert_to_eager_tensor currently raises a ValueError, not a
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/backend.py in rnn(step_function, inputs, initial_states, go_backwards, mask, constants, unroll, input_length, time_major, zero_output_for_mask)
4353 # the value is discarded.
4354 output_time_zero, _ = step_function(
-> 4355 input_time_zero, tuple(initial_states) + tuple(constants))
4356 output_ta = tuple(
4357 tf.TensorArray(
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/recurrent_v2.py in step(inputs, states)
1153
1154 def step(inputs, states):
-> 1155 return self.cell(inputs, states, **kwargs)
1156
1157 last_output, outputs, states = backend.rnn(
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
1035 with autocast_variable.enable_auto_cast_variables(
1036 self._compute_dtype_object):
-> 1037 outputs = call_fn(inputs, *args, **kwargs)
1038
1039 if self._activity_regularizer:
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/layers/recurrent.py in call(self, inputs, states, training)
2440 b_i, b_f, b_c, b_o = tf.split(
2441 self.bias, num_or_size_splits=4, axis=0)
-> 2442 x_i = backend.bias_add(x_i, b_i)
2443 x_f = backend.bias_add(x_f, b_f)
2444 x_c = backend.bias_add(x_c, b_c)
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
204 """Call target, and fall back on dispatchers if there is a TypeError."""
205 try:
--> 206 return target(*args, **kwargs)
207 except (TypeError, ValueError):
208 # Note: convert_to_eager_tensor currently raises a ValueError, not a
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/keras/backend.py in bias_add(x, bias, data_format)
5973 if len(bias_shape) == 1:
5974 if data_format == 'channels_first':
-> 5975 return tf.nn.bias_add(x, bias, data_format='NCHW')
5976 return tf.nn.bias_add(x, bias, data_format='NHWC')
5977 if ndim(x) in (3, 4, 5):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
204 """Call target, and fall back on dispatchers if there is a TypeError."""
205 try:
--> 206 return target(*args, **kwargs)
207 except (TypeError, ValueError):
208 # Note: convert_to_eager_tensor currently raises a ValueError, not a
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/ops/nn_ops.py in bias_add(value, bias, data_format, name)
3495 else:
3496 return gen_nn_ops.bias_add(
-> 3497 value, bias, data_format=data_format, name=name)
3498
3499
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/ops/gen_nn_ops.py in bias_add(value, bias, data_format, name)
689 data_format = _execute.make_str(data_format, "data_format")
690 _, _, _op, _outputs = _op_def_library._apply_op_helper(
--> 691 "BiasAdd", value=value, bias=bias, data_format=data_format, name=name)
692 _result = _outputs[:]
693 if _execute.must_record_gradient():
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(op_type_name, name, **keywords)
748 op = g._create_op_internal(op_type_name, inputs, dtypes=None,
749 name=scope, input_types=input_types,
--> 750 attrs=attr_protos, op_def=op_def)
751
752 # `outputs` is returned as a separate return value so that the output
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in _create_op_internal(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)
599 return super(FuncGraph, self)._create_op_internal( # pylint: disable=protected-access
600 op_type, captured_inputs, dtypes, input_types, name, attrs, op_def,
--> 601 compute_device)
602
603 def capture(self, tensor, name=None, shape=None):
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in _create_op_internal(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)
3567 input_types=input_types,
3568 original_op=self._default_original_op,
-> 3569 op_def=op_def)
3570 self._create_op_helper(ret, compute_device=compute_device)
3571 return ret
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in __init__(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)
2040 op_def = self._graph._get_op_def(node_def.op)
2041 self._c_op = _create_c_op(self._graph, node_def, inputs,
-> 2042 control_input_ops, op_def)
2043 name = compat.as_str(node_def.name)
2044
~/anaconda3/envs/JupyterSystemEnv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs, op_def)
1881 except errors.InvalidArgumentError as e:
1882 # Convert to ValueError for backwards compatibility.
-> 1883 raise ValueError(str(e))
1884
1885 return c_op
ValueError: Shape must be at least rank 3 but is rank 2 for '{{node bidirectional_6/forward_lstm_7/lstm_cell_20/BiasAdd}} = BiasAdd[T=DT_HALF, data_format="NCHW"](bidirectional_6/forward_lstm_7/lstm_cell_20/MatMul, bidirectional_6/forward_lstm_7/lstm_cell_20/split_1)' with input shapes: [?,200], [200].

Getting NotImplementedError when trying Keras SimpleRNN

I am trying to implement a very basic RNN model using Keras SimpleRNN in my Jupyter Labs Notebook using the below code.
Why is that I am getting the error? What Shall be done?
My Python version is 3.8.11 and Keras is 2.4.3. I tried using Numpy 1.20.1 and 1.18.5.
I did additionally try Keras through Tensorflow too.
from keras import models
from keras.layers import SimpleRNN
model = models.Sequential()
model.add(SimpleRNN(units=32, input_shape=(1,4), activation="relu"))
model.add(Dense(1))
model.summary()
Error:
NotImplementedError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_3896/2618924464.py in <module>
3
4 model = models.Sequential()
----> 5 model.add(SimpleRNN(units=32, input_shape=(1,4)))
6 model.add(Dense(1))
7 model.summary()
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\training\tracking\base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\sequential.py in add(self, layer)
204 # and create the node connecting the current layer
205 # to the input layer we just created.
--> 206 layer(x)
207 set_inputs = True
208
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in __call__(self, inputs, initial_state, constants, **kwargs)
661
662 if initial_state is None and constants is None:
--> 663 return super(RNN, self).__call__(inputs, **kwargs)
664
665 # If any of `initial_state` or `constants` are specified and are Keras
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\base_layer.py in __call__(self, *args, **kwargs)
923 # >> model = tf.keras.Model(inputs, outputs)
924 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
--> 925 return self._functional_construction_call(inputs, args, kwargs,
926 input_list)
927
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\engine\base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1115 try:
1116 with ops.enable_auto_cast_variables(self._compute_dtype_object):
-> 1117 outputs = call_fn(cast_inputs, *args, **kwargs)
1118
1119 except errors.OperatorNotAllowedInGraphError as e:
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in call(self, inputs, mask, training, initial_state)
1570 def call(self, inputs, mask=None, training=None, initial_state=None):
1571 self._maybe_reset_cell_dropout_mask(self.cell)
-> 1572 return super(SimpleRNN, self).call(
1573 inputs, mask=mask, training=training, initial_state=initial_state)
1574
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in call(self, inputs, mask, training, initial_state, constants)
732 self._validate_args_if_ragged(is_ragged_input, mask)
733
--> 734 inputs, initial_state, constants = self._process_inputs(
735 inputs, initial_state, constants)
736
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in _process_inputs(self, inputs, initial_state, constants)
860 initial_state = self.states
861 elif initial_state is None:
--> 862 initial_state = self.get_initial_state(inputs)
863
864 if len(initial_state) != len(self.states):
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in get_initial_state(self, inputs)
643 dtype = inputs.dtype
644 if get_initial_state_fn:
--> 645 init_state = get_initial_state_fn(
646 inputs=None, batch_size=batch_size, dtype=dtype)
647 else:
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in get_initial_state(self, inputs, batch_size, dtype)
1383
1384 def get_initial_state(self, inputs=None, batch_size=None, dtype=None):
-> 1385 return _generate_zero_filled_state_for_cell(self, inputs, batch_size, dtype)
1386
1387 def get_config(self):
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in _generate_zero_filled_state_for_cell(cell, inputs, batch_size, dtype)
2966 batch_size = array_ops.shape(inputs)[0]
2967 dtype = inputs.dtype
-> 2968 return _generate_zero_filled_state(batch_size, cell.state_size, dtype)
2969
2970
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in _generate_zero_filled_state(batch_size_tensor, state_size, dtype)
2984 return nest.map_structure(create_zeros, state_size)
2985 else:
-> 2986 return create_zeros(state_size)
2987
2988
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\keras\layers\recurrent.py in create_zeros(unnested_state_size)
2979 flat_dims = tensor_shape.as_shape(unnested_state_size).as_list()
2980 init_state_size = [batch_size_tensor] + flat_dims
-> 2981 return array_ops.zeros(init_state_size, dtype=dtype)
2982
2983 if nest.is_sequence(state_size):
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\util\dispatch.py in wrapper(*args, **kwargs)
199 """Call target, and fall back on dispatchers if there is a TypeError."""
200 try:
--> 201 return target(*args, **kwargs)
202 except (TypeError, ValueError):
203 # Note: convert_to_eager_tensor currently raises a ValueError, not a
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\array_ops.py in wrapped(*args, **kwargs)
2745
2746 def wrapped(*args, **kwargs):
-> 2747 tensor = fun(*args, **kwargs)
2748 tensor._is_zeros_tensor = True
2749 return tensor
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\array_ops.py in zeros(shape, dtype, name)
2792 # Create a constant if it won't be very big. Otherwise create a fill
2793 # op to prevent serialized GraphDefs from becoming too large.
-> 2794 output = _constant_if_small(zero, shape, dtype, name)
2795 if output is not None:
2796 return output
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\array_ops.py in _constant_if_small(value, shape, dtype, name)
2730 def _constant_if_small(value, shape, dtype, name):
2731 try:
-> 2732 if np.prod(shape) < 1000:
2733 return constant(value, shape=shape, dtype=dtype, name=name)
2734 except TypeError:
<__array_function__ internals> in prod(*args, **kwargs)
C:\ProgramData\Anaconda3\lib\site-packages\numpy\core\fromnumeric.py in prod(a, axis, dtype, out, keepdims, initial, where)
3028 10
3029 """
-> 3030 return _wrapreduction(a, np.multiply, 'prod', axis, dtype, out,
3031 keepdims=keepdims, initial=initial, where=where)
3032
C:\ProgramData\Anaconda3\lib\site-packages\numpy\core\fromnumeric.py in _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs)
85 return reduction(axis=axis, out=out, **passkwargs)
86
---> 87 return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
88
89
C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py in __array__(self)
843
844 def __array__(self):
--> 845 raise NotImplementedError(
846 "Cannot convert a symbolic Tensor ({}) to a numpy array."
847 " This error may indicate that you're trying to pass a Tensor to"
NotImplementedError: Cannot convert a symbolic Tensor (simple_rnn/strided_slice:0) to a numpy array. This error may indicate that you're trying to pass a Tensor to a NumPy call, which is not supported

ValueError: Can't convert Python sequence with out-of-range integer to Tensor

I'm trying to make a binary classifier, to check if an image is a letter or not. I've created the required dataset, and made a basic model to start, but I'm getting this error -
ValueError: Can't convert Python sequence with out-of-range integer to Tensor.
Looked it up, as saw that it could happen if the data was in different dtypes, but eve after converting it all to float64, the error persists.
Do you have any clue as to why this might be?
norm_train_x, y_ = emnist.extract_training_samples('letters')
norm_test_x, y__ = emnist.extract_test_samples('letters')
norm_train_y = np.array([1] * len(norm_train_x))
norm_test_y = np.array([1] * len(norm_test_x))
norm_train_x = np.reshape(norm_train_x, (norm_train_x.shape[0], 28, 28, 1))
norm_test_x = np.reshape(norm_test_x, (norm_test_x.shape[0], 28, 28, 1))
My target is to create a classifier to identify whether an image is a letter or not, so I'm creating a dataset with imgs and random lines, denoted with respective labels being 1s and 0s.
Here's the code to load all the data and prepare for final use. I've stored all the random images in two folders.
def load_data():
random_train_x = []
for x in range(len(norm_train_x)):
im = cv2.imread(r'/content/random_train_imgs/img_' + str(x) + '.png')
im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
im = np.reshape(im, (28, 28, 1))
random_train_x.append(im)
random_train_x = np.array(random_train_x)
random_train_y = np.array([0] * len(random_train_x))
print(random_train_x.shape, random_train_y.shape)
random_test_x = []
for x in range(len(norm_test_x)):
im = cv2.imread(r'/content/random_test_imgs/img_' + str(x) + '.png')
im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
im = np.reshape(im, (28, 28, 1))
random_test_x.append(im)
random_test_x = np.array(random_test_x)
random_test_y = np.array([0] * len(random_test_x))
print(random_test_x.shape, random_test_y.shape)
return (random_train_x, random_train_y), (random_test_x, random_test_y)
def finalise_data():
global random_train_x, random_train_y, random_test_x, random_test_y
global norm_train_X, norm_test_x, norm_train_y, norm_test_y
train_x = np.concatenate((random_train_x, norm_train_x), axis=0)
test_x = np.concatenate((random_test_x, norm_test_x), axis=0)
train_y = np.concatenate((random_train_y, norm_train_y), axis=0)
test_y = np.concatenate((random_test_y, norm_test_y), axis=0)
test_shuffle = np.random.permutation(len(test_y))
train_shuffle = np.random.permutation(len(train_y))
train_x, train_y = train_x[train_shuffle], train_y[train_shuffle]
test_x, test_y = test_x[test_shuffle], test_y[test_shuffle]
print(train_x.shape, train_y.shape)
print(test_x.shape, test_y.shape)
return (train_x, train_y), (test_x, test_y)
train_x = train_x.astype('float32')
train_y = train_y.astype('float32')
test_x = test_x.astype('float32')
test_y = test_y.astype('float32')
The model architecture
model = models.Sequential([
layers.Flatten(input_shape=[28, 28, 1]),
layers.Dense(28**28, activation='relu'),
layers.Dense(3)
])
model.summary()
model.compile(optimizer=tf.keras.optimizers.Adam(),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=['accuracy'])
model.fit(train_x, train_y, validation_data=(test_x, test_y), epochs=10)
The complete traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-8-761e0b4a48eb> in <module>()
2 layers.Flatten(input_shape=[28, 28, 1]),
3 layers.Dense(28**28, activation='relu'),
----> 4 layers.Dense(3)
5 ])
6 model.summary()
31 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
528 self._self_setattr_tracking = False # pylint: disable=protected-access
529 try:
--> 530 result = method(self, *args, **kwargs)
531 finally:
532 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.7/dist-packages/keras/engine/sequential.py in __init__(self, layers, name)
132 layers = [layers]
133 for layer in layers:
--> 134 self.add(layer)
135
136 #property
/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
528 self._self_setattr_tracking = False # pylint: disable=protected-access
529 try:
--> 530 result = method(self, *args, **kwargs)
531 finally:
532 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.7/dist-packages/keras/engine/sequential.py in add(self, layer)
215 # If the model is being built continuously on top of an input layer:
216 # refresh its output.
--> 217 output_tensor = layer(self.outputs[0])
218 if len(tf.nest.flatten(output_tensor)) != 1:
219 raise ValueError(SINGLE_LAYER_OUTPUT_ERROR_MSG)
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
975 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
976 return self._functional_construction_call(inputs, args, kwargs,
--> 977 input_list)
978
979 # Maintains info about the `Layer.call` stack.
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1113 # Check input assumptions set after layer building, e.g. input shape.
1114 outputs = self._keras_tensor_symbolic_call(
-> 1115 inputs, input_masks, args, kwargs)
1116
1117 if outputs is None:
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs)
846 return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature)
847 else:
--> 848 return self._infer_output_signature(inputs, args, kwargs, input_masks)
849
850 def _infer_output_signature(self, inputs, args, kwargs, input_masks):
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks)
884 # overridden).
885 # TODO(kaftan): do we maybe_build here, or have we already done it?
--> 886 self._maybe_build(inputs)
887 inputs = self._maybe_cast_inputs(inputs)
888 outputs = call_fn(inputs, *args, **kwargs)
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in _maybe_build(self, inputs)
2657 # operations.
2658 with tf_utils.maybe_init_scope(self):
-> 2659 self.build(input_shapes) # pylint:disable=not-callable
2660 # We must set also ensure that the layer is marked as built, and the build
2661 # shape is stored since user defined build functions may not be calling
/usr/local/lib/python3.7/dist-packages/keras/layers/core.py in build(self, input_shape)
1183 constraint=self.kernel_constraint,
1184 dtype=self.dtype,
-> 1185 trainable=True)
1186 if self.use_bias:
1187 self.bias = self.add_weight(
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint, use_resource, synchronization, aggregation, **kwargs)
661 synchronization=synchronization,
662 aggregation=aggregation,
--> 663 caching_device=caching_device)
664 if regularizer is not None:
665 # TODO(fchollet): in the future, this should be handled at the
/usr/local/lib/python3.7/dist-packages/tensorflow/python/training/tracking/base.py in _add_variable_with_custom_getter(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)
816 dtype=dtype,
817 initializer=initializer,
--> 818 **kwargs_for_getter)
819
820 # If we set an initializer and the variable processed it, tracking will not
/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer_utils.py in make_variable(name, shape, dtype, initializer, trainable, caching_device, validate_shape, constraint, use_resource, collections, synchronization, aggregation, partitioner)
127 synchronization=synchronization,
128 aggregation=aggregation,
--> 129 shape=variable_shape if variable_shape else None)
130
131
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
264 def __call__(cls, *args, **kwargs):
265 if cls is VariableV1:
--> 266 return cls._variable_v1_call(*args, **kwargs)
267 elif cls is Variable:
268 return cls._variable_v2_call(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py in _variable_v1_call(cls, initial_value, trainable, collections, validate_shape, caching_device, name, variable_def, dtype, expected_shape, import_scope, constraint, use_resource, synchronization, aggregation, shape)
225 synchronization=synchronization,
226 aggregation=aggregation,
--> 227 shape=shape)
228
229 def _variable_v2_call(cls,
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py in <lambda>(**kwargs)
203 shape=None):
204 """Call on Variable class. Useful to force the signature."""
--> 205 previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
206 for _, getter in ops.get_default_graph()._variable_creator_stack: # pylint: disable=protected-access
207 previous_getter = _make_getter(getter, previous_getter)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variable_scope.py in default_variable_creator(next_creator, **kwargs)
2624 synchronization=synchronization,
2625 aggregation=aggregation,
-> 2626 shape=shape)
2627 else:
2628 return variables.RefVariable(
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
268 return cls._variable_v2_call(*args, **kwargs)
269 else:
--> 270 return super(VariableMetaclass, cls).__call__(*args, **kwargs)
271
272
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py in __init__(self, initial_value, trainable, collections, validate_shape, caching_device, name, dtype, variable_def, import_scope, constraint, distribute_strategy, synchronization, aggregation, shape)
1611 aggregation=aggregation,
1612 shape=shape,
-> 1613 distribute_strategy=distribute_strategy)
1614
1615 def _init_from_args(self,
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/resource_variable_ops.py in _init_from_args(self, initial_value, trainable, collections, caching_device, name, dtype, constraint, synchronization, aggregation, distribute_strategy, shape)
1738 with ops.name_scope("Initializer"), device_context_manager(None):
1739 if init_from_fn:
-> 1740 initial_value = initial_value()
1741 if isinstance(initial_value, trackable.CheckpointInitialValue):
1742 self._maybe_initialize_trackable()
/usr/local/lib/python3.7/dist-packages/keras/initializers/initializers_v2.py in __call__(self, shape, dtype, **kwargs)
515 else:
516 limit = math.sqrt(3.0 * scale)
--> 517 return self._random_generator.random_uniform(shape, -limit, limit, dtype)
518
519 def get_config(self):
/usr/local/lib/python3.7/dist-packages/keras/initializers/initializers_v2.py in random_uniform(self, shape, minval, maxval, dtype)
971 op = tf.random.uniform
972 return op(
--> 973 shape=shape, minval=minval, maxval=maxval, dtype=dtype, seed=self.seed)
974
975 def truncated_normal(self, shape, mean, stddev, dtype):
/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
204 """Call target, and fall back on dispatchers if there is a TypeError."""
205 try:
--> 206 return target(*args, **kwargs)
207 except (TypeError, ValueError):
208 # Note: convert_to_eager_tensor currently raises a ValueError, not a
/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/random_ops.py in random_uniform(shape, minval, maxval, dtype, seed, name)
294 maxval = 1
295 with ops.name_scope(name, "random_uniform", [shape, minval, maxval]) as name:
--> 296 shape = tensor_util.shape_tensor(shape)
297 # In case of [0,1) floating results, minval and maxval is unused. We do an
298 # `is` comparison here since this is cheaper than isinstance or __eq__.
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/tensor_util.py in shape_tensor(shape)
1078 # not convertible to Tensors because of mixed content.
1079 shape = tuple(map(tensor_shape.dimension_value, shape))
-> 1080 return ops.convert_to_tensor(shape, dtype=dtype, name="shape")
1081
1082
/usr/local/lib/python3.7/dist-packages/tensorflow/python/profiler/trace.py in wrapped(*args, **kwargs)
161 with Trace(trace_name, **trace_kwargs):
162 return func(*args, **kwargs)
--> 163 return func(*args, **kwargs)
164
165 return wrapped
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
1564
1565 if ret is None:
-> 1566 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
1567
1568 if ret is NotImplemented:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
344 as_ref=False):
345 _ = as_ref
--> 346 return constant(v, dtype=dtype, name=name)
347
348
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name)
270 """
271 return _constant_impl(value, dtype, shape, name, verify_shape=False,
--> 272 allow_broadcast=True)
273
274
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in _constant_impl(value, dtype, shape, name, verify_shape, allow_broadcast)
281 with trace.Trace("tf.constant"):
282 return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
--> 283 return _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
284
285 g = ops.get_default_graph()
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in _constant_eager_impl(ctx, value, dtype, shape, verify_shape)
306 def _constant_eager_impl(ctx, value, dtype, shape, verify_shape):
307 """Creates a constant on the current device."""
--> 308 t = convert_to_eager_tensor(value, ctx, dtype)
309 if shape is None:
310 return t
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
104 dtype = dtypes.as_dtype(dtype).as_datatype_enum
105 ctx.ensure_initialized()
--> 106 return ops.EagerTensor(value, ctx.device_name, dtype)
107
108
ValueError: Can't convert Python sequence with out-of-range integer to Tensor.

Keras Concatenate layer dimensions acting up

So I'm creating a model using the functional API in tf.keras in which I'm doing a multi-input model.
The input for training is of shape (n_examples = 58667, n_dim = 2748). Each example is a concatenate of a 2048 and a 700 dimensions vector.
But I'm getting an error message that I don't understand:
InvalidArgumentError: Dimension 0 in both shapes must be equal, but are 1 and 0. Shapes are [1] and [0]. for 'model_27/concatenate_28/concat' (op: 'ConcatV2') with input shapes: [1,100], [0,100], [] and with computed input tensors: input[2] = <1>.
Here are dummy inputs and imports to make it runnable:
from tensorflow.keras import models, layers, losses, metrics, optimizers
from tensorflow.keras.layers import Dense, Concatenate, Input, Lambda
from tensorflow.keras.models import Model
from tensorflow.keras import backend as K
from sklearn.model_selection import train_test_split
import numpy as np
fake_train = np.random.rand(10000,2748)
fake_test = np.random.randint(0,1,(10000,1))
x_train, x_dev, y_train, y_dev = train_test_split(fake_train, fake_test, test_size = 0.2)
My model is created with this function:
def build_model():
input0 = Input(shape=(2748,))
branch1 = Lambda(lambda x:x[:2048])(input0)
branch1 = Dense(1000, activation='relu')(branch1)
branch1 = Dense(100, activation='relu')(branch1)
branch1 = Dense(100, activation='relu')(branch1)
branch2 = Lambda(lambda x:x[2048:])(input0)
branch2 = Dense(1000, activation='relu')(branch2)
branch2 = Dense(100, activation='relu')(branch2)
branch2 = Dense(100, activation='relu')(branch2)
out = layers.concatenate([branch1, branch2],axis=-1)
out = Dense(10, activation = 'relu')(out)
out = Dense(1, activation='sigmoid')(out)
model = Model(inputs=input0, outputs=out)
model.compile(optimizer=optimizers.Adam(lr=0.001),
loss='binary_crossentropy',
metrics=['accuracy', recall_m, precision_m])
return model
Here are parameters for cross validation for the dummy data:
k = 3 #Number of folds for CV
num_epochs = 4 #for test only
batch_size = 1
And this is my cross-validation for the model, which started the error:
all_loss_histories = []
all_recall_histories = []
all_precision_histories = []
for i in range(k):
val_data = x_train[i * num_val_samples:(i+1) * num_val_samples]
val_targets = y_train[i * num_val_samples:(i+1) * num_val_samples]
partial_train_data = np.concatenate(
[x_train[:i*num_val_samples],
x_train[(i+1)*num_val_samples:]],
axis = 0)
partial_train_targets = np.concatenate(
[y_train[:i*num_val_samples],
y_train[(i+1)*num_val_samples:]],
axis = 0)
model = build_model()
history = model.fit(partial_train_data,
partial_train_targets,
epochs = num_epochs,
batch_size = batch_size,
verbose = 1,
validation_data = (val_data, val_targets),
use_multiprocessing=False)
print('Finished training fold '+str(i+1))
loss_history = history.history['val_loss']
recall_history = history.history['val_recall_m']
precision_history = history.history['val_precision_m']
all_loss_histories.append(loss_history)
all_recall_histories.append(recall_history)
all_precision_histories.append(precision_history)
Any idea why there is an error?
Using python3.7 and tf 2.0 on a MacBook Pro 2018 (on OSX, not on a linux VM)
Thanks!
The complete error:
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
1609 try:
-> 1610 c_op = c_api.TF_FinishOperation(op_desc)
1611 except errors.InvalidArgumentError as e:
InvalidArgumentError: Dimension 0 in both shapes must be equal, but are 1 and 0. Shapes are [1] and [0]. for 'model_33/concatenate_34/concat' (op: 'ConcatV2') with input shapes: [1,100], [0,100], [] and with computed input tensors: input[2] = <1>.
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
<timed exec> in <module>
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
726 max_queue_size=max_queue_size,
727 workers=workers,
--> 728 use_multiprocessing=use_multiprocessing)
729
730 def evaluate(self,
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py in fit(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, **kwargs)
322 mode=ModeKeys.TRAIN,
323 training_context=training_context,
--> 324 total_epochs=epochs)
325 cbks.make_logs(model, epoch_logs, training_result, ModeKeys.TRAIN)
326
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py in run_one_epoch(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)
121 step=step, mode=mode, size=current_batch_size) as batch_logs:
122 try:
--> 123 batch_outs = execution_function(iterator)
124 except (StopIteration, errors.OutOfRangeError):
125 # TODO(kaftan): File bug about tf function and errors.OutOfRangeError?
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py in execution_function(input_fn)
84 # `numpy` translates Tensors to values in Eager mode.
85 return nest.map_structure(_non_none_constant_value,
---> 86 distributed_function(input_fn))
87
88 return execution_function
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in __call__(self, *args, **kwds)
455
456 tracing_count = self._get_tracing_count()
--> 457 result = self._call(*args, **kwds)
458 if tracing_count == self._get_tracing_count():
459 self._call_counter.called_without_tracing()
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in _call(self, *args, **kwds)
501 # This is the first call of __call__, so we have to initialize.
502 initializer_map = object_identity.ObjectIdentityDictionary()
--> 503 self._initialize(args, kwds, add_initializers_to=initializer_map)
504 finally:
505 # At this point we know that the initialization is complete (or less
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
406 self._concrete_stateful_fn = (
407 self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
--> 408 *args, **kwds))
409
410 def invalid_creator_scope(*unused_args, **unused_kwds):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
1846 if self.input_signature:
1847 args, kwargs = None, None
-> 1848 graph_function, _, _ = self._maybe_define_function(args, kwargs)
1849 return graph_function
1850
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _maybe_define_function(self, args, kwargs)
2148 graph_function = self._function_cache.primary.get(cache_key, None)
2149 if graph_function is None:
-> 2150 graph_function = self._create_graph_function(args, kwargs)
2151 self._function_cache.primary[cache_key] = graph_function
2152 return graph_function, args, kwargs
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
2039 arg_names=arg_names,
2040 override_flat_arg_shapes=override_flat_arg_shapes,
-> 2041 capture_by_value=self._capture_by_value),
2042 self._function_attributes,
2043 # Tell the ConcreteFunction to clean up its graph once it goes out of
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
913 converted_func)
914
--> 915 func_outputs = python_func(*func_args, **func_kwargs)
916
917 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py in wrapped_fn(*args, **kwds)
356 # __wrapped__ allows AutoGraph to swap in a converted function. We give
357 # the function a weak reference to itself to avoid a reference cycle.
--> 358 return weak_wrapped_fn().__wrapped__(*args, **kwds)
359 weak_wrapped_fn = weakref.ref(wrapped_fn)
360
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py in distributed_function(input_iterator)
71 strategy = distribution_strategy_context.get_strategy()
72 outputs = strategy.experimental_run_v2(
---> 73 per_replica_function, args=(model, x, y, sample_weights))
74 # Out of PerReplica outputs reduce or pick values to return.
75 all_outputs = dist_utils.unwrap_output_dict(
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/distribute/distribute_lib.py in experimental_run_v2(self, fn, args, kwargs)
758 fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx(),
759 convert_by_default=False)
--> 760 return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
761
762 def reduce(self, reduce_op, value, axis):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/distribute/distribute_lib.py in call_for_each_replica(self, fn, args, kwargs)
1785 kwargs = {}
1786 with self._container_strategy().scope():
-> 1787 return self._call_for_each_replica(fn, args, kwargs)
1788
1789 def _call_for_each_replica(self, fn, args, kwargs):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/distribute/distribute_lib.py in _call_for_each_replica(self, fn, args, kwargs)
2130 self._container_strategy(),
2131 replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)):
-> 2132 return fn(*args, **kwargs)
2133
2134 def _reduce_to(self, reduce_op, value, destinations):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py in wrapper(*args, **kwargs)
290 def wrapper(*args, **kwargs):
291 with ag_ctx.ControlStatusCtx(status=ag_ctx.Status.DISABLED):
--> 292 return func(*args, **kwargs)
293
294 if inspect.isfunction(func) or inspect.ismethod(func):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py in train_on_batch(model, x, y, sample_weight, class_weight, reset_metrics)
262 y,
263 sample_weights=sample_weights,
--> 264 output_loss_metrics=model._output_loss_metrics)
265
266 if reset_metrics:
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in train_on_batch(model, inputs, targets, sample_weights, output_loss_metrics)
309 sample_weights=sample_weights,
310 training=True,
--> 311 output_loss_metrics=output_loss_metrics))
312 if not isinstance(outs, list):
313 outs = [outs]
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in _process_single_batch(model, inputs, targets, output_loss_metrics, sample_weights, training)
250 output_loss_metrics=output_loss_metrics,
251 sample_weights=sample_weights,
--> 252 training=training))
253 if total_loss is None:
254 raise ValueError('The model cannot be run '
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_eager.py in _model_loss(model, inputs, targets, output_loss_metrics, sample_weights, training)
125 inputs = nest.map_structure(ops.convert_to_tensor, inputs)
126
--> 127 outs = model(inputs, **kwargs)
128 outs = nest.flatten(outs)
129
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
845 outputs = base_layer_utils.mark_as_return(outputs, acd)
846 else:
--> 847 outputs = call_fn(cast_inputs, *args, **kwargs)
848
849 except errors.OperatorNotAllowedInGraphError as e:
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in call(self, inputs, training, mask)
706 return self._run_internal_graph(
707 inputs, training=training, mask=mask,
--> 708 convert_kwargs_to_constants=base_layer_utils.call_context().saving)
709
710 def compute_output_shape(self, input_shape):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/network.py in _run_internal_graph(self, inputs, training, mask, convert_kwargs_to_constants)
858
859 # Compute outputs.
--> 860 output_tensors = layer(computed_tensors, **kwargs)
861
862 # Update tensor_dict.
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
845 outputs = base_layer_utils.mark_as_return(outputs, acd)
846 else:
--> 847 outputs = call_fn(cast_inputs, *args, **kwargs)
848
849 except errors.OperatorNotAllowedInGraphError as e:
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/merge.py in call(self, inputs)
180 return y
181 else:
--> 182 return self._merge_function(inputs)
183
184 #tf_utils.shape_type_conversion
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/layers/merge.py in _merge_function(self, inputs)
392
393 def _merge_function(self, inputs):
--> 394 return K.concatenate(inputs, axis=self.axis)
395
396 #tf_utils.shape_type_conversion
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py in concatenate(tensors, axis)
2706 return sparse_ops.sparse_concat(axis, tensors)
2707 else:
-> 2708 return array_ops.concat([to_dense(x) for x in tensors], axis)
2709
2710
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/util/dispatch.py in wrapper(*args, **kwargs)
178 """Call target, and fall back on dispatchers if there is a TypeError."""
179 try:
--> 180 return target(*args, **kwargs)
181 except (TypeError, ValueError):
182 # Note: convert_to_eager_tensor currently raises a ValueError, not a
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/array_ops.py in concat(values, axis, name)
1429 dtype=dtypes.int32).get_shape().assert_has_rank(0)
1430 return identity(values[0], name=name)
-> 1431 return gen_array_ops.concat_v2(values=values, axis=axis, name=name)
1432
1433
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_array_ops.py in concat_v2(values, axis, name)
1255 _attr_N = len(values)
1256 _, _, _op = _op_def_lib._apply_op_helper(
-> 1257 "ConcatV2", values=values, axis=axis, name=name)
1258 _result = _op.outputs[:]
1259 _inputs_flat = _op.inputs
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
791 op = g.create_op(op_type_name, inputs, dtypes=None, name=scope,
792 input_types=input_types, attrs=attr_protos,
--> 793 op_def=op_def)
794 return output_structure, op_def.is_stateful, op
795
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py in create_op(***failed resolving arguments***)
546 return super(FuncGraph, self)._create_op_internal( # pylint: disable=protected-access
547 op_type, inputs, dtypes, input_types, name, attrs, op_def,
--> 548 compute_device)
549
550 def capture(self, tensor, name=None):
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in _create_op_internal(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_device)
3427 input_types=input_types,
3428 original_op=self._default_original_op,
-> 3429 op_def=op_def)
3430 self._create_op_helper(ret, compute_device=compute_device)
3431 return ret
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in __init__(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)
1771 op_def, inputs, node_def.attr)
1772 self._c_op = _create_c_op(self._graph, node_def, grouped_inputs,
-> 1773 control_input_ops)
1774 # pylint: enable=protected-access
1775
~/opt/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
1611 except errors.InvalidArgumentError as e:
1612 # Convert to ValueError for backwards compatibility.
-> 1613 raise ValueError(str(e))
1614
1615 return c_op
ValueError: Dimension 0 in both shapes must be equal, but are 1 and 0. Shapes are [1] and [0]. for 'model_33/concatenate_34/concat' (op: 'ConcatV2') with input shapes: [1,100], [0,100], [] and with computed input tensors: input[2] = <1>.```

I believe you want
Lambda(lambda x:x[:, :2048])(input0)
and
Lambda(lambda x:x[:,2048:])(input0)
since there is a batch axis which should be preserved. Your current code is splitting around 2048 along the batch axis, which results in one input to concatenate that has batch size 0, and one input that has batch size 1. Since the batch sizes don't match, they can't be concatenated.
Making this correction allows me to run your code without the error.

Simple LSTM model : No attr named '_XlaCompile' in name error

I am very new to machine learning and I came across an error while attempting to make a simple LSTM model, and I am absolutely clueless how to debug this. I am using Keras version 2.2.2.
My code looks more or less like this:
model = Sequential()
model.add(Embedding(400001, emb_dim, trainable=False, input_length = 56, weights = [emb_matrix]))
model.add(LSTM(128, return_sequences=False))
model.add(Dense(5, activation='softmax'))
model.summary()
model.fit(train_in, train_out, epochs = 50, batch_size = 32, shuffle=True)
My inputs were initially lists of sentences that I intend to do sentiment analysis on, I then use Glove vectors with 50 dim to convert the sentences into vectors with shape (sample size, 56, 50), since my maximum number of words per sentence is 56 (is this on the high side?).
My model summary:
Layer (type) Output Shape Param #
=================================================================
embedding_5 (Embedding) (None, 56, 50) 20000050
_________________________________________________________________
lstm_6 (LSTM) (None, 128) 91648
_________________________________________________________________
dense_4 (Dense) (None, 5) 645
=================================================================
Total params: 20,092,343
Trainable params: 92,293
Non-trainable params: 20,000,050
My inputs:
print(train_in.shape, train_out.shape)
>(156060, 56) (156060, 5)
emb_matrix.shape
>(400001, 50)
print(train_in.dtype, train_out.dtype, emb_matrix.dtype)
>float32 float32 float32
And finally my error message:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gradients_impl.py in _MaybeCompile(scope, op, func, grad_fn)
369 try:
--> 370 xla_compile = op.get_attr("_XlaCompile")
371 xla_separate_compiled_gradients = op.get_attr(
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\ops.py in get_attr(self, name)
2172 raise ValueError(
-> 2173 "No attr named '" + name + "' in " + str(self._node_def))
2174 x = self._node_def.attr[name]
ValueError: No attr named '_XlaCompile' in name: "lstm_6/while/TensorArrayWrite/TensorArrayWriteV3"
op: "TensorArrayWriteV3"
input: "lstm_6/while/TensorArrayWrite/TensorArrayWriteV3/Enter"
input: "lstm_6/while/Identity_1"
input: "lstm_6/while/mul_5"
input: "lstm_6/while/Identity_2"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
attr {
key: "_class"
value {
list {
s: "loc:#lstm_6/while/mul_5"
}
}
}
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
509 as_ref=input_arg.is_ref,
--> 510 preferred_dtype=default_dtype)
511 except TypeError as err:
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, ctx)
1021 if ret is None:
-> 1022 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
1023
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\ops.py in _TensorTensorConversionFunction(t, dtype, name, as_ref)
865 "Tensor conversion requested dtype %s for Tensor with dtype %s: %r" %
--> 866 (dtype.name, t.dtype.name, str(t)))
867 return t
ValueError: Tensor conversion requested dtype int32 for Tensor with dtype int64: 'Tensor("lstm_6/while/maximum_iterations:0", shape=(), dtype=int64)'
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-54-936a1189c2d5> in <module>()
----> 1 model.fit(train_in, train_out, epochs = 50, batch_size = 32, shuffle=True)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1006 else:
1007 ins = x + y + sample_weights
-> 1008 self._make_train_function()
1009 f = self.train_function
1010
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\engine\training.py in _make_train_function(self)
496 training_updates = self.optimizer.get_updates(
497 params=self._collected_trainable_weights,
--> 498 loss=self.total_loss)
499 updates = (self.updates +
500 training_updates +
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name +
90 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\optimizers.py in get_updates(self, loss, params)
633 #interfaces.legacy_get_updates_support
634 def get_updates(self, loss, params):
--> 635 grads = self.get_gradients(loss, params)
636 self.updates = [K.update_add(self.iterations, 1)]
637
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\optimizers.py in get_gradients(self, loss, params)
87
88 def get_gradients(self, loss, params):
---> 89 grads = K.gradients(loss, params)
90 if None in grads:
91 raise ValueError('An operation has `None` for gradient. '
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\keras\backend\tensorflow_backend.py in gradients(loss, variables)
2706 A gradients tensor.
2707 """
-> 2708 return tf.gradients(loss, variables, colocate_gradients_with_ops=True)
2709
2710
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gradients_impl.py in gradients(ys, xs, grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method, stop_gradients)
607 # functions.
608 in_grads = _MaybeCompile(
--> 609 grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
610 else:
611 # For function call ops, we add a 'SymbolicGradient'
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gradients_impl.py in _MaybeCompile(scope, op, func, grad_fn)
373 xla_scope = op.get_attr("_XlaScope").decode()
374 except ValueError:
--> 375 return grad_fn() # Exit early
376
377 if not xla_compile:
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gradients_impl.py in <lambda>()
607 # functions.
608 in_grads = _MaybeCompile(
--> 609 grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
610 else:
611 # For function call ops, we add a 'SymbolicGradient'
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\tensor_array_grad.py in _TensorArrayWriteGrad(op, flow)
129 colocate_with_first_write_call=False)
130 .grad(source=grad_source, flow=flow))
--> 131 grad = g.read(index)
132 return [None, None, grad, flow]
133
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\tensor_array_ops.py in read(self, index, name)
857 The tensor at index `index`.
858 """
--> 859 return self._implementation.read(index, name=name)
860
861 #tf_should_use.should_use_result
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\tensor_array_ops.py in read(self, index, name)
257 flow_in=self._flow,
258 dtype=self._dtype,
--> 259 name=name)
260 if self._element_shape:
261 value.set_shape(self._element_shape[0].dims)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gen_data_flow_ops.py in _tensor_array_read_v3(handle, index, flow_in, dtype, name)
4993 _, _, _op = _op_def_lib._apply_op_helper(
4994 "TensorArrayReadV3", handle=handle, index=index, flow_in=flow_in,
-> 4995 dtype=dtype, name=name)
4996 _result = _op.outputs[:]
4997 _inputs_flat = _op.inputs
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
785 op = g.create_op(op_type_name, inputs, output_types, name=scope,
786 input_types=input_types, attrs=attr_protos,
--> 787 op_def=op_def)
788 return output_structure, op_def.is_stateful, op
789
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\ops.py in create_op(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)
3158 input_types=input_types,
3159 original_op=self._default_original_op,
-> 3160 op_def=op_def)
3161 self._create_op_helper(ret, compute_shapes=compute_shapes,
3162 compute_device=compute_device)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\ops.py in __init__(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)
1672 control_flow_util.CheckInputFromValidContext(self, input_tensor.op)
1673 if self._control_flow_context is not None:
-> 1674 self._control_flow_context.AddOp(self)
1675 self._recompute_node_def()
1676
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\control_flow_ops.py in AddOp(self, op)
2249 op_input_ctxt._AddOpInternal(op)
2250 return
-> 2251 self._AddOpInternal(op)
2252
2253 def _AddOpInternal(self, op):
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\control_flow_ops.py in _AddOpInternal(self, op)
2272 for index in range(len(op.inputs)):
2273 x = op.inputs[index]
-> 2274 real_x = self.AddValue(x)
2275 if real_x != x:
2276 op._update_input(index, real_x)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\control_flow_ops.py in AddValue(self, val)
2205 forward_ctxt = forward_ctxt.GetWhileContext()
2206 if forward_ctxt == grad_ctxt.grad_state.forward_context:
-> 2207 real_val = grad_ctxt.grad_state.GetRealValue(val)
2208 self._external_values[val.name] = real_val
2209 return real_val
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\control_flow_ops.py in GetRealValue(self, value)
1048 # Record the history of this value in forward_ctxt.
1049 self._grad_context.Exit()
-> 1050 history_value = cur_grad_state.AddForwardAccumulator(cur_value)
1051 self._grad_context.Enter()
1052 break
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\control_flow_ops.py in AddForwardAccumulator(self, value, dead_branch)
906 max_size=maximum_iterations,
907 elem_type=value.dtype.base_dtype,
--> 908 name="f_acc")
909 # pylint: enable=protected-access
910 if curr_ctxt: curr_ctxt.Exit()
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\ops\gen_data_flow_ops.py in _stack_v2(max_size, elem_type, stack_name, name)
4014 _, _, _op = _op_def_lib._apply_op_helper(
4015 "StackV2", max_size=max_size, elem_type=elem_type,
-> 4016 stack_name=stack_name, name=name)
4017 _result = _op.outputs[:]
4018 _inputs_flat = _op.inputs
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\tensorflow\python\framework\op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
531 if input_arg.type != types_pb2.DT_INVALID:
532 raise TypeError("%s expected type of %s." %
--> 533 (prefix, dtypes.as_dtype(input_arg.type).name))
534 else:
535 # Update the maps with the default, if needed.
TypeError: Input 'max_size' of 'StackV2' Op has type int64 that does not match expected type of int32.

I was using version 1.5.0 of TF initially, upgraded to v1.8.0 and all is working. Issue resolved.