I want to create a bidirectional RNN Encoder in
embedding_attention_seq2seq
in
seq2seq_model.py :
here is the code blew
def embedding_attention_seq2seq(encoder_inputs, decoder_inputs, cell,
num_encoder_symbols, num_decoder_symbols,
num_heads=1, output_projection=None,
feed_previous=False, dtype=dtypes.float32,
scope=None, initial_state_attention=False):
with variable_scope.variable_scope(scope or"embedding_attention_seq2seq"):
# Encoder.
encoder_cell = rnn_cell.EmbeddingWrapper(cell, num_encoder_symbols)
encoder_outputs, encoder_state = rnn.rnn(
encoder_cell, encoder_inputs, dtype=dtype)
# First calculate a concatenation of encoder outputs to put attention on.
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size])
for e in encoder_outputs]
attention_states = array_ops.concat(1, top_states)
....
Here the code I changed.Borrowed from https://github.com/ematvey/tensorflow-seq2seq-tutorials/blob/master/2-seq2seq-advanced.ipynb
# Encoder.
encoder_cell = copy.deepcopy(cell)
encoder_cell = core_rnn_cell.EmbeddingWrapper(
encoder_cell,
embedding_classes=num_encoder_symbols,
embedding_size=embedding_size)
(encoder_outputs,
encoder_fw_final_state,
encoder_bw_final_state)=rnn.bidirectional_rnn(
cell_fw=encoder_cell,
cell_bw=encoder_cell,
encoder_inputs,
dtype=dtype)
encoder_final_state_c = tf.concat(
(encoder_fw_final_state.c, encoder_bw_final_state.c), 1)
encoder_final_state_h = tf.concat(
(encoder_fw_final_state.h, encoder_bw_final_state.h), 1)
encoder_state = LSTMStateTuple(
c=encoder_final_state_c,
h=encoder_final_state_h)
list of errors:
Traceback (most recent call last):
File "translate.py", line 301, in <module>
tf.app.run()
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/platform/app.py", line 43, in run
sys.exit(main(sys.argv[:1] + flags_passthrough))
File "translate.py", line 297, in main
train()
File "translate.py", line 156, in train
model = create_model(sess, False)
File "translate.py", line 134, in create_model
dtype=dtype)
File "/home/tensorflow/Downloads/NMT-jp-ch-master/seq2seq_model.py", line 185, in __init__
softmax_loss_function=softmax_loss_function)
File "/home/tensorflow/Downloads/NMT-jp-ch-master/seq2seq.py", line 628, in model_with_buckets
decoder_inputs[:bucket[1]])
File "/home/tensorflow/Downloads/NMT-jp-ch-master/seq2seq_model.py", line 184, in <lambda>
lambda x, y: seq2seq_f(x, y, False),
File "/home/tensorflow/Downloads/NMT-jp-ch-master/seq2seq_model.py", line 148, in seq2seq_f
dtype=dtype)
File "/home/tensorflow/Downloads/NMT-jp-ch-master/seq2seq.py", line 432, in embedding_attention_seq2seq
inputs=encoder_inputs
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/ops/rnn.py", line 652, in bidirectional_dynamic_rnn
time_major=time_major, scope=fw_scope)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/ops/rnn.py", line 789, in dynamic_rnn
for input_ in flat_input)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/ops/rnn.py", line 789, in <genexpr>
for input_ in flat_input)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/ops/array_ops.py", line 1280, in transpose
ret = gen_array_ops.transpose(a, perm, name=name)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3656, in transpose
result = _op_def_lib.apply_op("Transpose", x=x, perm=perm, name=name)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/ops.py", line 2242, in create_op
set_shapes_for_outputs(ret)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/ops.py", line 1617, in set_shapes_for_outputs
shapes = shape_func(op)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/ops.py", line 1568, in call_with_requiring
return call_cpp_shape_fn(op, require_shape_fn=True)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/common_shapes.py", line 610, in call_cpp_shape_fn
debug_python_shape_fn, require_shape_fn)
File "/home/tensorflow/anaconda3/envs/tf/lib/python3.4/site-packages/tensorflow/python/framework/common_shapes.py", line 675, in _call_cpp_shape_fn_impl
raise ValueError(err.message)
ValueError: Dimension must be 1 but is 3 for 'model_with_buckets/embedding_attention_seq2seq/BiRNN/FW/transpose' (op: 'Transpose') with input shapes: [?], [3].
I use py3.4 and tf-v0.12
How to create the proper bidirectional RNN Encoder with the https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/ops/rnn.py in seq2seq model ?
Thank you in advance.
The problem solved by
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size*2])
Yes，reshape should *2 .
Related
I am trying to build a model that uses transposed convolution operation ,But when I try to pass the weights and bias as a parameter to the model function it gives an error.
import tensorflow as tf
import cv2
class WeighsTest:
def __model_1(self, plh_var1,weights,bias):
conv = tf.nn.conv2d(plh_var1, weights["v1"], [1, 1, 1, 1], padding="SAME")
conv = tf.add(conv, bias["b1"])
conv = tf.nn.relu(conv)
tran_conv = tf.layers.conv2d_transpose(conv,32, 4, 3, padding="valid")
return tran_conv
def train(self, input_img):
plh = tf.placeholder(dtype=tf.float32, shape=(None, 84, 150, 3), name="input_img")
with tf.variable_scope("test", reuse=tf.AUTO_REUSE):
var_dict_1 = {
"v1": tf.get_variable("v1", shape=(2, 2, 3, 32), initializer=tf.contrib.layers.xavier_initializer())
}
bias_1 = {
"b1": tf.get_variable("b1", shape=32, initializer=tf.contrib.layers.xavier_initializer())
}
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
out_p = sess.run([self.__model_1(plh, var_dict_1, bias_1)], feed_dict={plh: [input_img]})
return out_p
if __name__ == '__main__':
obj = WeighsTest()
img = cv2.imread('./1.jpg')
output = obj.train(img)
traceback is given below
Traceback (most recent call last):
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1365, in _do_call
return fn(*args)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1350, in _run_fn
target_list, run_metadata)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value conv2d_transpose/bias
[[{{node conv2d_transpose/bias/read}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/strange/DEV/Python_Projects/slmv/testing_unit.py", line 43, in <module>
output = obj.train(img)
File "/home/strange/DEV/Python_Projects/slmv/testing_unit.py", line 36, in train
out_p = sess.run([self.__model_1(x1, var_dict_1, bias_1)], feed_dict={x1: [input_img]})
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value conv2d_transpose/bias
[[node conv2d_transpose/bias/read (defined at usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py:1748) ]]
Original stack trace for 'conv2d_transpose/bias/read':
File "home/strange/DEV/Python_Projects/slmv/testing_unit.py", line 43, in <module>
output = obj.train(img)
File "home/strange/DEV/Python_Projects/slmv/testing_unit.py", line 36, in train
out_p = sess.run([self.__model_1(x1, var_dict_1, bias_1)], feed_dict={x1: [input_img]})
File "home/strange/DEV/Python_Projects/slmv/testing_unit.py", line 10, in __model_1
tran_conv = tf.layers.conv2d_transpose(conv,32, 4, 3, padding="valid")
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/layers/convolutional.py", line 1279, in conv2d_transpose
return layer.apply(inputs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/base_layer.py", line 1700, in apply
return self.__call__(inputs, *args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/layers/base.py", line 548, in __call__
outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/base_layer.py", line 824, in __call__
self._maybe_build(inputs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/base_layer.py", line 2146, in _maybe_build
self.build(input_shapes)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/layers/convolutional.py", line 787, in build
dtype=self.dtype)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/layers/base.py", line 461, in add_weight
**kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/base_layer.py", line 529, in add_weight
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/training/tracking/base.py", line 712, in _add_variable_with_custom_getter
**kwargs_for_getter)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 1500, in get_variable
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 1243, in get_variable
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 567, in get_variable
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 519, in _true_getter
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 933, in _get_single_variable
aggregation=aggregation)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 258, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 219, in _variable_v1_call
shape=shape)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 197, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variable_scope.py", line 2519, in default_variable_creator
shape=shape)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 262, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 1688, in __init__
shape=shape)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/variables.py", line 1872, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/dispatch.py", line 180, in wrapper
return target(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/array_ops.py", line 203, in identity
ret = gen_array_ops.identity(input, name=name)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/ops/gen_array_ops.py", line 4239, in identity
"Identity", input=input, name=name)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "usr/local/lib/python3.7/dist-packages/tensorflow_core/python/framework/ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
It is important to pass the bias and weights as parameters to the model.
I used tensorflow-cpu 1.15.2 for the model.
Any idea how to solve this ?
Thank you
The model needed to be called before the tf.global_variabels_initializer() is used
ie. the train function is changed as below
def train(self, input_img):
plh = tf.placeholder(dtype=tf.float32, shape=(None, 84, 150, 3), name="input_img")
with tf.variable_scope("test", reuse=tf.AUTO_REUSE):
var_dict_1 = {
"v1": tf.get_variable("v1", shape=(2, 2, 3, 32), initializer=tf.contrib.layers.xavier_initializer())
}
bias_1 = {
"b1": tf.get_variable("b1", shape=32, initializer=tf.contrib.layers.xavier_initializer())
}
"""model is called before variable initialization"""
model = self.__model_1(plh, var_dict_1, bias_1)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
out_p = sess.run([model], feed_dict={plh: [input_img]})
return out_p
the line given below:
out_p = sess.run([self.__model_1(plh, var_dict_1, bias_1)], feed_dict={plh: [input_img]})
is changed into
out_p = sess.run([model], feed_dict={plh: [input_img]})
I'm trying to use a sigmoid to join the output of two models with different embedding matrix. but I keep getting the error at the concatenate line. I have tried other suggestions from similar questions but it keeps giving the same error. I feel I'm missing something but I can't find it. please help explain. Thanks
############################ MODEL 1 ######################################
input_tensor=Input(shape=(35,))
input_layer= Embedding(vocab_size, 300, input_length=35, weights=[embedding_matrix],trainable=True)(input_tensor)
conv_blocks = []
filter_sizes = (2,3,4)
for fx in filter_sizes:
conv_layer= Conv1D(100, kernel_size=fx, activation='relu', data_format='channels_first')(input_layer) #filters=100, kernel_size=3
maxpool_layer = MaxPooling1D(pool_size=4)(conv_layer)
flat_layer= Flatten()(maxpool_layer)
conv_blocks.append(flat_layer)
conc_layer=concatenate(conv_blocks, axis=1)
graph = Model(inputs=input_tensor, outputs=conc_layer)
model = Sequential()
model.add(graph)
model.add(Dropout(0.2))
############################ MODEL 2 ######################################
input_tensor_1=Input(shape=(35,))
input_layer_1= Embedding(vocab_size, 300, input_length=35, weights=[embedding_matrix_1],trainable=True)(input_tensor_1)
conv_blocks_1 = []
filter_sizes_1 = (2,3,4)
for fx in filter_sizes_1:
conv_layer_1= Conv1D(100, kernel_size=fx, activation='relu', data_format='channels_first')(input_layer_1) #filters=100, kernel_size=3
maxpool_layer_1 = MaxPooling1D(pool_size=4)(conv_layer_1)
flat_layer_1= Flatten()(maxpool_layer_1)
conv_blocks_1.append(flat_layer_1)
conc_layer_1=concatenate(conv_blocks_1, axis=1)
graph_1 = Model(inputs=input_tensor_1, outputs=conc_layer_1)
model_1 = Sequential()
model_1.add(graph_1)
model_1.add(Dropout(0.2))
fused = concatenate([graph, graph_1], axis=-1)
prediction = Dense(3, activation='sigmoid')(fused)
model = Model(inputs=[input_tensor,input_tensor_1], outputs=[prediction])
model.compile(loss='sparse_categorical_crossentropy',optimizer='Adagrad', metrics=['accuracy'])
model.summary()
This is the error trace
Traceback (most recent call last):
File "DL_Ensemble.py", line 145, in <module>
fused = concatenate([graph, graph_1], axis= 1 )
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/layers/merge.py", line 705, in concatenate
return Concatenate(axis=axis, **kwargs)(inputs)
File "/usr/pkg/lib/python3.8/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 887, in __call__
self._maybe_build(inputs)
File "/usr/pkg/lib/python3.8/site-packages/tensorflow_core/python/keras/engine/base_layer.py", line 2141, in _maybe_build
self.build(input_shapes)
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/utils/tf_utils.py", line 306, in wrapper
output_shape = fn(instance, input_shape)
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/layers/merge.py", line 378, in build
raise ValueError('A `Concatenate` layer should be called '
ValueError: A `Concatenate` layer should be called on a list of at least 2 inputs
UPDATE: I have reflected the answers given by #VivekMehta, however, I have this error.
File "DL_Ensemble.py", line 165, in <module>
model.fit([train_sequences,train_sequences], train_y, epochs=10,
verbose=False, batch_size=32, class_weight={0: 6.0, 1: 1.0, 2: 2.0})
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/engine/training.py", line 709, in fit
return func.fit(
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/engine/training_v2.py", line 313, in fit
training_result = run_one_epoch(
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/engine/training_v2.py", line 123, in run_one_epoch
batch_outs = execution_function(iterator)
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/keras/engine/training_v2_utils.py",
line
86, in execution_function
distributed_function(input_fn))
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/def_function.py", line 457, in __call__
result = self._call(*args, **kwds)
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/def_function.py", line 520, in _call
return self._stateless_fn(*args, **kwds)
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/function.py", line 1823, in __call__
return graph_function._filtered_call(args, kwargs) # pylint:
disable=protected-access
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/function.py", line 1137, in _filtered_call
return self._call_flat(
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/function.py", line 1223, in _call_flat
flat_outputs = forward_function.call(
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/function.py", line 506, in call
outputs = execute.execute(
File "/usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/eager/execute.py", line 67, in quick_execute
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError:
Conv2DCustomBackpropInputOp only supports NHWC.
[[node Conv2DBackpropInput (defined at /usr/pkg/lib/python3.8/site-
packages/tensorflow_core/python/framework/ops.py:1751) ]] [Op:__inference_distributed_function_2250]
Function call stack:
distributed_function
I also wanted to add that when the code is run on a GPU as opposed to a CPU, the error occurs on the same line as before but the message changes to :
File "DL_Ensemble.py", line 166, in <module>
model.fit([train_sequences,train_sequences], train_y, epochs=10, verbose=False, batch_size=32, class_weight={0: 6.0, 1: 1.0, 2: 2.0})
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 880, in fit
validation_steps=validation_steps)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 329, in model_iteration
batch_outs = f(ins_batch)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3073, in __call__
self._make_callable(feed_arrays, feed_symbols, symbol_vals, session)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3019, in _make_callable
callable_fn = session._make_callable_from_options(callable_opts)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1471, in _make_callable_from_options
return BaseSession._Callable(self, callable_options)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1425, in __init__
session._session, options_ptr, status)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/framework/errors_impl.py", line 528, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: Conv2DCustomBackpropInputOp only supports NHWC.
[[{{node training/Adagrad/gradients/conv1d_5/conv1d/Conv2D_grad/Conv2DBackpropInput}}]]
Exception ignored in: <function BaseSession._Callable.__del__ at 0x7fe4dd06a730>
Traceback (most recent call last):
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1455, in __del__
self._session._session, self._handle, status)
File "/home/kosimadukwe/.local/lib/python3.7/site-packages/tensorflow/python/framework/errors_impl.py", line 528, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: No such callable handle: 94697914208640
So from you stack trace, code is throwing error at:
fused = concatenate([graph, graph_1], axis= 1 )
print(type(graph))
# output: <class 'tensorflow.python.keras.engine.training.Model'>
This error is coming because concatenate expects list of tensors to be concatenated. While you are passing graph and graph_1 which is not tensor but a Model instance.
So from your code I assume that you want to concatenate output of these two models. In that case you'll have to change above line to:
fused = concatenate([graph.outputs[0], graph_1.outputs[0]], axis=-1)
Here, graph.outputs gives list of outputs by given by Model. Since each model is giving us one output, we will take 0th index from each output.
Change this part and you'll get model summary as you are expecting.
I compiled tensorflow from source with MKL in order to accelerate my DNN learning progress. And I have a ResNet model which is copy from tensorflow/models. The dataset is CIFAR-10. When I run the model with channels last format, everything goes ok. But in order to use MKL which is said only accelerate only for channels first format, I add some code to transpose the data into NCHW format, and run it. Then I get:
Caused by op 'stage/residual_v1/conv2d/Conv2D', defined at:
File "main.py", line 182, in <module>
main(args)
File "main.py", line 83, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
File "/home/holmescn/.pyenv/versions/anaconda35.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 447, in train_and_evaluate
return executor.run()
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 531, in run
return self.run_local()
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/training.py", line 669, in run_local
hooks=train_hooks)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 366, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1119, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1132, in _train_model_default
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1107, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/home/holmescn/Work/deep-learning-practice/tensorflow/estimator/utils.py", line 18, in _model_fn
logits = build_model(input_layer, mode == tf.estimator.ModeKeys.TRAIN, params=params, args=args)
File "/home/holmescn/Work/deep-learning-practice/tensorflow/estimator/estimators/resnet.py", line 175, in build_model
return resnet.build_model(input_layer, args.num_layers)
File "/home/holmescn/Work/deep-learning-practice/tensorflow/estimator/estimators/resnet.py", line 56, in build_model
x = res_func(x, 3, filters[i], filters[i + 1], strides[i])
File "/home/holmescn/Work/deep-learning-practice/tensorflow/estimator/estimators/resnet.py", line 79, in _residual_v1
x = self._conv(x, kernel_size, out_filter, stride)
File "/home/holmescn/Work/deep-learning-practice/tensorflow/estimator/estimators/base.py", line 59, in _conv
name=name)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/layers/convolutional.py", line 427, in conv2d
return layer.apply(inputs)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 759, in apply
return self.__call__(inputs, *args, **kwargs)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/layers/base.py", line 329, in __call__
outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py", line 688, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/keras/layers/convolutional.py", line 184, in call
outputs = self._convolution_op(inputs, self.kernel)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 868, in __call__
return self.conv_op(inp, filter)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 520, in __call__
return self.call(inp, filter)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 204, in __call__
name=self.name)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 956, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3414, in create_op
op_def=op_def)
File "/home/holmescn/.pyenv/versions/anaconda3-5.2.0/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1740, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): input and filter must have the same depth: 32 vs 16
[[Node: stage/residual_v1/conv2d/Conv2D = _MklConv2D[T=DT_FLOAT, _kernel="MklOp", data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](Relu, conv2d/kernel/read, Relu:1, DMT/_6)]]
the last trackback said input and filter must have the same depth, which IMO means the depth dim for both input tensor and filter should be same? But how could I do that if I want to generate more feature maps? What should I do?
No need to transpose the data for changing the dataformat.
You can pass the data format as channels first or channels last as argument
For example,
python cifar10_main.py --data-dir=${PWD}/cifar-10-data --data-format=channels_first --job-dir=/tmp/cifar10
I have two dataset in TFRecords, one holds around 20,000 entries, other hold 1.2 million.
This code perfectly work when I use TFRecord with 20,000 entries but give out of range error when I use 1.2 million .
def parse(serialized):
features = \
{
'train/image': tf.FixedLenFeature([], tf.string),
'train/label': tf.FixedLenFeature([], tf.int64)
}
parsed_example = tf.parse_single_example(serialized=serialized,
features=features)
image_raw = parsed_example['train/image']
image = tf.decode_raw(image_raw, tf.uint8)
image = tf.cast(image, tf.float32)
label = parsed_example['train/label']
return image, label
def input_fn(filenames, train, batch_size=32, buffer_size=2048):
dataset = tf.data.TFRecordDataset(filenames=filenames)
dataset = dataset.map(parse)
if train:
dataset = dataset.shuffle(buffer_size=buffer_size)
num_repeat = None
else:
num_repeat = 1
dataset = dataset.repeat(num_repeat)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
images_batch, labels_batch = iterator.get_next()
x = {'image':images_batch}
y = labels_batch
return x, y
x,y = input_fn('train.tfrecords',False)
print(x)
with tf.Session() as sess:
for i in range(10):
print(sess.run(x))
The error is coming is this:
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1374, in _do_call
raise type(e)(node_def, op, message)
OutOfRangeError: End of sequence
[[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]
Caused by op 'IteratorGetNext_2', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 268, in <module>
main()
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 264, in main
kernel.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 478, in start
self.io_loop.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2728, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2856, in run_ast_nodes
if self.run_code(code, result):
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2910, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-23a4ed6f3a2e>", line 1, in <module>
runfile('C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py', wdir='C:/Users/kakus/Desktop/landmark')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 84, in <module>
x,y = input_fn('train.tfrecords',False)
File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 76, in input_fn
images_batch, labels_batch = iterator.get_next()
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 330, in get_next
name=name)), self._output_types,
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 895, in iterator_get_next
output_shapes=output_shapes, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3271, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1650, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
OutOfRangeError (see above for traceback): End of sequence
[[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]
dataset.repeat(num_epochs) will repeat the data for the number of epochs specified. In the train mode you have specified it to be none, change it to the number of epochs you want to train the dataset.
I'm trying to read in data using the Tensorflow Dataset API. I have loaded filenames and label filenames into arrays which I load into a dataset. I then try to map these filenames to the actual image files, but get an error that seems to state that the input to the mapping function recieves placeholders rather than actual tensors.
class DatasetReader:
def __init__(self, records_list, batch_size=1):
self.batch_size = batch_size
self.records = {}
self.records["image"] = tf.convert_to_tensor([record['image'] for record in records_list])
self.records["filename"] = tf.convert_to_tensor([record['filename'] for record in records_list])
self.records["annotation"] = tf.convert_to_tensor([record['annotation'] for record in records_list])
self.dataset = Dataset.from_tensor_slices(self.records)
self.dataset = self.dataset.map(self._input_parser)
self.dataset = self.dataset.batch(batch_size)
self.dataset = self.dataset.repeat()
def _input_parser(self, record):
filename = record['filename']
image_name = record['image']
annotation_file = record['annotation']
image = tf.image.decode_image(tf.read_file(filename))
annotation = tf.image.decode_image(tf.read_file(annotation_file))
return self._augment_image(image, annotation)
The error I'm getting is in the line image = tf.image.decode_image(tf.read_file(filename)). The stack trace is below.
File "FCN.py", line 269, in <module>
tf.app.run()
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "FCN.py", line 179, in main
train_records, valid_records, image_options_train, image_options_val, FLAGS.batch_size, FLAGS.batch_size)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 89, in from_records
train_reader = DatasetReader(train_records, train_image_options, train_batch_size)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 34, in __init__
self.dataset = self.dataset.map(self._input_parser)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 964, in map
return MapDataset(self, map_func, num_threads, output_buffer_size)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 1735, in __init__
self._map_func.add_to_graph(ops.get_default_graph())
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/function.py", line 449, in add_to_graph
self._create_definition_if_needed()
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/framework/function.py", line 168, in _create_definition_if_needed
outputs = self._func(*inputs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/ops/dataset_ops.py", line 1723, in tf_map_func
ret = map_func(nested_args)
File "/home/ubuntu/FCN.tensorflow/TFReader.py", line 42, in _input_parser
image = tf.image.decode_image(tf.read_file(filename))
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_io_ops.py", line 223, in read_file
result = _op_def_lib.apply_op("ReadFile", filename=filename, name=name)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
op_def=op_def)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/data/python/framework/function.py", line 80, in create_op
data_types, **kwargs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/function.py", line 665, in create_op
**kwargs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2632, in create_op
set_shapes_for_outputs(ret)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1911, in set_shapes_for_outputs
shapes = shape_func(op)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1861, in call_with_requiring
return call_cpp_shape_fn(op, require_shape_fn=True)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 595, in call_cpp_shape_fn
require_shape_fn)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.py", line 659, in _call_cpp_shape_fn_impl
raise ValueError(err.message)
ValueError: Shape must be rank 0 but is rank 1 for 'ReadFile' (op: 'ReadFile') with input shapes: [?].
You cannot pass in a rank-1 tensor to tf.read_file. Here are some examples:
import tensorflow as tf
# Correct: input can be a string.
tf.image.decode_image(tf.read_file("filename"))
# Correct: input can be a rank-0 tensor.
tf.image.decode_image(tf.read_file(tf.convert_to_tensor("filename")))
# Wrong: input cannot be a list.
tf.image.decode_image(tf.read_file(["filename"]))
# Wrong: input cannot be a rank-1 tensor
tf.image.decode_image(tf.read_file(tf.convert_to_tensor(["filename"])))
In your code, it seems like self.records["filename"] is a rank-1 tensor; you might mistakenly passed it as a parameter to tf.read_file in _input_parser