I want to test a new network structure which requires changing some of the elements of a tensor in a keras model. If I could find a way to convert/copy the tensor to a numpy array and then later transform it back into a tensor, then I should be able to make the model work.
I tried using the .eval() method to convert the tensor to a numpy array, but it gives me errors. I am also using this model with a DQN Agent from keras-rl, so it is possible the error is from how keras-rl uses the model. Here is my code:
def Filter_Features(F):
sess = Session()
with sess.as_default():
F_np = F.eval()
min_pos = np.argmin(F_np)
F_np[min_pos] = 0
return convert_to_tensor(F_np)
def create_model(nb_actions, num_frames=4):
inputs = Input(shape = (num_frames,84,84))
F = Conv2D(16,(8,8), activation='relu', strides=(4,4), data_format = "channels_first")(inputs)
...
F_k = Lambda(Filter_Features)(F)
actions = Dense(nb_actions, activation = 'linear')(F_k)
nnf_model = Model(inputs = inputs, outputs = actions)
return nnf_model
Note that the code runs if I remove the lambda layer, so the issue must be originating there. I received the error:
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1356, in _do_call
return fn(*args)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1341, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value conv2d_1/bias
[[{{node conv2d_1/bias/read}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "Atari_Test.py", line 32, in <module>
model = Model_StackExchange.create_model(nb_actions = nb_actions)
File "/Users/j/deep-rl/Model_StackExchange.py", line 26, in create_model
F_k = Lambda(Filter_Features)(F)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/base_layer.py", line 457, in __call__
output = self.call(inputs, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/layers/core.py", line 687, in call
return self.function(inputs, **arguments)
File "/Users/j/deep-rl/Model_StackExchange.py", line 11, in Filter_Features
F_np = F.eval()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 731, in eval
return _eval_using_default_session(self, feed_dict, self.graph, session)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 5579, in _eval_using_default_session
return session.run(tensors, feed_dict)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 950, in run
run_metadata_ptr)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1173, in _run
feed_dict_tensor, options, run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1350, in _do_run
run_metadata)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1370, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value conv2d_1/bias
[[node conv2d_1/bias/read (defined at /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:402) ]]
Original stack trace for 'conv2d_1/bias/read':
File "Atari_Test.py", line 32, in <module>
model = Model_StackExchange.create_model(nb_actions = nb_actions)
File "/Users/j/deep-rl/Model_StackExchange.py", line 21, in create_model
F = Conv2D(16,(8,8), activation='relu', strides=(4,4), data_format = "channels_first")(inputs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/base_layer.py", line 431, in __call__
self.build(unpack_singleton(input_shapes))
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/layers/convolutional.py", line 147, in build
constraint=self.bias_constraint)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/base_layer.py", line 252, in add_weight
constraint=constraint)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py", line 402, in variable
v = tf.Variable(value, dtype=tf.as_dtype(dtype), name=name)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 259, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 220, in _variable_v1_call
shape=shape)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 198, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py", line 2511, in default_variable_creator
shape=shape)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 263, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 1568, in __init__
shape=shape)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 1755, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py", line 180, in wrapper
return target(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py", line 86, in identity
ret = gen_array_ops.identity(input, name=name)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 4253, in identity
"Identity", input=input, name=name)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3616, in create_op
op_def=op_def)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
Please let me know if you know how to access and change the elements of a tensor within a keras model. Thank you.
I am trying to build a custom object detection model using the Tensorflow Object Detection API. Specifically, I want to add a distance estimation to each detected object. So, I extended the SSDMetaArch class and adapted/added some crucial parts. Among other things:
added distance to the input
adapted protos to handle distance
created a new prediction head for distance estimations
created new loss taking distances into account
adapted postprocessing step to output distances
After all these steps the training seems to work and the losses (including the distance loss) go down.
But when I want to evaluate the model I get a
RuntimeError: Graph is finalized and cannot be modified.
Specifically the following line in the (adapted) visualize_boxes_and_labels_on_image_array() method in visualization_utils.py (around line 700) causes the error
#the follwing line throws the error
display_str = '{}: {}% -> {}'.format(display_str, int(100 * scores[i]), int(distances[i]))
#No error is thrown using the original code
#display_str = '{}: {}%'.format(display_str, int(100 * scores[i]))
It seems that just by printing the value the above exception is thrown.
My Question: I do not understand why printing a value will modify the graph. Could you help me understand why this is the case and if possible give me hint how I can avoid this modification to the graph.
Find the complete error below
INFO:tensorflow:Restoring parameters from E:/Data/OD_DE/tensorflow/models/model\model.ckpt-5103
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
2019-04-01 13:14:30.795741: W tensorflow/core/framework/op_kernel.cc:1261]
Unknown: RuntimeError: Graph is finalized and cannot be modified.
Traceback (most recent call last):
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 206, in __call__
ret = func(*args)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 271, in _visualize_boxes
image, boxes, classes, scores, category_index=category_index, **kwargs)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 730, in visualize_boxes_and_labels_on_image_array
print("We were definitely here:" + str(int(distances[i])))
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 525, in _slice_helper
name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 6037, in __exit__
self._name_scope.__exit__(type_arg, value_arg, traceback_arg)
File "C:\Users\JD\AppData\Local\Continuum\Anaconda3\Lib\contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 4016, in name_scope
yield "" if new_stack is None else new_stack + "/"
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 500, in _slice_helper
packed_begin, packed_end, packed_strides = (stack(begin), stack(end),
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 863, in stack
return ops.convert_to_tensor(values, name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1050, in convert_to_tensor
as_ref=False)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1146, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 229, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 214, in constant
name=name).outputs[0]
File "..\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 3246, in create_op
self._check_not_finalized()
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 2919, in _check_not_finalized
raise RuntimeError("Graph is finalized and cannot be modified.")
RuntimeError: Graph is finalized and cannot be modified.
Traceback (most recent call last):
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1334, in _do_call
return fn(*args)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.UnknownError: RuntimeError: Graph is finalized and cannot be modified.
Traceback (most recent call last):
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 206, in __call__
ret = func(*args)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 271, in _visualize_boxes
image, boxes, classes, scores, category_index=category_index, **kwargs)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 730, in visualize_boxes_and_labels_on_image_array
print("We were definitely here:" + str(int(distances[i])))
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 525, in _slice_helper
name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 6037, in __exit__
self._name_scope.__exit__(type_arg, value_arg, traceback_arg)
File "C:\Users\JD\AppData\Local\Continuum\Anaconda3\Lib\contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 4016, in name_scope
yield "" if new_stack is None else new_stack + "/"
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 500, in _slice_helper
packed_begin, packed_end, packed_strides = (stack(begin), stack(end),
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 863, in stack
return ops.convert_to_tensor(values, name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1050, in convert_to_tensor
as_ref=False)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1146, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 229, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 214, in constant
name=name).outputs[0]
File "..\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 3246, in create_op
self._check_not_finalized()
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 2919, in _check_not_finalized
raise RuntimeError("Graph is finalized and cannot be modified.")
RuntimeError: Graph is finalized and cannot be modified.
[[{{node map_1/while/PyFunc}} = PyFunc[Tin=[DT_UINT8, DT_FLOAT, DT_INT64, DT_FLOAT], Tout=[DT_UINT8], _class=["loc:#map_1/while/TensorArrayWrite/TensorArrayWriteV3"], token="pyfunc_1", _device="/job:localhost/replica:0/task:0/device:CPU:0"](map_1/while/Squeeze/_2907, map_1/while/TensorArrayReadV3_3/_2909, map_1/while/TensorArrayReadV3_4/_2911, map_1/while/TensorArrayReadV3_5/_2913)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\pydevd.py", line 1596, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\pydevd.py", line 1023, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/JD/tensorflowod/Lib/site-packages/tensorflow/models/research/object_detection/model_main.py", line 110, in <module>
tf.app.run()
File "..\lib\site-packages\tensorflow\python\platform\app.py", line 125, in run
_sys.exit(main(argv))
File "C:/Users/JD/tensorflowod/Lib/site-packages/tensorflow/models/research/object_detection/model_main.py", line 105, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 471, in train_and_evaluate
return executor.run()
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 610, in run
return self.run_local()
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 711, in run_local
saving_listeners=saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1241, in _train_model_default
saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1471, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 671, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1156, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1255, in run
raise six.reraise(*original_exc_info)
File "c:\users\JD\appdata\local\continuum\anaconda3\lib\site-packages\six.py", line 686, in reraise
raise value
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1240, in run
return self._sess.run(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1320, in run
run_metadata=run_metadata))
File "..\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py", line 582, in after_run
if self._save(run_context.session, global_step):
File "..\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py", line 607, in _save
if l.after_save(session, step):
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 517, in after_save
self._evaluate(global_step_value) # updates self.eval_result
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 537, in _evaluate
self._evaluator.evaluate_and_export())
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 912, in evaluate_and_export
hooks=self._eval_spec.hooks)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 478, in evaluate
return _evaluate()
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 467, in _evaluate
output_dir=self.eval_dir(name))
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1591, in _evaluate_run
config=self._session_config)
File "..\lib\site-packages\tensorflow\python\training\evaluation.py", line 274, in _evaluate_once
session.run(eval_ops, feed_dict)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 671, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1156, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1255, in run
raise six.reraise(*original_exc_info)
File "c:\users\JD\appdata\local\continuum\anaconda3\lib\site-packages\six.py", line 686, in reraise
raise value
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1240, in run
return self._sess.run(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1312, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1076, in run
return self._sess.run(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 929, in run
run_metadata_ptr)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1328, in _do_run
run_metadata)
File "..\lib\site-packages\tensorflow\python\client\session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.UnknownError: RuntimeError: Graph is finalized and cannot be modified.
Traceback (most recent call last):
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 206, in __call__
ret = func(*args)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 271, in _visualize_boxes
image, boxes, classes, scores, category_index=category_index, **kwargs)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 730, in visualize_boxes_and_labels_on_image_array
print("We were definitely here:" + str(int(distances[i])))
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 525, in _slice_helper
name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 6037, in __exit__
self._name_scope.__exit__(type_arg, value_arg, traceback_arg)
File "C:\Users\JD\AppData\Local\Continuum\Anaconda3\Lib\contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 4016, in name_scope
yield "" if new_stack is None else new_stack + "/"
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 500, in _slice_helper
packed_begin, packed_end, packed_strides = (stack(begin), stack(end),
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 863, in stack
return ops.convert_to_tensor(values, name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1050, in convert_to_tensor
as_ref=False)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1146, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 229, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 214, in constant
name=name).outputs[0]
File "..\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 3246, in create_op
self._check_not_finalized()
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 2919, in _check_not_finalized
raise RuntimeError("Graph is finalized and cannot be modified.")
RuntimeError: Graph is finalized and cannot be modified.
[[node map_1/while/PyFunc (defined at ..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py:439) = PyFunc[Tin=[DT_UINT8, DT_FLOAT, DT_INT64, DT_FLOAT], Tout=[DT_UINT8], _class=["loc:#map_1/while/TensorArrayWrite/TensorArrayWriteV3"], token="pyfunc_1", _device="/job:localhost/replica:0/task:0/device:CPU:0"](map_1/while/Squeeze/_2907, map_1/while/TensorArrayReadV3_3/_2909, map_1/while/TensorArrayReadV3_4/_2911, map_1/while/TensorArrayReadV3_5/_2913)]]
Caused by op 'map_1/while/PyFunc', defined at:
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\pydevd.py", line 1596, in <module>
globals = debugger.run(setup['file'], None, None, is_module)
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\pydevd.py", line 1023, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "C:\Program Files\JetBrains\PyCharm Community Edition 2017.2\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/JD/tensorflowod/Lib/site-packages/tensorflow/models/research/object_detection/model_main.py", line 110, in <module>
tf.app.run()
File "..\lib\site-packages\tensorflow\python\platform\app.py", line 125, in run
_sys.exit(main(argv))
File "C:/Users/JD/tensorflowod/Lib/site-packages/tensorflow/models/research/object_detection/model_main.py", line 105, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 471, in train_and_evaluate
return executor.run()
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 610, in run
return self.run_local()
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 711, in run_local
saving_listeners=saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 354, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1207, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1241, in _train_model_default
saving_listeners)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1471, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 671, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1156, in run
run_metadata=run_metadata)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1240, in run
return self._sess.run(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\training\monitored_session.py", line 1320, in run
run_metadata=run_metadata))
File "..\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py", line 582, in after_run
if self._save(run_context.session, global_step):
File "..\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py", line 607, in _save
if l.after_save(session, step):
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 517, in after_save
self._evaluate(global_step_value) # updates self.eval_result
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 537, in _evaluate
self._evaluator.evaluate_and_export())
File "..\lib\site-packages\tensorflow\python\estimator\training.py", line 912, in evaluate_and_export
hooks=self._eval_spec.hooks)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 478, in evaluate
return _evaluate()
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 460, in _evaluate
self._evaluate_build_graph(input_fn, hooks, checkpoint_path))
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1484, in _evaluate_build_graph
self._call_model_fn_eval(input_fn, self.config))
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1520, in _call_model_fn_eval
features, labels, model_fn_lib.ModeKeys.EVAL, config)
File "..\lib\site-packages\tensorflow\python\estimator\estimator.py", line 1195, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\model_lib.py", line 443, in model_fn
eval_dict)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 931, in get_estimator_eval_metric_ops
images = self.images_from_evaluation_dict(eval_dict)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 995, in images_from_evaluation_dict
self._min_score_thresh, self._use_normalized_coordinates)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 547, in draw_side_by_side_evaluation_image
eval_dict[input_data_fields.groundtruth_distances][indx], axis=0) if input_data_fields.groundtruth_distances in eval_dict else None
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 442, in draw_bounding_boxes_on_image_tensors
images = tf.map_fn(draw_boxes, elems, dtype=tf.uint8, back_prop=False)
File "..\lib\site-packages\tensorflow\python\ops\functional_ops.py", line 494, in map_fn
maximum_iterations=n)
File "..\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3291, in while_loop
return_same_structure)
File "..\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3004, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "..\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2939, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "..\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3260, in <lambda>
body = lambda i, lv: (i + 1, orig_body(*lv))
File "..\lib\site-packages\tensorflow\python\ops\functional_ops.py", line 483, in compute
packed_fn_values = fn(packed_values)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 439, in draw_boxes
tf.uint8)
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 457, in py_func
func=func, inp=inp, Tout=Tout, stateful=stateful, eager=False, name=name)
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 281, in _internal_py_func
input=inp, token=token, Tout=Tout, name=name)
File "..\lib\site-packages\tensorflow\python\ops\gen_script_ops.py", line 132, in py_func
"PyFunc", input=input, token=token, Tout=Tout, name=name)
File "..\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "..\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 3274, in create_op
op_def=op_def)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1770, in __init__
self._traceback = tf_stack.extract_stack()
UnknownError (see above for traceback): RuntimeError: Graph is finalized and cannot be modified.
Traceback (most recent call last):
File "..\lib\site-packages\tensorflow\python\ops\script_ops.py", line 206, in __call__
ret = func(*args)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 271, in _visualize_boxes
image, boxes, classes, scores, category_index=category_index, **kwargs)
File "..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py", line 730, in visualize_boxes_and_labels_on_image_array
print("We were definitely here:" + str(int(distances[i])))
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 525, in _slice_helper
name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 6037, in __exit__
self._name_scope.__exit__(type_arg, value_arg, traceback_arg)
File "C:\Users\JD\AppData\Local\Continuum\Anaconda3\Lib\contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 4016, in name_scope
yield "" if new_stack is None else new_stack + "/"
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 500, in _slice_helper
packed_begin, packed_end, packed_strides = (stack(begin), stack(end),
File "..\lib\site-packages\tensorflow\python\ops\array_ops.py", line 863, in stack
return ops.convert_to_tensor(values, name=name)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1050, in convert_to_tensor
as_ref=False)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 1146, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 229, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "..\lib\site-packages\tensorflow\python\framework\constant_op.py", line 214, in constant
name=name).outputs[0]
File "..\lib\site-packages\tensorflow\python\util\deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 3246, in create_op
self._check_not_finalized()
File "..\lib\site-packages\tensorflow\python\framework\ops.py", line 2919, in _check_not_finalized
raise RuntimeError("Graph is finalized and cannot be modified.")
RuntimeError: Graph is finalized and cannot be modified.
[[node map_1/while/PyFunc (defined at ..\Lib\site-packages\tensorflow\models\research\object_detection\utils\visualization_utils.py:439) = PyFunc[Tin=[DT_UINT8, DT_FLOAT, DT_INT64, DT_FLOAT], Tout=[DT_UINT8], _class=["loc:#map_1/while/TensorArrayWrite/TensorArrayWriteV3"], token="pyfunc_1", _device="/job:localhost/replica:0/task:0/device:CPU:0"](map_1/while/Squeeze/_2907, map_1/while/TensorArrayReadV3_3/_2909, map_1/while/TensorArrayReadV3_4/_2911, map_1/while/TensorArrayReadV3_5/_2913)]]
It appears that in your case distances is a tensor and calling distances[0] attempts to create an array_slice operation on a finalized graph. The right way of handling that would be to feed distances tensor to your distances_np = session.run(distances) or call distances_np = distances.eval() providing necessary parameters to acquire some sort of numpy array and only then do indexing and/or printing on distances_np.
cuda 9.0
cudnn 7.5
python 3.5.2
tensorflow-gpu 1.8
I don't know where the error occurred, I also tried python 3.6.3. This error will also occur. Please help.
I am training model_main.py file, but I get the following error.
python model_main.py --model_dir=F:/cindy/cindybackup/tensorflow1/test/training -pipeline_config_path=F:/cindy/cindybackup/tensorflow1/test/data/faster_rcnn_inception_v2_pets.config --alsologtostderr --num_train_steps=1000 --num_eval_steps=10
It shows the following:
WARNING:tensorflow:Forced number of epochs for all eval validations to
be 1.
WARNING:tensorflow:Expected number of evaluation epochs is 1, but instead encountered eval_on_train_input_config.num_epochs = 0.
Overwriting num_epochs to 1.
WARNING:tensorflow:Using temporary folder as model directory: C:\Users\wyh\AppData\Local\Temp\tmplh3q4jn2
WARNING:tensorflow:Estimator's model_fn (.model_fn at 0x00000256FF7F1400>) includes
params argument, but params are not passed to Estimator.
WARNING:tensorflow:num_readers has been reduced to 1 to match input file shards.
Traceback (most recent call last):
File "model_main.py", line 109, in
tf.app.run()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\platform\app.py",
line 126, in run
_sys.exit(main(argv))
File "model_main.py", line 105, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 439, in train_and_evaluate
executor.run()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 518, in run
self.run_local()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 650, in run_local
hooks=train_hooks)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 363, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 843, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 853, in _train_model_default
input_fn, model_fn_lib.ModeKeys.TRAIN))
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 691, in _get_features_and_labels_from_input_fn
result = self._call_input_fn(input_fn, mode)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 798, in _call_input_fn
return input_fn(**kwargs)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 525, in _train_input_fn
batch_size=params['batch_size'] if params else train_config.batch_size)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\builders\dataset_builder.py",
line 149, in build
dataset = data_map_fn(process_fn, num_parallel_calls=num_parallel_calls)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 853, in map
return ParallelMapDataset(self, map_func, num_parallel_calls)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1870, in init
super(ParallelMapDataset, self).init(input_dataset, map_func)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1839, in init
self._map_func.add_to_graph(ops.get_default_graph())
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 484, in add_to_graph
self._create_definition_if_needed()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 319, in _create_definition_if_needed
self._create_definition_if_needed_impl()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 336, in _create_definition_if_needed_impl
outputs = self._func(*inputs)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1804, in tf_map_func
ret = map_func(nested_args)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\builders\dataset_builder.py",
line 130, in process_fn
processed_tensors = transform_input_data_fn(processed_tensors)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 515, in transform_and_pad_input_data_fn
tensor_dict=transform_data_fn(tensor_dict),
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 129, in transform_input_data
tf.expand_dims(tf.to_float(image), axis=0))
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\meta_architectures\faster_rcnn_meta_arch.py",
line 543, in preprocess
parallel_iterations=self._parallel_iterations)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\utils\shape_utils.py",
line 237, in static_or_dynamic_map_fn
outputs = [fn(arg) for arg in tf.unstack(elems)]
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\utils\shape_utils.py",
line 237, in
outputs = [fn(arg) for arg in tf.unstack(elems)]
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2264, in resize_to_range
lambda: _resize_portrait_image(image))
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\util\deprecation.py",
line 432, in new_func
return func(*args, **kwargs)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\ops\control_flow_ops.py",
line 2063, in cond
orig_res_t, res_t = context_t.BuildCondBranch(true_fn)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\ops\control_flow_ops.py",
line 1913, in BuildCondBranch
original_result = fn()
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2263, in
lambda: _resize_landscape_image(image),
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2245, in _resize_landscape_image
align_corners=align_corners, preserve_aspect_ratio=True)
TypeError: resize_images() got an unexpected keyword argument 'preserve_aspect_ratio'
Thanks~
The problem is not resolved yet in tensorflow models(https://github.com/tensorflow/models/)
I just removed the preserve_aspect_ratio in object_detection/core/preprocessor.py
align_corners=align_corners, preserve_aspect_ratio=True)
align_corners=align_corners)
I have two dataset in TFRecords, one holds around 20,000 entries, other hold 1.2 million.
This code perfectly work when I use TFRecord with 20,000 entries but give out of range error when I use 1.2 million .
def parse(serialized):
features = \
{
'train/image': tf.FixedLenFeature([], tf.string),
'train/label': tf.FixedLenFeature([], tf.int64)
}
parsed_example = tf.parse_single_example(serialized=serialized,
features=features)
image_raw = parsed_example['train/image']
image = tf.decode_raw(image_raw, tf.uint8)
image = tf.cast(image, tf.float32)
label = parsed_example['train/label']
return image, label
def input_fn(filenames, train, batch_size=32, buffer_size=2048):
dataset = tf.data.TFRecordDataset(filenames=filenames)
dataset = dataset.map(parse)
if train:
dataset = dataset.shuffle(buffer_size=buffer_size)
num_repeat = None
else:
num_repeat = 1
dataset = dataset.repeat(num_repeat)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
images_batch, labels_batch = iterator.get_next()
x = {'image':images_batch}
y = labels_batch
return x, y
x,y = input_fn('train.tfrecords',False)
print(x)
with tf.Session() as sess:
for i in range(10):
print(sess.run(x))
The error is coming is this:
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1374, in _do_call
raise type(e)(node_def, op, message)
OutOfRangeError: End of sequence
[[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]
Caused by op 'IteratorGetNext_2', defined at:
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 268, in <module>
main()
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\ipython\start_kernel.py", line 264, in main
kernel.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelapp.py", line 478, in start
self.io_loop.start()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\ProgramData\Anaconda3\lib\site-packages\ipykernel\zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2728, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2856, in run_ast_nodes
if self.run_code(code, result):
File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 2910, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-23a4ed6f3a2e>", line 1, in <module>
runfile('C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py', wdir='C:/Users/kakus/Desktop/landmark')
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
execfile(filename, namespace)
File "C:\ProgramData\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 84, in <module>
x,y = input_fn('train.tfrecords',False)
File "C:/Users/kakus/Desktop/landmark/tfrecord_test_outputv2.py", line 76, in input_fn
images_batch, labels_batch = iterator.get_next()
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\data\ops\iterator_ops.py", line 330, in get_next
name=name)), self._output_types,
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_dataset_ops.py", line 895, in iterator_get_next
output_shapes=output_shapes, name=name)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3271, in create_op
op_def=op_def)
File "C:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 1650, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
OutOfRangeError (see above for traceback): End of sequence
[[Node: IteratorGetNext_2 = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_FLOAT, DT_INT64], _device="/job:localhost/replica:0/task:0/device:CPU:0"](OneShotIterator_2)]]
dataset.repeat(num_epochs) will repeat the data for the number of epochs specified. In the train mode you have specified it to be none, change it to the number of epochs you want to train the dataset.