Develop Reference

Can't load the model from checkpoint

Using object detection API and mask-rcnn from the model zoo, when I start training with the default command:
python model_main_tf2.py --pipeline_config_path=pipeline.config --model_dir=model/v1/
I get this error
Two checkpoint references resolved to different objects (<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x7f5e94075c10> and <tensorflow.python.keras.layers.convolutional.Conv2D object at 0x7f5e9403b710>).
Traceback (most recent call last):
File "model_main_tf2.py", line 115, in <module>
tf.compat.v1.app.run()
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/opt/anaconda/envs/tensorflow2.3/lib/python3.7/site-packages/absl/app.py", line 300, in run
_run_main(main, args)
File "/opt/anaconda/envs/tensorflow2.3/lib/python3.7/site-packages/absl/app.py", line 251, in _run_main
sys.exit(main(argv))
File "/home/TF/workspace/Antenna/model_main_tf2.py", line 112, in main
record_summaries=FLAGS.record_summaries)
File "/home/TF/models/research/object_detection/model_lib_v2.py", line 603, in train_loop
train_input, unpad_groundtruth_tensors)
File "/home/TF/models/research/object_detection/model_lib_v2.py", line 401, in load_fine_tune_checkpoint
checkpoint_path).expect_partial().assert_existing_objects_matched()
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/util.py", line 1721, in restore
status = self._saver.restore(save_path=save_path)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/util.py", line 1320, in restore
checkpoint=checkpoint, proto_id=0).restore(self._graph_view.root)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py", line 209, in restore
restore_ops = trackable._restore_from_checkpoint_position(self) # pylint: disable=protected-access
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py", line 914, in _restore_from_checkpoint_position
tensor_saveables, python_saveables))
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/tracking/util.py", line 297, in restore_saveables
validated_saveables).restore(self.save_path_tensor, self.options)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/saving/functional_saver.py", line 340, in restore
restore_ops = restore_fn()
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/saving/functional_saver.py", line 316, in restore_fn
restore_ops.update(saver.restore(file_prefix, options))
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/training/saving/functional_saver.py", line 111, in restore
restored_tensors, restored_shapes=None)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/distribute/values.py", line 890, in restore
for v in self._mirrored_variable.values))
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/distribute/values.py", line 890, in <genexpr>
for v in self._mirrored_variable.values))
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/distribute/values_util.py", line 195, in assign_on_device
return variable.assign(tensor)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 858, in assign
self._shape.assert_is_compatible_with(value_tensor.shape)
File "/home/.local/lib/python3.7/site-packages/tensorflow/python/framework/tensor_shape.py", line 1134, in assert_is_compatible_with
raise ValueError("Shapes %s and %s are incompatible" % (self, other))
ValueError: Shapes (1, 1, 1088, 256) and (1, 1, 1024, 512) are incompatible
I use the default config from tf-api models/research/object-detection/configs
I have already tried to fix it by using config-file from the model zoo or fine_tune_checkpoint_type: "detection"

The input shape in the pipeline.config file doesn't match the one from the model checkpoint

IndexError in random.shuffle when class is executed concurrently

I have created a custom environment for reinforcement learning with tf-agents (not needed to answer this question), which works fine if I instantiate one thread by setting num_parallel_environments to 1, but throws infrequent and seemingly random errors like an IndexError inside random.shuffle(), when I increase num_parallel_environments to 50. Here's the code:
inside train.py
tf_env = tf_py_environment.TFPyEnvironment(
batched_py_environment.BatchedPyEnvironment(
[environment.CardGameEnv()] * num_parallel_environments))
inside my environment, this is run in threads
self.cardStack = getFullDeck()
random.shuffle(self.cardStack)
this is a normal function, imported in every thread class
def getFullDeck():
deck = []
for rank in Ranks:
for suit in Suits:
deck.append(Card(rank, suit))
return deck
And here's one of the possible errors:
Traceback (most recent call last):
File "e:\Users\tmp\.vscode\extensions\ms-python.python-2019.1.0\pythonFiles\ptvsd_launcher.py", line 45, in <module>
main(ptvsdArgs)
File "e:\Users\tmp\.vscode\extensions\ms-python.python-2019.1.0\pythonFiles\lib\python\ptvsd\__main__.py", line 348, in main
run()
File "e:\Users\tmp\.vscode\extensions\ms-python.python-2019.1.0\pythonFiles\lib\python\ptvsd\__main__.py", line 253, in run_file
runpy.run_path(target, run_name='__main__')
File "C:\Python37\lib\runpy.py", line 263, in run_path
pkg_name=pkg_name, script_name=fname)
File "C:\Python37\lib\runpy.py", line 96, in _run_module_code
mod_name, mod_spec, pkg_name, script_name)
File "C:\Python37\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "e:\Users\tmp\Documents\Programming\Neural Nets\Poker_AI\train_v2.py", line 320, in <module>
app.run(main)
File "C:\Python37\lib\site-packages\absl\app.py", line 300, in run
_run_main(main, args)
File "C:\Python37\lib\site-packages\absl\app.py", line 251, in _run_main
sys.exit(main(argv))
File "e:\Users\tmp\Documents\Programming\Neural Nets\Poker_AI\train_v2.py", line 315, in main
num_eval_episodes=FLAGS.num_eval_episodes)
File "E:\Users\tmp\AppData\Roaming\Python\Python37\site-packages\gin\config.py", line 1032, in wrapper
utils.augment_exception_message_and_reraise(e, err_str)
File "E:\Users\tmp\AppData\Roaming\Python\Python37\site-packages\gin\utils.py", line 49, in augment_exception_message_and_reraise
six.raise_from(proxy.with_traceback(exception.__traceback__), None)
File "<string>", line 3, in raise_from
File "E:\Users\tmp\AppData\Roaming\Python\Python37\site-packages\gin\config.py", line 1009, in wrapper
return fn(*new_args, **new_kwargs)
File "e:\Users\tmp\Documents\Programming\Neural Nets\Poker_AI\train_v2.py", line 251, in train_eval
collect_driver.run()
File "C:\Python37\lib\site-packages\tf_agents\drivers\dynamic_episode_driver.py", line 149, in run
maximum_iterations=maximum_iterations)
File "C:\Python37\lib\site-packages\tf_agents\utils\common.py", line 111, in with_check_resource_vars
return fn(*fn_args, **fn_kwargs)
File "C:\Python37\lib\site-packages\tf_agents\drivers\dynamic_episode_driver.py", line 180, in _run
name='driver_loop'
File "C:\Python37\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2457, in while_loop_v2
return_same_structure=True)
File "C:\Python37\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2689, in while_loop
loop_vars = body(*loop_vars)
File "C:\Python37\lib\site-packages\tf_agents\drivers\dynamic_episode_driver.py", line 103, in loop_body
next_time_step = self.env.step(action_step.action)
File "C:\Python37\lib\site-packages\tf_agents\environments\tf_environment.py", line 232, in step
return self._step(action)
File "C:\Python37\lib\site-packages\tensorflow\python\autograph\impl\api.py", line 232, in graph_wrapper
return func(*args, **kwargs)
File "C:\Python37\lib\site-packages\tf_agents\environments\tf_py_environment.py", line 218, in _step
_step_py, flat_actions, self._time_step_dtypes, name='step_py_func')
File "C:\Python37\lib\site-packages\tensorflow\python\ops\script_ops.py", line 488, in numpy_function
return py_func_common(func, inp, Tout, stateful=True, name=name)
File "C:\Python37\lib\site-packages\tensorflow\python\ops\script_ops.py", line 452, in py_func_common
result = func(*[x.numpy() for x in inp])
File "C:\Python37\lib\site-packages\tf_agents\environments\tf_py_environment.py", line 203, in _step_py
self._time_step = self._env.step(packed)
File "C:\Python37\lib\site-packages\tf_agents\environments\py_environment.py", line 174, in step
self._current_time_step = self._step(action)
File "C:\Python37\lib\site-packages\tf_agents\environments\batched_py_environment.py", line 140, in _step
zip(self._envs, unstacked_actions))
File "C:\Python37\lib\multiprocessing\pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Python37\lib\multiprocessing\pool.py", line 657, in get
raise self._value
File "C:\Python37\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Python37\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Python37\lib\site-packages\tf_agents\environments\batched_py_environment.py", line 139, in <lambda>
lambda env_action: env_action[0].step(env_action[1]),
File "C:\Python37\lib\site-packages\tf_agents\environments\py_environment.py", line 174, in step
self._current_time_step = self._step(action)
File "e:\Users\tmp\Documents\Programming\Neural Nets\Poker_AI\environment.py", line 116, in _step
canRoundContinue = self._table.runUntilChoice(action)
File "e:\Users\tmp\Documents\Programming\Neural Nets\Poker_AI\table.py", line 326, in runUntilChoice
random.shuffle(self.cardStack)
File "C:\Python37\lib\random.py", line 278, in shuffle
x[i], x[j] = x[j], x[i]
IndexError: list index out of range
In call to configurable 'train_eval' (<function train_eval at 0x000002722713A158>)
I suspect this error occurs because the threads are changing the array simultaneously, but I do not see why this would be the case:
Everything happens inside a class instance and the array getFullDeck() is returning is recreated every time the function is called, so there should be no way multiple threads have access to the same reference, right?

tf_env = tf_py_environment.TFPyEnvironment(
batched_py_environment.BatchedPyEnvironment(
[environment.CardGameEnv()] * num_parallel_environments))
You are reusing the same environment for each of the parallel instances rather than creating a new environment for each one. You might want to try something like
tf_env = tf_py_environment.TFPyEnvironment(
batched_py_environment.BatchedPyEnvironment(
[environment.CardGameEnv() for _ in range(num_parallel_environments)]))

<Tensorflow object detection>TypeError: resize_images() got an unexpected keyword argument 'preserve_aspect_ratio'

cuda 9.0
cudnn 7.5
python 3.5.2
tensorflow-gpu 1.8
I don't know where the error occurred, I also tried python 3.6.3. This error will also occur. Please help.
I am training model_main.py file, but I get the following error.
python model_main.py --model_dir=F:/cindy/cindybackup/tensorflow1/test/training -pipeline_config_path=F:/cindy/cindybackup/tensorflow1/test/data/faster_rcnn_inception_v2_pets.config --alsologtostderr --num_train_steps=1000 --num_eval_steps=10
It shows the following:
WARNING:tensorflow:Forced number of epochs for all eval validations to
be 1.
WARNING:tensorflow:Expected number of evaluation epochs is 1, but instead encountered eval_on_train_input_config.num_epochs = 0.
Overwriting num_epochs to 1.
WARNING:tensorflow:Using temporary folder as model directory: C:\Users\wyh\AppData\Local\Temp\tmplh3q4jn2
WARNING:tensorflow:Estimator's model_fn (.model_fn at 0x00000256FF7F1400>) includes
params argument, but params are not passed to Estimator.
WARNING:tensorflow:num_readers has been reduced to 1 to match input file shards.
Traceback (most recent call last):
File "model_main.py", line 109, in
tf.app.run()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\platform\app.py",
line 126, in run
_sys.exit(main(argv))
File "model_main.py", line 105, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 439, in train_and_evaluate
executor.run()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 518, in run
self.run_local()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\training.py",
line 650, in run_local
hooks=train_hooks)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 363, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 843, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 853, in _train_model_default
input_fn, model_fn_lib.ModeKeys.TRAIN))
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 691, in _get_features_and_labels_from_input_fn
result = self._call_input_fn(input_fn, mode)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\estimator\estimator.py",
line 798, in _call_input_fn
return input_fn(**kwargs)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 525, in _train_input_fn
batch_size=params['batch_size'] if params else train_config.batch_size)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\builders\dataset_builder.py",
line 149, in build
dataset = data_map_fn(process_fn, num_parallel_calls=num_parallel_calls)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 853, in map
return ParallelMapDataset(self, map_func, num_parallel_calls)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1870, in init
super(ParallelMapDataset, self).init(input_dataset, map_func)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1839, in init
self._map_func.add_to_graph(ops.get_default_graph())
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 484, in add_to_graph
self._create_definition_if_needed()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 319, in _create_definition_if_needed
self._create_definition_if_needed_impl()
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\framework\function.py",
line 336, in _create_definition_if_needed_impl
outputs = self._func(*inputs)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py",
line 1804, in tf_map_func
ret = map_func(nested_args)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\builders\dataset_builder.py",
line 130, in process_fn
processed_tensors = transform_input_data_fn(processed_tensors)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 515, in transform_and_pad_input_data_fn
tensor_dict=transform_data_fn(tensor_dict),
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\inputs.py",
line 129, in transform_input_data
tf.expand_dims(tf.to_float(image), axis=0))
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\meta_architectures\faster_rcnn_meta_arch.py",
line 543, in preprocess
parallel_iterations=self._parallel_iterations)
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\utils\shape_utils.py",
line 237, in static_or_dynamic_map_fn
outputs = [fn(arg) for arg in tf.unstack(elems)]
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\utils\shape_utils.py",
line 237, in
outputs = [fn(arg) for arg in tf.unstack(elems)]
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2264, in resize_to_range
lambda: _resize_portrait_image(image))
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\util\deprecation.py",
line 432, in new_func
return func(*args, **kwargs)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\ops\control_flow_ops.py",
line 2063, in cond
orig_res_t, res_t = context_t.BuildCondBranch(true_fn)
File "C:\Users\wyh\AppData\Local\conda\conda\envs\py352\lib\site-packages\tensorflow\python\ops\control_flow_ops.py",
line 1913, in BuildCondBranch
original_result = fn()
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2263, in
lambda: _resize_landscape_image(image),
File "F:\cindy\cindybackup\tensorflow1\models\research\object_detection\core\preprocessor.py",
line 2245, in _resize_landscape_image
align_corners=align_corners, preserve_aspect_ratio=True)
TypeError: resize_images() got an unexpected keyword argument 'preserve_aspect_ratio'
Thanks~

The problem is not resolved yet in tensorflow models(https://github.com/tensorflow/models/)
I just removed the preserve_aspect_ratio in object_detection/core/preprocessor.py
align_corners=align_corners, preserve_aspect_ratio=True)
align_corners=align_corners)

Tensorflow Object Detection API multi-scale inference variable reuse

I was trying to build multi-scale object detection inference code based on Tensorflow Object Detection API. However I don't quite know how to get around with variable reuse issue when predicting boxes on difference scale of a same image in one session. Here's what I did and where I don't understand:
In https://github.com/tensorflow/models/blob/master/research/object_detection/evaluator.py, I duplicate the line 'prediction_dict = model.predict(preprocessed_image, true_image_shapes)' one more time as you can see below.
def _extract_predictions_and_losses(model,
create_input_dict_fn,
ignore_groundtruth=False):
"""Constructs tensorflow detection graph and returns output tensors.
Args:
model: model to perform predictions with.
create_input_dict_fn: function to create input tensor dictionaries.
ignore_groundtruth: whether groundtruth should be ignored.
Returns:
prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed
by standard_fields.DetectionResultsFields) and optional groundtruth
tensors (keyed by standard_fields.InputDataFields).
losses_dict: A dictionary containing detection losses. This is empty when
ignore_groundtruth is true.
"""
input_dict = create_input_dict_fn()
prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
input_dict = prefetch_queue.dequeue()
original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
preprocessed_image, true_image_shapes = model.preprocess(
tf.to_float(original_image))
prediction_dict1 = model.predict(preprocessed_image, true_image_shapes)
/****Some code to resize preprocessed_image****/
prediction_dict2 = model.predict(preprocessed_image, true_image_shapes)
detections = model.postprocess(prediction_dict, true_image_shapes)
groundtruth = None
losses_dict = {}
if not ignore_groundtruth:
This gives me the following error:
Traceback (most recent call last):
File "object_detection/eval_fddb.py", line 167, in <module>
tf.app.run()
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 48, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "object_detection/eval_fddb.py", line 163, in main
FLAGS.checkpoint_dir, FLAGS.eval_dir)
File "/local/mnt/workspace/chris/projects/models/object_detection/evaluator_fddb.py", line 261, in evaluate
create_input_dict_fn=create_input_dict_fn)
File "/local/mnt/workspace/chris/projects/models/object_detection/evaluator_fddb.py", line 187, in _extract_prediction_tensors
prediction_dict = model.predict(preprocessed_image)
File "/local/mnt/workspace/chris/projects/models/object_detection/meta_architectures/umd_meta_arch.py", line 362, in predict
image_shape) = self._extract_rpn_feature_maps(preprocessed_inputs)
File "/local/mnt/workspace/chris/projects/models/object_detection/meta_architectures/umd_meta_arch.py", line 278, in _extract_rpn_feature_maps
preprocessed_inputs, scope=self.first_stage_feature_extractor_scope)
File "/local/mnt/workspace/chris/projects/models/object_detection/meta_architectures/faster_rcnn_meta_arch.py", line 154, in extract_proposal_features_w_internal_layers
return self._extract_proposal_features_w_internal_layers(preprocessed_inputs, scope)
File "/local/mnt/workspace/chris/projects/models/object_detection/models/faster_rcnn_resnet_v1_feature_extractor.py", line 173, in _extract_proposal_features_w_internal_layers
scope=var_scope)
File "/local/mnt/workspace/chris/projects/models/slim/nets/resnet_v1.py", line 300, in resnet_v1_101
reuse=reuse, scope=scope)
File "/local/mnt/workspace/chris/projects/models/slim/nets/resnet_v1.py", line 214, in resnet_v1
net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1')
File "/local/mnt/workspace/chris/projects/models/slim/nets/resnet_utils.py", line 122, in conv2d_same
rate=rate, padding='VALID', scope=scope)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args
return func(*args, **current_args)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py", line 1027, in convolution
outputs = layer.apply(inputs)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 503, in apply
return self.__call__(inputs, *args, **kwargs)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 443, in __call__
self.build(input_shapes[0])
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/convolutional.py", line 137, in build
dtype=self.dtype)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 383, in add_variable
trainable=trainable and self.trainable)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 1065, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 962, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 360, in get_variable
validate_shape=validate_shape, use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py", line 1561, in layer_variable_getter
return _model_variable_getter(getter, *args, **kwargs)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py", line 1553, in _model_variable_getter
custom_getter=getter, use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args
return func(*args, **current_args)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/variables.py", line 261, in model_variable
use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args
return func(*args, **current_args)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/variables.py", line 216, in variable
use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 352, in _true_getter
use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 664, in _get_single_variable
name, "".join(traceback.format_list(tb))))
ValueError: Variable FirstStageFeatureExtractor/resnet_v1_101/conv1/weights already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/variables.py", line 216, in variable
use_resource=use_resource)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args
return func(*args, **current_args)
File "/local/mnt/workspace/chris/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/variables.py", line 261, in model_variable
use_resource=use_resource)
My understanding is that when each time I call model.predict(), it creates a model with all variables. But in the second time, I can't create all the variables because all of them have been existing with the same name. I tried to add an variable scope with 'reuse=True' for the second model.predict(), but it wouldn't load the variables at all because mismatch names.
My questions is:
How to get around with this by running the same model on two different scale images in one sess.run()?
Thank you very much.

Have you tried using tf.variable_scope() before you build the model?
with tf.variable_scope('first_prediction'):
prediction_dict1 = model.predict(preprocessed_image, true_image_shapes)
with tf.variable_scope('second_prediction'):
prediction_dict2 = model.predict(preprocessed_image, true_image_shapes)
This way both model should have different prefix to their original variable name, hence preventing the reuse problem.

ValueError: Variable d_h0_conv/w/Adam/ does not exist, or was not created with tf.get_variable(). Did you mean to set reuse=None in VarScope?

I use text-to-image code,have an error. I did the following but there was still a problem.please help me. Thank you very much.
with tf.variable_scope(tf.get_variable_scope(), reuse=False):
d_optim = tf.train.AdamOptimizer(args.learning_rate, beta1 = args.beta1).minimize(loss['d_loss'], var_list=variables['d_vars'])
g_optim = tf.train.AdamOptimizer(args.learning_rate, beta1 = args.beta1).minimize(loss['g_loss'], var_list=variables['g_vars'])
error:
Traceback (most recent call last):
File "train.py", line 243, in <module>
main()
File "train.py", line 83, in main
d_optim = tf.train.AdamOptimizer(args.learning_rate, beta1 = args.beta1).minimize(loss['d_loss'], var_list=variables['d_vars'])
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\optimizer.py", line 325, in minimize
name=name)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\optimizer.py", line 446, in apply_gradients
self._create_slots([_get_variable_for(v) for v in var_list])
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\adam.py", line 128, in _create_slots
self._zeros_slot(v, "m", self._name)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\optimizer.py", line 766, in _zeros_slot
named_slots[_var_key(var)] = slot_creator.create_zeros_slot(var, op_name)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\slot_creator.py", line 174, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\slot_creator.py", line 146, in create_slot_with_initializer
dtype)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\training\slot_creator.py", line 66, in _create_slot_var
validate_shape=validate_shape)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 1065, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 962, in get_variable
use_resource=use_resource, custom_getter=custom_getter)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 367, in get_variable
validate_shape=validate_shape, use_resource=use_resource)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 352, in _true_getter
use_resource=use_resource)
File "C:\Users\Administrator\Anaconda3\lib\site-packages\tensorflow\python\ops\variable_scope.py", line 682, in _get_single_variable
"VarScope?" % name)
ValueError: Variable d_h0_conv/w/Adam/ does not exist, or was not created with tf.get_variable(). Did you mean to set reuse=None in VarScope?