Develop Reference

How can I fix this tensorflow saver restore error which occurs when repeatedly using the def function of estimation model

I made an estimation model using tensorflow and keras, and made this model into a def function with tf.train.Saver.
However, repeated use of this def function occurs the following error.
What is the reason for this error and how can I fix it?
the def function of estimation model in 'Model.py'
def LSTM_Model(input_data):
x = tf.placeholder(tf.float32, [None, 20, 121], name="input")
with tf.variable_scope('LSTM'):
y_pred = Lsss(x)
with tf.Session() as sess:
saver = tf.train.Saver(max_to_keep=4)
tf.global_variables_initializer().run()
ckpt = tf.train.get_checkpoint_state(
'C:\LSTM\LSTM_model3\LSTM_paper_2_mean20_4_3_epoch_31_look_back_20_1\Result3')
saver.restore(sess, ckpt.model_checkpoint_path)
K.set_learning_phase(False)
height = sess.run([y_pred], {x: input_data})
return height[0][0]
main function
from Model import LSTM_Model
from numpy import *
import hdf5storage
import numpy as np
def main():
filename = 'Datafile'
mat_file = hdf5storage.loadmat('C:/LSTM/Data/'+filename+'.mat')
TrainSet = mat_file['Trainingset'][0]
Train_time = mat_file['Train_time'][0]
trainnum = 0
data1 = TrainSet[trainnum]
before_data = data1[Train_time[trainnum]-20:Train_time[trainnum], 0:121]
input_data = np.reshape(before_data, (1, 20, 121))
height = LSTM_Model(input_data)*1000
print(height)
height = LSTM_Model(input_data)*1000
print(height)
Occurred error
C:\Users\username\Anaconda3\python.exe C:/LSTM/main.py
[544.562]
Traceback (most recent call last):
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1356, in _do_call
return fn(*args)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1341, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[{{node save_1/RestoreV2}}]]
(1) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[{{node save_1/RestoreV2}}]]
[[save_1/RestoreV2/_37]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1286, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 950, in run
run_metadata_ptr)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1173, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1350, in _do_run
run_metadata)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1370, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[node save_1/RestoreV2 (defined at \LSTM\Model.py:35) ]]
(1) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[node save_1/RestoreV2 (defined at \LSTM\Model.py:35) ]]
[[save_1/RestoreV2/_37]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'save_1/RestoreV2':
File "/LSTM/main.py", line 114, in <module>
main()
File "/LSTM/main.py", line 96, in main
height = LSTM_Model(input_data)*1000
File "\LSTM\Model.py", line 35, in LSTM_Model
saver = tf.train.Saver(max_to_keep=4)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
self.build()
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
self._build(self._filename, build_save=True, build_restore=True)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
build_restore=build_restore)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
restore_sequentially, reshape)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
restore_sequentially)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1780, in restore_v2
name=name)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
op_def=op_def)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1296, in restore
names_to_keys = object_graph_key_mapping(save_path)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1614, in object_graph_key_mapping
object_graph_string = reader.get_tensor(trackable.OBJECT_GRAPH_PROTO_KEY)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\pywrap_tensorflow_internal.py", line 678, in get_tensor
return CheckpointReader_GetTensor(self, compat.as_bytes(tensor_str))
tensorflow.python.framework.errors_impl.NotFoundError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/LSTM/main.py", line 114, in <module>
main()
File "C:/LSTM/main.py", line 96, in main
height = LSTM_Model(input_data)*1000
File "C:\LSTM\Model.py", line 39, in LSTM_Model
saver.restore(sess, ckpt.model_checkpoint_path)
File "C:\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1302, in restore
err, "a Variable name or other graph key that is missing")
tensorflow.python.framework.errors_impl.NotFoundError: Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:
2 root error(s) found.
(0) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[node save_1/RestoreV2 (defined at \LSTM\Model.py:35) ]]
(1) Not found: Key LSTM_1/cu_dnnlstm_3/bias not found in checkpoint
[[node save_1/RestoreV2 (defined at \LSTM\Model.py:35) ]]
[[save_1/RestoreV2/_37]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'save_1/RestoreV2':
File "/LSTM/main.py", line 114, in <module>
main()
File "/LSTM/main.py", line 96, in main
height = LSTM_Model(input_data)*1000
File "\LSTM\Model.py", line 35, in LSTM_Model
saver = tf.train.Saver(max_to_keep=4)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
self.build()
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
self._build(self._filename, build_save=True, build_restore=True)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
build_restore=build_restore)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
restore_sequentially, reshape)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
restore_sequentially)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1780, in restore_v2
name=name)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
op_def=op_def)
File "\Users\username\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
Process finished

How can I use single GPU to run multiple models at the same time in tensorflow python?

I am trying to use my laptop as server for faceRecognition and speechSynthesis for my project on raspberry pi. So I created a program which loads both the models initially and then wait for the request to come. But when I start the program, initially faceRecognition model loads successfully but then at the time of loading speechSynthesis model, it gives me as error regarding the tf.saver.
code:
Server-
def findFaceMatch():
image_file = request.files.get("imagefile")
image_file.save("image.jpg")
print("sent for check")
response = face_match_demo.recognizeFace(os.path.join(os.getcwd(),"image.jpg"))
return response, 200
#api.route("/synthesize/<string:text>")
def synthesizeVoice(text):
print(text)
with open("F:/file.wav", 'wb') as f:
f.write(synthesizer.synthesize(text))
return send_from_directory("F:/","file.wav", as_attachment=True), 200
Face Recognition-
import tensorflow as tf
import numpy as np
from . import facenet
from .align import detect_face
import cv2
import imutils
import os
import pickle
import time
minsize = 20
threshold = [0.6, 0.7, 0.7]
factor = 0.709
margin = 44
input_image_size = 160
def load_models(session):
global sess
sess = session
global pnet, rnet, onet
pnet, rnet, onet = detect_face.create_mtcnn(sess, os.path.join(os.getcwd(),"Face_recognition","align"))
facenet.load_model(os.path.join(os.getcwd(),"Face_recognition","20170512-110547\\20170512-110547.pb"))
global images_placeholder
images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
global embeddings
embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
global phase_train_placeholder
phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
global embedding_size
embedding_size = embeddings.get_shape()[1]
def getFace(img):
faces = []
img_size = np.asarray(img.shape)[0:2]
bounding_boxes, _ = detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor)
if not len(bounding_boxes) == 0:
for face in bounding_boxes:
if face[4] > 0.50:
det = np.squeeze(face[0:4])
bb = np.zeros(4, dtype=np.int32)
bb[0] = np.maximum(det[0] - margin / 2, 0)
bb[1] = np.maximum(det[1] - margin / 2, 0)
bb[2] = np.minimum(det[2] + margin / 2, img_size[1])
bb[3] = np.minimum(det[3] + margin / 2, img_size[0])
cropped = img[bb[1]:bb[3], bb[0]:bb[2], :]
resized = cv2.resize(cropped, (input_image_size,input_image_size),interpolation=cv2.INTER_CUBIC)
prewhitened = facenet.prewhiten(resized)
faces.append(getEmbedding(prewhitened))
return faces
def getEmbedding(resized):
reshaped = resized.reshape(-1,input_image_size,input_image_size,3)
feed_dict = {images_placeholder: reshaped, phase_train_placeholder: False}
embedding = sess.run(embeddings, feed_dict=feed_dict)
return embedding
def compare2face(img1):
print("checking")
face2 = getFace(img1)
face1 = []
with open(os.path.join(os.getcwd(),"Face_recognition","trained_knn_model.PB"), 'rb') as f:
for i in range(4):
face1.append(pickle.load(f))
names = ["x","y","z","p"]
print("verifying name")
for i in range(0,len(face1)):
if face1[i] and face2:
# calculate Euclidean distance
dist = np.sqrt(np.sum(np.square(np.subtract(face1[i], face2[0]))))
if dist <= 0.8:
return "dist: "+str(dist)+"\nhello "+names[i]
return "Person not found"
def recognizeFace(image_path):
image = cv2.imread(image_path)
response = compare2face(image)
return response
Speech Synthesis:
import io
import numpy as np
import tensorflow as tf
from .hparams import hparams
from librosa import effects
from .models import create_model
from .text import text_to_sequence
from .util import audio
class Synthesizer:
def load(self, checkpoint_path, sess, model_name='tacotron'):
print('Constructing model: %s' % model_name)
inputs = tf.placeholder(tf.int32, [1, None], 'inputs')
input_lengths = tf.placeholder(tf.int32, [1], 'input_lengths')
with tf.variable_scope('model') as scope:
self.model = create_model(model_name, hparams)
self.model.initialize(inputs, input_lengths)
self.wav_output = audio.inv_spectrogram_tensorflow(self.model.linear_outputs[0])
print('Loading checkpoint: %s' % checkpoint_path)
# self.session = tf.Session()
self.session = sess
self.session.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(self.session, checkpoint_path)
def synthesize(self, text):
cleaner_names = [x.strip() for x in hparams.cleaners.split(',')]
seq = text_to_sequence(text, cleaner_names)
feed_dict = {
self.model.inputs: [np.asarray(seq, dtype=np.int32)],
self.model.input_lengths: np.asarray([len(seq)], dtype=np.int32)
}
wav = self.session.run(self.wav_output, feed_dict=feed_dict)
wav = audio.inv_preemphasis(wav)
wav = wav[:audio.find_endpoint(wav)]
out = io.BytesIO()
audio.save_wav(wav, out)
return out.getvalue()
error is as follows:
2019-11-11 21:48:04.408636: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at save_restore_v2_ops.cc:184 : Not found: Key onet/conv1/biases not found in checkpoint
Traceback (most recent call last):
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1356, in _do_call
return fn(*args)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1341, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key onet/conv1/biases not found in checkpoint
[[{{node save/RestoreV2}}]]
[[save/RestoreV2/_617]]
(1) Not found: Key onet/conv1/biases not found in checkpoint
[[{{node save/RestoreV2}}]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1286, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 950, in run
run_metadata_ptr)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1173, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1350, in _do_run
run_metadata)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1370, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: 2 root error(s) found.
(0) Not found: Key onet/conv1/biases not found in checkpoint
[[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
[[save/RestoreV2/_617]]
(1) Not found: Key onet/conv1/biases not found in checkpoint
[[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'save/RestoreV2':
File "commonServer.py", line 38, in <module>
synthesizer.load(model_path,sess)
File "F:\Backend\Text_To_Speech\synthesizer.py", line 25, in load
saver = tf.train.Saver()
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
self.build()
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
self._build(self._filename, build_save=True, build_restore=True)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
build_restore=build_restore)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
restore_sequentially, reshape)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
restore_sequentially)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1696, in restore_v2
name=name)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
op_def=op_def)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1296, in restore
names_to_keys = object_graph_key_mapping(save_path)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1614, in object_graph_key_mapping
object_graph_string = reader.get_tensor(trackable.OBJECT_GRAPH_PROTO_KEY)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\pywrap_tensorflow_internal.py", line 678, in get_tensor
return CheckpointReader_GetTensor(self, compat.as_bytes(tensor_str))
tensorflow.python.framework.errors_impl.NotFoundError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "commonServer.py", line 38, in <module>
synthesizer.load(model_path,sess)
File "F:\Backend\Text_To_Speech\synthesizer.py", line 26, in load
saver.restore(self.session, checkpoint_path)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 1302, in restore
err, "a Variable name or other graph key that is missing")
tensorflow.python.framework.errors_impl.NotFoundError: Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the checkpoint. Original error:
2 root error(s) found.
(0) Not found: Key onet/conv1/biases not found in checkpoint
[[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
[[save/RestoreV2/_617]]
(1) Not found: Key onet/conv1/biases not found in checkpoint
[[node save/RestoreV2 (defined at F:\Backend\Text_To_Speech\synthesizer.py:25) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'save/RestoreV2':
File "commonServer.py", line 38, in <module>
synthesizer.load(model_path,sess)
File "F:\Backend\Text_To_Speech\synthesizer.py", line 25, in load
saver = tf.train.Saver()
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 825, in __init__
self.build()
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 837, in build
self._build(self._filename, build_save=True, build_restore=True)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 875, in _build
build_restore=build_restore)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 508, in _build_internal
restore_sequentially, reshape)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 328, in _AddRestoreOps
restore_sequentially)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\training\saver.py", line 575, in bulk_restore
return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 1696, in restore_v2
name=name)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 3616, in create_op
op_def=op_def)
File "C:\Users\Jaydip Bari\Anaconda3\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()

At least up to TF 2.0 (not sure about the version you have), loading multiple models into one graph could cause problems: See
load multiple models in Tensorflow
I hope this helps.

Tensorflow InvalidArgumentError: indices[40] = 2000 0 is not in [0, 20000)

I was runnig this code (https://github.com/monkut/tensorflow_chatbot main code in execute.py) on my Windows7 with python 3.5 and tensorflow r0.12 cpu and an error occured after just 300 steps. Then I tried to change the vocabulary size to 30000 and set a checkpiont every 100 steps. With 1 layer of 128 units the error occured after 3900 steps and with 3 layers of 256 units it occured after 5400 steps.
What kind of error is that? Is there a way to solve it?
Error:
>> Mode : train
Preparing data in working_dir/
Creating vocabulary working_dir/vocab20000.enc from data/train.enc
processing line 100000
>> Full Vocabulary Size : 45408
>>>> Vocab Truncated to: 20000
Creating vocabulary working_dir/vocab20000.dec from data/train.dec
processing line 100000
>> Full Vocabulary Size : 44271
>>>> Vocab Truncated to: 20000
Tokenizing data in data/train.enc
tokenizing line 100000
Tokenizing data in data/train.dec
tokenizing line 100000
Tokenizing data in data/test.enc
Creating 3 layers of 256 units.
Created model with fresh parameters.
Reading development and training data (limit: 0).
reading data line 100000
global step 300 learning rate 0.5000 step-time 3.34 perplexity 377.45
eval: bucket 0 perplexity 96.25
eval: bucket 1 perplexity 210.94
eval: bucket 2 perplexity 267.86
eval: bucket 3 perplexity 365.77
Traceback (most recent call last):
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 1021, in _do_call
return fn(*args)
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 1003, in _run_fn
status, run_metadata)
File "C:\Python35 64\lib\contextlib.py", line 66, in __exit__
next(self.gen)
File "C:\Python35 64\lib\site-packages\tensorflow\python\framework\errors_impl
.py", line 469, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[40] = 2000
0 is not in [0, 20000)
[[Node: model_with_buckets/sequence_loss_3/sequence_loss_by_example/sam
pled_softmax_loss_28/embedding_lookup_1 = Gather[Tindices=DT_INT64, Tparams=DT_F
LOAT, _class=["loc:#proj_b"], validate_indices=true, _device="/job:localhost/rep
lica:0/task:0/cpu:0"](proj_b/read, model_with_buckets/sequence_loss_3/sequence_l
oss_by_example/sampled_softmax_loss_28/concat)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "execute.py", line 352, in <module>
train()
File "execute.py", line 180, in train
target_weights, bucket_id, False)
File "C:\Users\Администратор\Downloads\tensorflow_chatbot-master (1)\tensorflo
w_chatbot-master\seq2seq_model.py", line 230, in step
outputs = session.run(output_feed, input_feed)
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 766, in run
run_metadata_ptr)
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 964, in _run
feed_dict_string, options, run_metadata)
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 1014, in _do_run
target_list, options, run_metadata)
File "C:\Python35 64\lib\site-packages\tensorflow\python\client\session.py", l
ine 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[40] = 2000
0 is not in [0, 20000)
[[Node: model_with_buckets/sequence_loss_3/sequence_loss_by_example/sam
pled_softmax_loss_28/embedding_lookup_1 = Gather[Tindices=DT_INT64, Tparams=DT_F
LOAT, _class=["loc:#proj_b"], validate_indices=true, _device="/job:localhost/rep
lica:0/task:0/cpu:0"](proj_b/read, model_with_buckets/sequence_loss_3/sequence_l
oss_by_example/sampled_softmax_loss_28/concat)]]
Caused by op 'model_with_buckets/sequence_loss_3/sequence_loss_by_example/sample
d_softmax_loss_28/embedding_lookup_1', defined at:
File "execute.py", line 352, in <module>
train()
File "execute.py", line 148, in train
model = create_model(sess, False)
File "execute.py", line 109, in create_model
gConfig['learning_rate_decay_factor'], forward_only=forward_only)
File "C:\Users\Администратор\Downloads\tensorflow_chatbot-master (1)\tensorflo
w_chatbot-master\seq2seq_model.py", line 158, in __init__
softmax_loss_function=softmax_loss_function)
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\seq2seq.py", line
1130, in model_with_buckets
softmax_loss_function=softmax_loss_function))
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\seq2seq.py", line
1058, in sequence_loss
softmax_loss_function=softmax_loss_function))
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\seq2seq.py", line
1022, in sequence_loss_by_example
crossent = softmax_loss_function(logit, target)
File "C:\Users\Администратор\Downloads\tensorflow_chatbot-master (1)\tensorflo
w_chatbot-master\seq2seq_model.py", line 101, in sampled_loss
self.target_vocab_size)
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\nn.py", line 1412
, in sampled_softmax_loss
name=name)
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\nn.py", line 1184
, in _compute_sampled_logits
all_b = embedding_ops.embedding_lookup(biases, all_ids)
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\embedding_ops.py"
, line 110, in embedding_lookup
validate_indices=validate_indices)
File "C:\Python35 64\lib\site-packages\tensorflow\python\ops\gen_array_ops.py"
, line 1293, in gather
validate_indices=validate_indices, name=name)
File "C:\Python35 64\lib\site-packages\tensorflow\python\framework\op_def_libr
ary.py", line 759, in apply_op
op_def=op_def)
File "C:\Python35 64\lib\site-packages\tensorflow\python\framework\ops.py", li
ne 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Python35 64\lib\site-packages\tensorflow\python\framework\ops.py", li
ne 1128, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): indices[40] = 20000 is not in [0
, 20000)
[[Node: model_with_buckets/sequence_loss_3/sequence_loss_by_example/sam
pled_softmax_loss_28/embedding_lookup_1 = Gather[Tindices=DT_INT64, Tparams=DT_F
LOAT, _class=["loc:#proj_b"], validate_indices=true, _device="/job:localhost/rep
lica:0/task:0/cpu:0"](proj_b/read, model_with_buckets/sequence_loss_3/sequence_l
oss_by_example/sampled_softmax_loss_28/concat)]]

It seems using virtualenv and tensorflow-gpu 0.12.0 solves the problem for me.

The notation [) means Inclusive Exclusive in interval notation.
[ means including that number. ( means excluding that number.
the same goes for right parentheses and brackets ie ] & ). For example [0,20000)
means from Zero inclusive to 20000 not inclusive. Brackets mean "Yes include this" parenthesis mean "no, don't go all the way up to this number"

A magic error about TensorFlow when recognize more than one img(Key Variable_10 not found in checkpoint)

I have trained a TensorFlow model and it worked fine when tested with one img. But when I wanted to test more than one img, an error occurred.
Error:
Traceback (most recent call last):
File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 1139, in _do_call
return fn(*args)
File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 1121, in _run_fn
status, run_metadata)
File "C:\Anaconda\lib\contextlib.py", line 89, in __exit__
next(self.gen)
File "C:\Anaconda\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 466, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.NotFoundError: Key Variable_10 not found in checkpoint
[[Node: save_1/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_arg_save_1/Const_0_0, save_1/RestoreV2_2/tensor_names, save_1/RestoreV2_2/shape_and_slices)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "C:/Users/Louis Song/Desktop/LetsFuckDog/captcha_server.py", line 317, in <module>
fuck_captcha("data/bv22.jpg") File "C:/Users/Louis Song/Desktop/LetsFuckDog/captcha_server.py", line 251, in fuck_captcha
saver.restore(sess, tf.train.latest_checkpoint('.')) File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 1548, in restore
{self.saver_def.filename_tensor_name: save_path}) File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 789, in run
run_metadata_ptr) File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 997, in _run
feed_dict_string, options, run_metadata) File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 1132, in _do_run
target_list, options, run_metadata) File "C:\Anaconda\lib\site-packages\tensorflow\python\client\session.py", line 1152, in _do_call
raise type(e)(node_def, op, message) tensorflow.python.framework.errors_impl.NotFoundError: Key Variable_10 not found in checkpoint [[Node: save_1/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT],
_device="/job:localhost/replica:0/task:0/cpu:0"](_arg_save_1/Const_0_0, save_1/RestoreV2_2/tensor_names, save_1/RestoreV2_2/shape_and_slices)]]
Caused by op 'save_1/RestoreV2_2', defined at: File "C:/Users/Louis Song/Desktop/LetsFuckDog/captcha_server.py", line 317, in <module>
fuck_captcha("data/bv22.jpg") File "C:/Users/Louis Song/Desktop/LetsFuckDog/captcha_server.py", line 249, in fuck_captcha
saver = tf.train.Saver() File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 1139, in __init__
self.build() File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 1170, in build
restore_sequentially=self._restore_sequentially) File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 691, in build
restore_sequentially, reshape) File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 407, in _AddRestoreOps
tensors = self.restore_op(filename_tensor, saveable, preferred_shard) File "C:\Anaconda\lib\site-packages\tensorflow\python\training\saver.py", line 247, in restore_op
[spec.tensor.dtype])[0]) File "C:\Anaconda\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 640, in restore_v2
dtypes=dtypes, name=name) File "C:\Anaconda\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 767, in apply_op
op_def=op_def) File "C:\Anaconda\lib\site-packages\tensorflow\python\framework\ops.py", line 2506, in create_op
original_op=self._default_original_op, op_def=op_def) File "C:\Anaconda\lib\site-packages\tensorflow\python\framework\ops.py", line 1269, in __init__
self._traceback = _extract_stack()
NotFoundError (see above for traceback): Key Variable_10 not found in checkpoint [[Node: save_1/RestoreV2_2 = RestoreV2[dtypes=[DT_FLOAT],
_device="/job:localhost/replica:0/task:0/cpu:0"](_arg_save_1/Const_0_0, save_1/RestoreV2_2/tensor_names, save_1/RestoreV2_2/shape_and_slices)]]
Demo code:
def crack_captcha():
output = crack_captcha_cnn()
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, tf.train.latest_checkpoint('.'))
n = 1
while n <= 10:
name, image = get_name_and_image()
print(image.shape)
print(name)
if image.shape !=(60, 160, 3):
print('原始图片错误，请核查')
pass
else:
image = convert2gray(image)
# middle=image.flatten() /255
try:
image = image.flatten() / 255
predict = tf.argmax(tf.reshape(output, [-1, MAX_CAPTCHA, ALL_SET_LEN]), 2)
text_list = sess.run(predict, feed_dict={X: [image], keep_prob: 1})
text = text_list[0].tolist()
vector = np.zeros(MAX_CAPTCHA * ALL_SET_LEN)
i = 0
for n in text:
vector[i * ALL_SET_LEN + n] = 1
i += 1
print(vector)
predict_text = vec2name(vector)
print("正确: {} 预测: {}".format(name, predict_text))
if name !=predict_text:
print('预测失败')
global error_time
error_time+=1
else:
print('预测成功')
global correct_time
correct_time+=1
n += 1
print(n)
except TypeError as e :
print(e)
n += 1
print(n)
pass
Magic reason:
When I call more than once crack_captcha function, here is the error. But when I just call one time crack_captcha function, It can give my predict result.

You're building a graph every time you try to predict. Instead, build your graph first and then just call session.run when you want to predict.

may i ask how it was resolved in the above code, please?
i have the following code still facing the issue "Key Variable_10 not found in checkpoint" and have to restart the Spyder console every time. thanks!
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
saver.restore(sess, "model\\model.ckpt")
prediction=tf.argmax(y_conv,1)
predint=prediction.eval(feed_dict={x: [result],keep_prob: 1.0}, session=sess)
print('recognize result:')
print(predint[0])

Output files returned after training a alexnet model...?

Code is written in Python 3.5.X
Please try to make the answer simple for a 3rd year Computer Science Student
The output files from train_model.py seems to be a model.meta file but the test_model.py is asking for a .model file. The tutorial user has a .model file as well I can't seem to understand why i am getting a file with .model.meta
I am trying to play GTA San Andreas through Python or more precisely the car in GTA is driven by the model.
It takes screen frames as Input and recorded the key i Input during training. This training data is used to train the model.
Code for training the model
import numpy as np
from alexnet import alexnet
WIDTH = 80
HEIGHT = 60
LR = 1e-3
EPOCHS = 8
MODEL_NAME = 'pygta_sa-car-{}-{}-{}-epochs.model'.format(LR, 'alextnetv2', EPOCHS)
model = alexnet(WIDTH, HEIGHT, LR)
train_data = np.load('training_data_v2.npy')
train = train_data[:-500]
test = train_data[-500:]
X = np.array([i[0] for i in train]).reshape(-1,WIDTH,HEIGHT,1)
Y = [i[1] for i in train]
test_x = np.array([i[0] for i in test]).reshape(-1,WIDTH,HEIGHT,1)
test_y = [i[1] for i in test]
model.fit({'input': X}, {'targets': Y}, n_epoch=EPOCHS, validation_set=({'input': test_x}, {'targets': test_y}),
snapshot_step=500, show_metric=True, run_id=MODEL_NAME)
# tensorboard --logdir=foo:F:\play_gta_sa\log
model.save(MODEL_NAME)
training completes successfully and returns files
Files returned on the video of the Tutorial i am using to do this project
sent_dex files returned
Content of Checkpoint file
model_checkpoint_path: "F:\play_gta_sa\pygta_sa-car-0.001-alextnetv2-8-epochs.model"
all_model_checkpoint_paths: "F:\play_gta_sa\pygta_sa-car-0.001-alextnetv2-8-epochs.model"
Code for testing the model on the game
import numpy as np
import cv2
import time
from grabscreen import grab_screen
from getkeys import key_check
from directkeys import PressKey, ReleaseKey, W, A, S, D
from alexnet import alexnet
WIDTH = 80
HEIGHT = 60
LR = 1e-3
EPOCHS = 8
MODEL_NAME = 'pygta_sa-car-{}-{}-{}-epochs.model'.format(LR, 'alexnetv2',EPOCHS)
def straight():
PressKey(W)
ReleaseKey(A)
ReleaseKey(D)
def left():
PressKey(W)
PressKey(A)
ReleaseKey(D)
def right():
PressKey(W)
PressKey(D)
ReleaseKey(A)
model = alexnet(WIDTH, HEIGHT, LR)
model.load(MODEL_NAME)
def main():
for i in list(range(10))[::-1]:
print(i+1)
time.sleep(1)
last_time = time.time()
paused = False
while True:
if not paused:
screen = grab_screen(region=(0,40,800,640))
screen = cv2.cvtColor(screen,cv2.COLOR_BGR2GRAY)
screen = cv2.resize(screen,(80,60))
print('Frame took {} seconds'.format(time.time()-last_time))
last_time = time.time()
moves = list(np.around(model.predict([screen.reshape(80,60,1)])[0]))
print(moves, prediction)
if moves == [1,0,0]:
left()
elif moves == [0,1,0]:
straight()
elif moves == [0,0,1]:
right()
keys = key_check()
# p pauses game and can get annoying.
if 'T' in keys:
if paused:
paused = False
time.sleep(1)
else:
paused = True
ReleaseKey(A)
ReleaseKey(W)
ReleaseKey(D)
time.sleep(1)
main()
the error message on running test model
Traceback (most recent call last):
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1039, in _do_call
return fn(*args)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1021, in _run_fn
status, run_metadata)
File "C:\Program Files\Python35\lib\contextlib.py", line 66, in __exit__
next(self.gen)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 466, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.NotFoundError: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for F:\play_gta_sa\pygta_sa-car-0.001-alexnetv2-8-epochs.model
[[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save_1/Const_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "F:\play_gta_sa\test_model.py", line 33, in <module>
model.load(MODEL_NAME)
File "C:\Program Files\Python35\lib\site-packages\tflearn\models\dnn.py", line 282, in load
self.trainer.restore(model_file, weights_only, **optargs)
File "C:\Program Files\Python35\lib\site-packages\tflearn\helpers\trainer.py", line 452, in restore
self.restorer.restore(self.session, model_file)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 1457, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 778, in run
run_metadata_ptr)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 982, in _run
feed_dict_string, options, run_metadata)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1032, in _do_run
target_list, options, run_metadata)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\client\session.py", line 1052, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for F:\play_gta_sa\pygta_sa-car-0.001-alexnetv2-8-epochs.model
[[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save_1/Const_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]
Caused by op 'save_1/RestoreV2', defined at:
File "<string>", line 1, in <module>
File "C:\Program Files\Python35\lib\idlelib\run.py", line 124, in main
ret = method(*args, **kwargs)
File "C:\Program Files\Python35\lib\idlelib\run.py", line 351, in runcode
exec(code, self.locals)
File "F:\play_gta_sa\test_model.py", line 32, in <module>
model = alexnet(WIDTH, HEIGHT, LR)
File "F:\play_gta_sa\alexnet.py", line 40, in alexnet
max_checkpoints=1, tensorboard_verbose=0, tensorboard_dir='log')
File "C:\Program Files\Python35\lib\site-packages\tflearn\models\dnn.py", line 64, in __init__
best_val_accuracy=best_val_accuracy)
File "C:\Program Files\Python35\lib\site-packages\tflearn\helpers\trainer.py", line 147, in __init__
allow_empty=True)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 1056, in __init__
self.build()
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 1086, in build
restore_sequentially=self._restore_sequentially)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 691, in build
restore_sequentially, reshape)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 407, in _AddRestoreOps
tensors = self.restore_op(filename_tensor, saveable, preferred_shard)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\training\saver.py", line 247, in restore_op
[spec.tensor.dtype])[0])
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\ops\gen_io_ops.py", line 669, in restore_v2
dtypes=dtypes, name=name)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Program Files\Python35\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
NotFoundError (see above for traceback): Unsuccessful TensorSliceReader constructor: Failed to find any matching files for F:\play_gta_sa\pygta_sa-car-0.001-alexnetv2-8-epochs.model
[[Node: save_1/RestoreV2 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:localhost/replica:0/task:0/cpu:0"](_recv_save_1/Const_0, save_1/RestoreV2/tensor_names, save_1/RestoreV2/shape_and_slices)]]

Thanks for the update.
Edit:
Try add this line before tf.reset_default_graph() loading model. That is
import tensorflow as tf
tf.reset_default_graph()
model = alexnet(WIDTH, HEIGHT, LR)
model.load(MODEL_NAME)

The crux of this error is:
Unsuccessful TensorSliceReader constructor: Failed to find any matching files for F:\play_gta_sa\pygta_sa-car-0.001-alexnetv2-8-epochs.model
Okay, so a typical "file not found." Are we confident that we have this file? Maybe, but, if it was there, it would have been found. Our first guess should be we've typoed or otherwise made a mistake. Let's look at your model files:
For training the model, you have:
MODEL_NAME = 'pygta_sa-car-{}-{}-{}-epochs.model'.format(LR, 'alextnetv2', EPOCHS)
For testing the model, you have:
MODEL_NAME = 'pygta_sa-car-{}-{}-{}-epochs.model'.format(LR, 'alexnetv2',EPOCHS)
Do you see difference yet? There's a typo. alextnetv2 vs alexnetv2
Fix that, and the file will at least be found.