I would like to save my trained Tensorflow model, so it can be deployed by restoring the model file (I'm following this example, which seems to make sense). To do this, however, I need to have named tensors, so that I can do reload the variables with something like:
graph = tf.get_default_graph()
w1 = graph.get_tensor_by_name("my_tensor:0")
I am queuing images from a list of filenames using string_input_producer (code below), but how do I name the tensors so that I can reload them at a later stage?
import tensorflow as tf
flags = tf.app.flags
conf = flags.FLAGS
class ImageDataSet(object):
def __init__(self, img_list_path, num_epoch, batch_size):
# Build the record list queue
input_file = open(images_list_path, 'r')
self.record_list = []
for line in input_file:
line = line.strip()
self.record_list.append(line)
filename_queue = tf.train.string_input_producer(self.record_list, num_epochs=num_epoch)
image_reader = tf.WholeFileReader()
_, image_file = image_reader.read(filename_queue)
image = tf.image.decode_jpeg(image_file, conf.img_colour_channels)
# preprocess
# ...
min_after_dequeue = 1000
capacity = min_after_dequeue + 400 * batch_size
self.images = tf.train.shuffle_batch(image, batch_size=batch_size, capacity=capacity,
min_after_dequeue=min_after_dequeue)
I assume that you want to restore the graph for testing or deploying.
For these purposes, you can edit your graph by insert a placeholder as an entrance of the testing data.
To edit the graph, you can use tf's graph editor, or build an new graph with placeholder and save it.
Related
this is my first time using Simple Transformers - NERModelPermalink and I want to use the offline mode for loading the pretrained bert-base-cased model, I downloaded the files and put it in same directory, now I cant load the model and I cant start the training part, this is the code
# test.ipynb
args = NERArgs()
args.num_train_epochs = 1
args.learning_rate = 4e-5
args.overwrite_output_dir =True
args.train_batch_size = 32
args.eval_batch_size = 32
args.save_steps = -1
args.save_model_every_epoch = False
model = NERModel('bert', 'bert-base-cased',labels=label,args =args, use_cuda=False)
this is the directory map
root_
|__down_model__
|__config.json
|__flax_model.msgpack
|__.gitattributes
|__pytorch_model.bin
|__README.md
|__tf_model.h5
|__tokenizer.json
|__tokenizer_config.json
|__Unconfirmed 363105.crdownload
|__vocab.txt
root_
|__test.ipynb
how can I use the model I downloaded (root/down_model) for this line
model = NERModel('bert', 'bert-base-cased',labels=label,args =args, use_cuda=False)
I am trying to serve a tensorflow.keras.Model in a Flask + nginx + uwsgi application, using Tensorflow v1.14.
I load the model in the constructor of a class named Prediction in my Flask's application factory function and save the graph as an attribute of
the Flask app, as suggested here.
Then I run the prediction by calling a method Prediction.process in a route named _process of my Flask app, but it gets stuck during the call of tf.keras.Model.predict (self.model.summary() in predict.py is executed, i.e. the summary is shown, but not print("Never gets here :(")).
If I initialize my class Prediction in _process (which I want to avoid to not have to load the model for every prediction), everything works fine.
If I use Flask server, it works fine, too. So it seems that it is related to uwsgi config.
Any suggestion ?
init.py
def create_app():
app = Flask(__name__)
#(...)
app.register_blueprint(bp)
load_tf_model(app)
return app
def load_tf_model(app):
sess = tf.Session(graph=tf.Graph())
app.sess = sess
with sess.graph.as_default():
weights = os.path.join(app.static_folder, 'weights/model.32-0.81.h5')
app.prediction = Prediction(weights)
predict.py
class Prediction:
def __init__(self, weights):
# build model and set weights
inputs = tf.keras.Input(shape=SHAPE, batch_size=1)
outputs = simple_cnn.build_model(inputs, N_CLASSES)
self.model = tf.keras.Model(inputs=inputs, outputs=outputs)
self.model.load_weights(weights)
self.model._make_predict_function()
# create TF mel extractor
self.melspec_ex = tf_feature_utils.MelSpectrogram()
def process(self, audio, sr):
# compute features (in NCHW format) and labels
data = audio2data(
audio,
sr,
class_list=np.arange(N_CLASSES))
features = np.asarray([d[0] for d in data])
features = tf.reshape(features, (features.shape[0], 1, features.shape[1], features.shape[2]))
labels = np.asarray([d[1] for d in data])
# make tf.data.Dataset
dataset = tf.data.Dataset.from_tensor_slices((features, labels))
dataset = dataset.batch(1)
dataset = dataset.map(lambda data, labels: (
tf.expand_dims(self.melspec_ex.process(tf.squeeze(data, axis=[1,2])), 1)))
# show model (debug)
self.model.summary()
# run prediction
predictions = self.model.predict(dataset)
print("Never gets here :(")
# integrate predictions over time
return np.mean(predictions, axis=0)
routes.py
#bp.route('/_process', methods=['POST'])
def _process():
with current_app.graph.as_default():
# load audio
filepath = session['filepath']
audio, sr = librosa.load(filepath)
# predict
predictions = current_app.prediction.process(audio, sr)
# delete file
os.remove(filepath)
return jsonify(prob=predictions.tolist())
It was a threading issue. I had to add configure uwsgi with the following options:
master = false
processes = 1
cheaper = 0
I wanted to share my findings on how to export a tf model for serving directly from session without creating model checkpoint. my use case requires minimum time to create a pb file, therefore I wanted to get a model.pb file directly from session without creating model checkpoint.
most examples online (and documentation refers to the common case of creating a model checkpoint and loading it in order to create a tf-serving (pb) file. of course this use case is good in case export performance time is not an issue.
import tensorflow as tf
from tensorflow.python.framework import importer
output_path = '/export_directory' # be sure to create it before export
input_ops = ['name/s_of_model_input/s']
output_ops = ['name/s_of_model_output/s']
session = tf.compat.v1.Session()
def get_ops_dict(ops, graph, name='op_'):
out_dict = dict()
for i, op in enumerate(ops):
out_dict[name + str(i)] = tf.compat.v1.saved_model.build_tensor_info(graph.get_tensor_by_name(op + ':0'))
return out_dict
def add_meta_graph(pbtxt_tmp_path, graph_def):
with tf.Graph().as_default() as graph:
importer.import_graph_def(graph_def, name="")
os.unlink(pbtxt_tmp_path)
# used to rename model input/outputs
inputs_dict = get_ops_dict(input_ops, graph, name='input_')
outputs_dict = get_ops_dict(output_ops, graph, name='output_')
prediction_signature = (
tf.compat.v1.saved_model.signature_def_utils.build_signature_def(
inputs=inputs_dict,
outputs=outputs_dict,
method_name=tf.saved_model.PREDICT_METHOD_NAME))
legacy_init_op = tf.group(tf.compat.v1.tables_initializer(), name='legacy_init_op')
builder = tf.compat.v1.saved_model.builder.SavedModelBuilder(output_path+'/export')
builder.add_meta_graph_and_variables(
session,
tags=[tf.saved_model.SERVING],
signature_def_map={
tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature},
legacy_init_op=legacy_init_op)
builder.save()
return prediction_signature
def export_model(session, output_path, output_ops):
graph_def = session.graph_def
tf.io.write_graph(graph_or_graph_def=graph_def, logdir=output_path,
name='model.pbtxt', as_text=False)
frozen_graph_def = tf.compat.v1.graph_util.convert_variables_to_constants(
session, graph_def, output_ops)
prediction_signature = add_meta_graph(output_path+'/model.pbtxt', frozen_graph_def)
I have trained multiple tensorflow models for the same set of data, each with slightly different configuration.
Now I want to run prediction for the given input file utilizing each tensorflow model and store the prediction in a csv.
It seems I am unable to get tensorflow to completely unload/reset before loading new model.
Here is my code. It works fine for the first model, then it generates error. I have tried changing sequence of models, it always run the first model without any issue, no matter which model is first.
import tensorflow as tf
import os
import numpy as np
predictionoutputfile = 'data\\prediction.csv'
predictioninputfile = 'data\\today.csv'
modelslist = 'data\\models.csv'
def predict(dirname,testfield,testper,threshold,prediction_OutFile):
with tf.Session() as sess:
print(dirname)
exported_path = 'imp\\exported\\' + dirname
tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], exported_path)
# get the predictor , refer tf.contrib.predictor
predictor = tf.contrib.predictor.from_saved_model(exported_path)
with open(predictioninputfile) as inf:
# Skip header
#next(inf)
for line in inf:
# Read data, using python, into our features
var1,var2,var3,var4,var5 = line.strip().split(",")
# Create a feature_dict for train.example - Get Feature Columns using
feature_dict = {
'var1': _bytes_feature(value=var1.encode()),
'var2': _bytes_feature(value=var2.encode()),
'var3': _bytes_feature(value=var3.encode()),
'var4':_float_feature(value=int(var4)),
'var5':_float_feature(value=int(var5)),
}
# Prepare model input
model_input = tf.train.Example(features=tf.train.Features(feature=feature_dict))
model_input = model_input.SerializeToString()
output_dict = predictor({"inputs": [model_input]})
# Positive label = 1
if float(output_dict['scores'][0][1])>=float(threshold) :
prediction_OutFile.write(str(var1)+ "," + str(var2)+ "," + str(var3)+ "," + str(var4)+ "," + str(var5)+ ",")
label_index = tf.argmax(output_dict['scores'])
prediction_OutFile.write(str(output_dict['classes'][0][1]))
prediction_OutFile.write(',')
prediction_OutFile.write(str(output_dict['scores'][0][1]))
prediction_OutFile.write('\n')
def main():
prediction_OutFile = open(predictionoutputfile, 'w')
prediction_OutFile.write("model,SYMBOL,RECORDDATE,TESTFIELD,TESTPER,prediction,probability")
prediction_OutFile.write('\n')
with open(modelslist) as modlist:
#Skip header
next(modlist)
for mline in modlist:
try:
dirname = ''
modelname,datafield,dataper,testfield,testper,threshold,truepositive,falsepositive,truenegative,falsenegative,correct,incorrect,accuracy,precision = mline.strip().split(",")
# load the current model
predict(modelname,testfield,testper,threshold,prediction_OutFile)
# Read file and create feature_dict for each record
except:
print('error' + modelname)
prediction_OutFile.close()
def _float_feature(value):
return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
if __name__ == "__main__":
main()
You can, just use tf.reset_default_graph
# some stuff
with tf.Session() as sess:
# more stuff
tf.reset_default_graph()
# some otherstuff (again)
with tf.Session() as sess:
# more other stuff
The elephant in the room: Why not using flags call the python script multiple times?
I'm trying to fit multiple small Keras models in parallel on a single GPU. Because of reasons i need to get them out of a list and train them one step at a time. Since I was not lucky with the standard multiprocessing module i use pathos.
What I tried to do is something like this:
from pathos.multiprocessing import ProcessPool as Pool
import tensorflow as tf
import keras.backend as K
def multiprocess_step(self, model):
K.set_session(sess)
with sess.graph.as_default():
model = step(model, sess)
return model
def step(model, sess):
K.set_session(sess)
with sess.graph.as_default():
model.fit(x=data['X_train'], y=data['y_train'],
batch_size=batch_size
validation_data=(data['X_test'], data['y_test']),
verbose=verbose,
shuffle=True,
initial_epoch=self.step_num - 1)
return model
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = "0"
sess = tf.Session(config=config)
K.set_session(sess)
with sess.graph.as_default():
pool = Pool(8).map
model_list = pool(multiprocess_step, model_list)
but whatever I try I keep getting an error claiming that the models dont seem to be on the same graph...
ValueError: Tensor("training/RMSprop/Variable:0", shape=(25, 352), dtype=float32_ref) must be from the same graph as Tensor("RMSprop/rho/read:0", shape=(), dtype=float32).
The exception originates in the model.fit() row so I must have done something wrong with the assignment of the session graph even though I tried to set that in every possible location?
Does anyone have experience with something similar?
The following was suggested on the Keras issue tracker. I'm not sure about the relative merits of the approach compared to using multiprocessing.
in_1 = Input()
lstm_1 = LSTM(...)(in_1)
out_1 = Dense(...)(lstm_1)
in_2 = Input()
lstm_2 = LSTM(...)(in_2)
out_2 = Dense(...)(lstm_2)
model_1 = Model(input=in_1, output=out_1)
model_2 = Model(input=in_2, output=out_2)
model = Model(input = [in_1, in_2], output = [out_1, out_2])
model.compile(...)
model.fit(...)
model_1.predict(...)
model_2.predict(...)
Considering the backend is set to tensorflow for the keras. you can use code and do parallel processing for multiple model invocation/ multiple model loading.
def model1(dir_model):
model = os.path.join(dir_model, 'model.json')
dir_weights = os.path.join(dir_model, 'model.h5')
graph1 = Graph()
with graph1.as_default():
session1 = Session(graph=graph1, config=config)
with session1.as_default():
with open(model, 'r') as data:
model_json = data.read()
model_1 = model_from_json(model_json)
model_1.load_weights(dir_weights)
return model_1,gap_weights,session1,graph1
def model_2(dir_model):
model = os.path.join(dir_model, 'model.json')
dir_weights = os.path.join(dir_model, 'model.h5')
graph2 = Graph()
with graph2.as_default():
session2 = Session(graph=graph2, config=config)
with session2.as_default():
with open(model, 'r') as data:
model_json = data.read()
model_2 = model_from_json(model_json)
model_2.load_weights(dir_weights)
return model_2,session2,graph2
and for invocation of the specific model do the following experiments.
for model 1 predict do the following
K.set_session(session2)
with graph2.as_default():
img_pred[img_name] =
patch_dict[np.argmax(np.squeeze(model_2.predict(img_invoke)))
and for the model 2 it follows same as
K.set_session(session2)
with graph2.as_default():
img_pred[img_name] =
patch_dict[np.argmax(np.squeeze(model_2.predict(img_invoke)))]