I tried to build my own deep dream algorithm with this code using the Inception Neural Network from Google:
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
#I am using the Kadenze CADL helper function ---> https://github.com/pkmital/CADL/tree/master/session-4/libs
import inception
img = np.random.rand(1,1920,1080,3)
net = inception.get_inception_model()
tf.import_graph_def(net['graph_def'], name='inception')
graph = tf.get_default_graph()
layer = graph.get_tensor_by_name('inception/mixed5b_pool_reduce_pre_relu:0')
gradient = tf.gradients(tf.reduce_mean(layer), img)
sess = tf.Session()
init = tf.global_variables_initializer()
iters = 1440
sess.run(init)
for i in range(iters):
print(i+1)
grad = sess.run(gradient[0])[0]
img += grad
plt.imshow(img[0])
plt.savefig('output/'+str(i+1)+'.png')
plt.close('all')
But the line tf.gradients(tf.reduce_mean(layer), img) only returns [None]. This (of course) causes an error. Can anyone tell me how to fix it?
Related
I am trying to optimize my filter activation using a pretrained model (vgg16) and reduce mean for filter score calculation. I am constantly getting an error that "No gradient provided for any variable".
I would really appreciate any help. Thanks!
Here you can see the code:
import numpy as np
import tensorflow as tf
from tensorflow import keras
np.random.seed(1)
image_f = np.random.normal(size=[1, 32, 32, 3], scale=0.01).astype(np.float32)
img = tf.nn.sigmoid(image_f)
tf.compat.v1.keras.backend.set_image_data_format('channels_last')
model = keras.applications.VGG16(weights="imagenet", include_top=False)
optimizer = tf.keras.optimizers.Adam(epsilon=1e-08, learning_rate=0.05)
layer_weight =keras.Model(inputs=model.inputs, outputs=model.get_layer(name="block3_conv1").output)
for i in range(5):
img = tf.Variable(img)
filter_activation = layer_weight(img)[:,:,:,5]
def compute_activation():
score = -1 * tf.reduce_mean(filter_activation)
print(score)
return score
optimizer.minimize(compute_activation, [img])
print(img)
I think the problem is your variable img is not included in the calculation of your loss function. I modified your code according to the documentation: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer.
import numpy as np
import tensorflow as tf
from tensorflow import keras
np.random.seed(1)
image_f = np.random.normal(size=[1, 32, 32, 3], scale=0.01).astype(np.float32)
img = tf.nn.sigmoid(image_f)
tf.compat.v1.keras.backend.set_image_data_format('channels_last')
model = keras.applications.VGG16(weights="imagenet", include_top=False)
optimizer = tf.keras.optimizers.Adam(epsilon=1e-08, learning_rate=0.05)
layer_weight =keras.Model(inputs=model.inputs, outputs=model.get_layer(name="block3_conv1").output)
# Variable only need to define once
img = tf.Variable(img)
def compute_activation():
# Include variable img here
filter_activation = layer_weight(img)[:,:,:,5]
score = -1 * tf.reduce_mean(filter_activation)
print(score)
return score
for i in range(5):
optimizer.minimize(compute_activation, [img])
print(img)
How to choose the value of classifier_fn in tensorflow, I couldn't find any example about it:
tf.contrib.gan.eval.frechet_classifier_distance(
real_images,
generated_images,
classifier_fn,
num_batches=1
)
If you need the inception distance, then you can use a less generic function called tf.contrib.gan.eval.frechet_inception_distance which doesn't ask for a classifier_fn argument:
fid = tf.contrib.gan.eval.frechet_inception_distance(real_images, fake_images)
However, when I had tried to use this function using v1.14 with eager execution mode, I got errors of various kinds. So eventually, I've decided to go with a custom solution. Probably it would be helpful for you as well.
I encountered the following implementation by Jason Brownlee that seems to match the description from the original paper:
import numpy as np
import scipy.linalg
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.applications.inception_v3 import InceptionV3, preprocess_input
from tensorflow.compat.v1 import ConfigProto
from skimage.transform import resize
tf.enable_eager_execution()
config = ConfigProto()
config.gpu_options.allow_growth = True
tf.keras.backend.set_session(tf.Session(config=config))
def scale_images(images, new_shape):
return np.asarray([resize(image, new_shape, 0) for image in images])
def calculate_fid(model, images1, images2):
f1, f2 = [model.predict(im) for im in (images1, images2)]
mean1, sigma1 = f1.mean(axis=0), np.cov(f1, rowvar=False)
mean2, sigma2 = f2.mean(axis=0), np.cov(f2, rowvar=False)
sum_sq_diff = np.sum((mean1 - mean2)**2)
cov_mean = scipy.linalg.sqrtm(sigma1.dot(sigma2))
if np.iscomplexobj(cov_mean):
cov_mean = cov_mean.real
fid = sum_sq_diff + np.trace(sigma1 + sigma2 - 2.0*cov_mean)
return fid
if __name__ == '__main__':
input_shape = (299, 299, 3)
inception = InceptionV3(include_top=False, pooling='avg', input_shape=input_shape)
(dataset, _), _ = keras.datasets.cifar10.load_data()
dataset = dataset[:100]
dataset = scale_images(dataset, input_shape)
noise = preprocess_input(np.clip(255*np.random.uniform(size=dataset.shape), 0, 255))
noise = scale_images(noise, input_shape)
print('FID:', calculate_fid(inception, dataset, noise))
So we're performing the following steps:
re-scale images to the shape expected by InceptionV3;
transform the images using inception_v3.preprocess_input;
pass both tensors through InceptionV3 network (without top layer);
use the formula from the original paper with the computed features as input parameters.
Here is an excerpt from the mentioned paper.
I have saved a trained model based on recurrent neural networks. When I run the following function 'lstm_vector_predict()', it returns a different value every time even though it is loading the same model. Does tensor flow use some random number generation when predicting values?
import get_list_of_values_to_input
import tensorflow as tf
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers
from tensorflow.contrib.learn.python.learn import learn_runner
import tensorflow.contrib.metrics as metrics
import tensorflow.contrib.rnn as rnn
import numpy as np
from backend.common.numpy_array_to_numpy_array_of_arrays import get_numpy_arrays_from_numpy_matrix
def lstm_vector_predict(model_name='sample_model_vector.meta', number_of_tickers=2, batch_size=20,number_of_points=100, start_time=1489462200):
tf.reset_default_graph()
inputs = number_of_tickers
hidden = 100
output = number_of_tickers
current_time = start_time
X = tf.placeholder(tf.float32, [None, batch_size, inputs])
# This is low level tensor flow stuff used for preparing output of data generation
basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=hidden, activation=tf.nn.relu)
rnn_output, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)
stacked_rnn_output = tf.reshape(rnn_output, [-1, hidden])
stacked_outputs = tf.layers.dense(stacked_rnn_output, output)
outputs = tf.reshape(stacked_outputs, [-1, batch_size, output])
# We get the saver ready
saver = tf.train.import_meta_graph(model_name)
init = tf.global_variables_initializer()
# Later, launch the model, use the saver to restore variables from disk, and
# do some work with the model.
return_values = []
with tf.Session() as sess:
# Restore variables from disk.
saver.restore(sess, tf.train.latest_checkpoint('./'))
print("Model restored.")
# Check the values of the variables
sess.run(init)
for i in range(number_of_points):
last_values = get_list_of_values_to_input()
print("Generating point", i)
#x_generators = last_values[-batch_size:]
x_generators = last_values[-batch_size:].reshape(-1, batch_size, number_of_tickers)
y_forecast = sess.run(outputs, feed_dict={X: x_generators})
return_values.append(y_forecast[-1][-1])
current_time += 300
return return_values
You will see different results because of the stochastic nature of the LSTM model, and because it is hard to fix the random seed for LSTM models to get 100% reproducible results.
I want to use a pretrained Inception-V3 model from Keras, paired with an input pipeline from Tensorflow (i.e. giving the network's input input via a tensor).
This is my code:
import tensorflow as tf
from keras.preprocessing.image import load_img, img_to_array
from keras.applications.inception_v3 import InceptionV3, decode_predictions, preprocess_input
import numpy as np
img_sample_filename = 'my_image.jpg'
img = img_to_array(load_img(img_sample_filename, target_size=(299,299)))
img = preprocess_input(img)
img_tensor = tf.constant(img[None,:])
# WITH KERAS:
model = InceptionV3()
pred = model.predict(img[None,:])
pred = decode_predictions(np.asarray(pred)) #<------ correct prediction!
print(pred)
# WITH TF:
model = InceptionV3(input_tensor=img_tensor)
init = tf.global_variables_initializer()
with tf.Session() as sess:
from keras import backend as K
K.set_session(sess)
sess.run(init)
pred = sess.run([model.output], feed_dict={K.learning_phase(): 0})
pred = decode_predictions(np.asarray(pred)[0])
print(pred) #<------ wrong prediction!
where my_image.jpg is any image I want to classify.
If I use keras' predict function to compute the prediction, the result is correct. If, however, I make a tensor out of the image array and feed that tensor to the model via input_tensor=... and then compute the prediction via sess.run([model.output], ...) the results are very wrong.
What is the reason of the different behaviour? Can't I use the Keras network in this way?
Finally, digging through the InceptionV3 code, I found the issue: sess.run(init) overwrites the weigts loaded in InceptionV3's constructor.
The -dirty- fix I found to this problem is reloading the weights after the sess.run(init).
from keras.applications.inception_v3 import get_file, WEIGHTS_PATH
with tf.Session() as sess:
from keras import backend as K
K.set_session(sess)
sess.run(init)
weights_path = get_file(
'inception_v3_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='9a0d58056eeedaa3f26cb7ebd46da564')
model.load_weights(weights_path)
pred = sess.run([model.output], feed_dict={K.learning_phase(): 0})
Note: The parameters for get_file() are taken directly from InceptionV3's constructor and, in my example, are specific to restoring the full network's weights with image_data_format='channels_last'.
I asked in this Github issue if there's a better workaround for this. I'll update this answer if I should get more information.
I am trying to use the trained model based on the Cifar10 tutorial and would like to feed
it with an external image 32x32 (jpg or png).
My goal is to be able to get the label as an output.
In other words, I want to feed the Network with a single jpeg image of size 32 x 32, 3 channels with no label as an input and have the inference process give me the tf.argmax(logits, 1).
Basically I would like to be able to use the trained cifar10 model on an external image and see what class it will spit out.
I have been trying to do that based on the Cifar10 Tutorial and unfortunately always have issues. especially with the Session concept and the batch concept.
Any help doing that with Cifar10 would be greatly appreciated.
Here is the implemented code so far with compilation issues :
#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from datetime import datetime
import math
import time
import tensorflow.python.platform
from tensorflow.python.platform import gfile
import numpy as np
import tensorflow as tf
import cifar10
import cifar10_input
import os
import faultnet_flags
from PIL import Image
FLAGS = tf.app.flags.FLAGS
def evaluate():
filename_queue = tf.train.string_input_producer(['/home/tensor/.../inputImage.jpg'])
reader = tf.WholeFileReader()
key, value = reader.read(filename_queue)
input_img = tf.image.decode_jpeg(value)
init_op = tf.initialize_all_variables()
# Problem in here with Graph / session
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(1):
image = input_img.eval()
print(image.shape)
Image.fromarray(np.asarray(image)).show()
# Problem in here is that I have only one image as input and have no label and would like to have
# it compatible with the Cifar10 network
reshaped_image = tf.cast(image, tf.float32)
height = FLAGS.resized_image_size
width = FLAGS.resized_image_size
resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image, width, height)
float_image = tf.image.per_image_whitening(resized_image) # reshaped_image
num_preprocess_threads = 1
images = tf.train.batch(
[float_image],
batch_size=128,
num_threads=num_preprocess_threads,
capacity=128)
coord.request_stop()
coord.join(threads)
logits = faultnet.inference(images)
# Calculate predictions.
#top_k_predict_op = tf.argmax(logits, 1)
# print('Current image is: ')
# print(top_k_predict_op[0])
# this does not work since there is a problem with the session
# and the Graph conflicting
my_classification = sess.run(tf.argmax(logits, 1))
print ('Predicted ', my_classification[0], " for your input image.")
def main(argv=None):
evaluate()
if __name__ == '__main__':
tf.app.run() '''
Some basics first:
First you define your graph: image queue, image preprocessing, inference of the convnet, top-k accuracy
Then you create a tf.Session() and work inside it: starting the queue runners, and calls to sess.run()
Here is what your code should look like
# 1. GRAPH CREATION
filename_queue = tf.train.string_input_producer(['/home/tensor/.../inputImage.jpg'])
... # NO CREATION of a tf.Session here
float_image = ...
images = tf.expand_dims(float_image, 0) # create a fake batch of images (batch_size=1)
logits = faultnet.inference(images)
_, top_k_pred = tf.nn.top_k(logits, k=5)
# 2. TENSORFLOW SESSION
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
top_indices = sess.run([top_k_pred])
print ("Predicted ", top_indices[0], " for your input image.")
EDIT:
As #mrry suggests, if you only need to work on a single image, you can remove the queue runners:
# 1. GRAPH CREATION
input_img = tf.image.decode_jpeg(tf.read_file("/home/.../your_image.jpg"), channels=3)
reshaped_image = tf.image.resize_image_with_crop_or_pad(tf.cast(input_img, width, height), tf.float32)
float_image = tf.image.per_image_withening(reshaped_image)
images = tf.expand_dims(float_image, 0) # create a fake batch of images (batch_size = 1)
logits = faultnet.inference(images)
_, top_k_pred = tf.nn.top_k(logits, k=5)
# 2. TENSORFLOW SESSION
with tf.Session() as sess:
sess.run(init_op)
top_indices = sess.run([top_k_pred])
print ("Predicted ", top_indices[0], " for your input image.")
The original source code in cifar10_eval.py can also be used for testing own individual images as it is shown in the following console output
nbatfai#robopsy:~/Robopsychology/repos/gpu/tensorflow/tensorflow/models/image/cifar10$ python cifar10_eval.py --run_once True 2>/dev/null
[ -0.63916457 -3.31066918 2.32452989 1.51062226 15.55279636
-0.91585422 1.26451302 -4.11891603 -7.62230825 -4.29096413]
deer
nbatfai#robopsy:~/Robopsychology/repos/gpu/tensorflow/tensorflow/models/image/cifar10$ python cifar2bin.py matchbox.png input.bin
nbatfai#robopsy:~/Robopsychology/repos/gpu/tensorflow/tensorflow/models/image/cifar10$ python cifar10_eval.py --run_once True 2>/dev/null
[ -1.30562115 12.61497402 -1.34208572 -1.3238833 -6.13368177
-1.17441642 -1.38651907 -4.3274951 2.05489922 2.54187846]
automobile
nbatfai#robopsy:~/Robopsychology/repos/gpu/tensorflow/tensorflow/models/image/cifar10$
and code snippet
#while step < num_iter and not coord.should_stop():
# predictions = sess.run([top_k_op])
print(sess.run(logits[0]))
classification = sess.run(tf.argmalogits[0], 0))
cifar10classes = ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"]
print(cifar10classes[classification])
#true_count += np.sum(predictions)
step += 1
# Compute precision # 1.
precision = true_count / total_sample_count
# print('%s: precision # 1 = %.3f' % (datetime.now(), precision))
More details can be found in the post How can I test own image to Cifar-10 tutorial on Tensorflow?