How to choose the value of classifier_fn in tensorflow, I couldn't find any example about it:
tf.contrib.gan.eval.frechet_classifier_distance(
real_images,
generated_images,
classifier_fn,
num_batches=1
)
If you need the inception distance, then you can use a less generic function called tf.contrib.gan.eval.frechet_inception_distance which doesn't ask for a classifier_fn argument:
fid = tf.contrib.gan.eval.frechet_inception_distance(real_images, fake_images)
However, when I had tried to use this function using v1.14 with eager execution mode, I got errors of various kinds. So eventually, I've decided to go with a custom solution. Probably it would be helpful for you as well.
I encountered the following implementation by Jason Brownlee that seems to match the description from the original paper:
import numpy as np
import scipy.linalg
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.applications.inception_v3 import InceptionV3, preprocess_input
from tensorflow.compat.v1 import ConfigProto
from skimage.transform import resize
tf.enable_eager_execution()
config = ConfigProto()
config.gpu_options.allow_growth = True
tf.keras.backend.set_session(tf.Session(config=config))
def scale_images(images, new_shape):
return np.asarray([resize(image, new_shape, 0) for image in images])
def calculate_fid(model, images1, images2):
f1, f2 = [model.predict(im) for im in (images1, images2)]
mean1, sigma1 = f1.mean(axis=0), np.cov(f1, rowvar=False)
mean2, sigma2 = f2.mean(axis=0), np.cov(f2, rowvar=False)
sum_sq_diff = np.sum((mean1 - mean2)**2)
cov_mean = scipy.linalg.sqrtm(sigma1.dot(sigma2))
if np.iscomplexobj(cov_mean):
cov_mean = cov_mean.real
fid = sum_sq_diff + np.trace(sigma1 + sigma2 - 2.0*cov_mean)
return fid
if __name__ == '__main__':
input_shape = (299, 299, 3)
inception = InceptionV3(include_top=False, pooling='avg', input_shape=input_shape)
(dataset, _), _ = keras.datasets.cifar10.load_data()
dataset = dataset[:100]
dataset = scale_images(dataset, input_shape)
noise = preprocess_input(np.clip(255*np.random.uniform(size=dataset.shape), 0, 255))
noise = scale_images(noise, input_shape)
print('FID:', calculate_fid(inception, dataset, noise))
So we're performing the following steps:
re-scale images to the shape expected by InceptionV3;
transform the images using inception_v3.preprocess_input;
pass both tensors through InceptionV3 network (without top layer);
use the formula from the original paper with the computed features as input parameters.
Here is an excerpt from the mentioned paper.
Related
I have the following code that uses tensorflow to calculate a custom average loss when the image is consistently rotated:
import tensorflow as tf
import cv2
#initialize x_hat
img = cv2.imread("4.jpg")
x_hat = tf.Variable(img,name = 'x_hat') #img we want to attack
#tf.function
def cost2():
image=x_hat
#Now it will generate 100 samples rotated
num_samples = 100
average_loss = 0
for j in range(num_samples):
#ADD ROTATION (there may be a problem here)
rotated = tf.keras.preprocessing.image.random_rotation(image,
tf.random.uniform(shape=(),minval=40, maxval=90),channel_axis=2)
#get logits
rotated_logits, _ = resnet(rotated)
#get average CUSTOM loss
average_loss+=-1 * tf.nn.softmax_cross_entropy_with_logits(logits=rotated_logits, labels=labels)/ num_samples
return average_loss
and here is how I call it
learning_rate = 1e-1
optim = tf.optimizers.SGD (learning_rate=learning_rate)
epsilon = 2.0/255.0 # a really small perturbation
below = x - epsilon
above = x + epsilon
demo_steps = 200
# projected gradient descent
for i in range(demo_steps):
loss = optim.minimize(cost2, var_list=[x_hat])
if (i+1) % 10 == 0:
print('step %d, loss=%g' % (i+1, loss.numpy()))
projected = tf.clip_by_value(tf.clip_by_value(x_hat, below, above), 0, 1)
with tf.control_dependencies([projected]):
x_hat.assign(projected)
adv_robust = x_hat.numpy()
However, the following error returns to me once I run the code:
TypeError: in user code:
<ipython-input-183-abde02909da7>:14 cost2 *
rotated = tf.keras.preprocessing.image.random_rotation(image,
tf.random.uniform(shape=(),minval=40, maxval=90),channel_axis=2)
/home/me/.local/lib/python3.8/site-
packages/keras_preprocessing/image/affine_transformations.py:55 random_rotation *
theta = np.random.uniform(-rg, rg)
mtrand.pyx:1111 numpy.random.mtrand.RandomState.uniform **
TypeError: __array__() takes 1 positional argument but 2 were given
I am on Tensorflow 2.4.0 and the random_rotation and random.uniform functions are correct according to the TF 2.4.0 documentation HERE and HERE. So, what am I missing here?
The error might be coming from using TF tensors. As stated in the docs you linked regarding random_rotation:
Performs a random rotation of a Numpy image tensor.
Meaning you cannot use TF tensors with this operation. If you are in eager execution mode you can use tensor.numpy():
import tensorflow as tf
image = tf.random.normal((180, 180, 3))
rotated = tf.keras.preprocessing.image.random_rotation(image.numpy(),
tf.random.uniform(shape=(),minval=40, maxval=90).numpy(),channel_axis=2)
Otherwise, it is recommended to use the preprocessing layer: tf.keras.layers.RandomRotation, since using numpy in graph mode (for example in a function decorated with #tf.function) is not recommended.
Here is an example using the tf.keras.layers.RandomRotation:
import tensorflow as tf
import os
import matplotlib.pyplot as plt
_URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'
path_to_zip = tf.keras.utils.get_file('cats_and_dogs.zip', origin=_URL, extract=True)
PATH = os.path.join(os.path.dirname(path_to_zip), 'cats_and_dogs_filtered')
train_dir = os.path.join(PATH, 'train')
validation_dir = os.path.join(PATH, 'validation')
BATCH_SIZE = 1
IMG_SIZE = (160, 160)
train_ds = tf.keras.utils.image_dataset_from_directory(train_dir,
shuffle=True,
batch_size=BATCH_SIZE,
image_size=IMG_SIZE)
data_augmentation = tf.keras.Sequential([
tf.keras.layers.RandomRotation(tf.random.uniform(shape=(),minval=40, maxval=90)),
])
for image, _ in train_ds.take(1):
plt.figure(figsize=(10, 10))
first_image = image[0]
for i in range(9):
ax = plt.subplot(3, 3, i + 1)
augmented_image = data_augmentation(tf.expand_dims(first_image, 0), training=True)
plt.imshow(augmented_image[0] / 255)
plt.axis('off')
I am trying to optimize my filter activation using a pretrained model (vgg16) and reduce mean for filter score calculation. I am constantly getting an error that "No gradient provided for any variable".
I would really appreciate any help. Thanks!
Here you can see the code:
import numpy as np
import tensorflow as tf
from tensorflow import keras
np.random.seed(1)
image_f = np.random.normal(size=[1, 32, 32, 3], scale=0.01).astype(np.float32)
img = tf.nn.sigmoid(image_f)
tf.compat.v1.keras.backend.set_image_data_format('channels_last')
model = keras.applications.VGG16(weights="imagenet", include_top=False)
optimizer = tf.keras.optimizers.Adam(epsilon=1e-08, learning_rate=0.05)
layer_weight =keras.Model(inputs=model.inputs, outputs=model.get_layer(name="block3_conv1").output)
for i in range(5):
img = tf.Variable(img)
filter_activation = layer_weight(img)[:,:,:,5]
def compute_activation():
score = -1 * tf.reduce_mean(filter_activation)
print(score)
return score
optimizer.minimize(compute_activation, [img])
print(img)
I think the problem is your variable img is not included in the calculation of your loss function. I modified your code according to the documentation: https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer.
import numpy as np
import tensorflow as tf
from tensorflow import keras
np.random.seed(1)
image_f = np.random.normal(size=[1, 32, 32, 3], scale=0.01).astype(np.float32)
img = tf.nn.sigmoid(image_f)
tf.compat.v1.keras.backend.set_image_data_format('channels_last')
model = keras.applications.VGG16(weights="imagenet", include_top=False)
optimizer = tf.keras.optimizers.Adam(epsilon=1e-08, learning_rate=0.05)
layer_weight =keras.Model(inputs=model.inputs, outputs=model.get_layer(name="block3_conv1").output)
# Variable only need to define once
img = tf.Variable(img)
def compute_activation():
# Include variable img here
filter_activation = layer_weight(img)[:,:,:,5]
score = -1 * tf.reduce_mean(filter_activation)
print(score)
return score
for i in range(5):
optimizer.minimize(compute_activation, [img])
print(img)
I have a CNN that after a lot of work is now performing multiclass (8) classification at 99% accuracy.
While the classification itself has a lot of value, going to a prediction engine would be a game changer.
The catch is that the ability to predict is needed by a human in real life (IRL), not in being processed by a computer.
In this case the CNN is able to classify things faster than the human. It would be significant if the model could provide insights into how it is classifying things
Is there any way to decompose a Keras CNN model and/or it's weights to glean how it's arriving at it's decision? I don't believe so, but would hate not to ask and find out it's possible.
It's not that I'm looking for it to be exact, but if I can find one or two things that heavily influence the prediction/classification that a human being could key on, that could be significant.
Thoughts?
Is there any way to decompose a Keras CNN model and/or it's weights to glean how it's arriving at it's decision?
You can look at the Grad-CAM algorithm to see where the neural network was looking at before taking its final decision. Here's an implementation in Keras using a pre-trained model.
import numpy as np
import tensorflow as tf
from tensorflow import keras
from IPython.display import Image, display
import matplotlib.pyplot as plt
import matplotlib.cm as cm
model_builder = keras.applications.xception.Xception
img_size = (299, 299)
preprocess_input = keras.applications.xception.preprocess_input
decode_predictions = keras.applications.xception.decode_predictions
last_conv_layer_name = "block14_sepconv2_act"
classifier_layer_names = [
"avg_pool",
"predictions",
]
img_path = keras.utils.get_file(
"african_elephant.jpg", "https://i.imgur.com/Bvro0YD.png"
)
display(Image(img_path))
def get_img_array(img_path, size):
img = keras.preprocessing.image.load_img(img_path, target_size=size)
array = keras.preprocessing.image.img_to_array(img)
array = np.expand_dims(array, axis=0)
return array
def make_gradcam_heatmap(
img_array, model, last_conv_layer_name, classifier_layer_names
):
last_conv_layer = model.get_layer(last_conv_layer_name)
last_conv_layer_model = keras.Model(model.inputs, last_conv_layer.output)
classifier_input = keras.Input(shape=last_conv_layer.output.shape[1:])
x = classifier_input
for layer_name in classifier_layer_names:
x = model.get_layer(layer_name)(x)
classifier_model = keras.Model(classifier_input, x)
with tf.GradientTape() as tape:
last_conv_layer_output = last_conv_layer_model(img_array)
tape.watch(last_conv_layer_output)
preds = classifier_model(last_conv_layer_output)
top_pred_index = tf.argmax(preds[0])
top_class_channel = preds[:, top_pred_index]
grads = tape.gradient(top_class_channel, last_conv_layer_output)
pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))
last_conv_layer_output = last_conv_layer_output.numpy()[0]
pooled_grads = pooled_grads.numpy()
for i in range(pooled_grads.shape[-1]):
last_conv_layer_output[:, :, i] *= pooled_grads[i]
heatmap = np.mean(last_conv_layer_output, axis=-1)
heatmap = np.maximum(heatmap, 0) / np.max(heatmap)
return heatmap
img_array = preprocess_input(get_img_array(img_path, size=img_size))
model = model_builder(weights="imagenet")
preds = model.predict(img_array)
print("Predicted:", decode_predictions(preds, top=1)[0])
heatmap = make_gradcam_heatmap(
img_array, model, last_conv_layer_name, classifier_layer_names
)
img = keras.preprocessing.image.load_img(img_path)
img = keras.preprocessing.image.img_to_array(img)
heatmap = np.uint8(255 * heatmap)
jet = cm.get_cmap("jet")
jet_colors = jet(np.arange(256))[:, :3]
jet_heatmap = jet_colors[heatmap]
jet_heatmap = keras.preprocessing.image.array_to_img(jet_heatmap)
jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
jet_heatmap = keras.preprocessing.image.img_to_array(jet_heatmap)
superimposed_img = jet_heatmap * 0.4 + img
superimposed_img = keras.preprocessing.image.array_to_img(superimposed_img)
save_path = "elephant_cam.jpg"
superimposed_img.save(save_path)
display(Image(save_path))
I have found code online to get the derivative of the total loss with respect to the deep learning weights. I am trying to find the derivative of the weights with respect to the loss of a single class instead of all classes.
I used the following code to get the gradient of an input image with respect to the total loss. If I visualize it, it shows the importance of the pixels for all predictions. But, I would like to compute the derivative of the input image with respect to a particular class (e.g. "lady_bug"). This should show the importance of the pixels for the prediction of lady_bug. Do you have an idea how I can do that?
from keras.applications.vgg19 import VGG19
import numpy as np
import cv2
from keras import backend as K
import matplotlib.pyplot as plt
from keras.applications.inception_v3 import decode_predictions
def get_model():
model = VGG19(include_top=True, weights='imagenet')
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
def predict(model, images):
numeric_prediction = model.predict(images)
categorical_prediction = decode_predictions(numeric_prediction, top=1)
return [(x[0][1], x[0][2]) for x in categorical_prediction]
def get_test_image():
# Image
image_path = "lady_bug.jpg"
image = cv2.imread(image_path)
my_image = cv2.resize(image, (224,224))
my_image = np.expand_dims(my_image, axis=0)
return my_image
def visualize_sample(sample, file_path):
plt.figure()
plt.imshow(sample)
plt.savefig(file_path, bbox_inches='tight')
def test_input_gradient():
images = get_test_image()
model = get_model()
prediction = predict(model, images)
print(prediction)
gradients = K.gradients(model.output, model.input) #Gradient of output wrt the input of the model (Tensor)
print(gradients)
sess = K.get_session()
evaluated_gradients = sess.run(gradients[0], feed_dict={model.input:
images})
visualize_sample((evaluated_gradients[0]*(10**9.5)).clip(0,255), "test.png")
if __name__ == "__main__":
test_input_gradient()
Output:
[('ladybug', 0.53532666)]
[<tf.Tensor 'gradients/block1_conv1/convolution_grad/Conv2DBackpropInput:0' shape=(?, 224, 224, 3) dtype=float32>]
It seems the code is taking the gradients of the outputs wrt the inputs.
So, this is just taking a single slice from the outputs.
Warning: This considers a regular model output. I have no idea of what you're doing in decode predictions and the following list.
gradients = K.gradients(model.output[:, lady_bug_class], model.input)
I have trained a neural network model on MNIST dataset using the script mnist_3.1_convolutional_bigger_dropout.py provided in this tutorial.
I wanted to test the trained model on the custom dataset, hence I wrote a small script predict.py which loads the trained model and feed the data to it. I tried 2 methods for preprocessing images so that they are compatible with MNIST format.
Method 1: Resizing the image to 28x28
Method 2: Technique mentioned here is used
Both of these methods result in the error
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder_2' with dtype float
predict.py
# Importing libraries
from scipy.misc import imread
import tensorflow as tf
import numpy as np
import cv2 as cv
import glob
from test import imageprepare
files = glob.glob('data2/*.*')
#print(files)
# Method 1
'''
img_data = []
for fl in files:
img = imageprepare(fl)
img = img.reshape(img.shape[0], img.shape[1], 1)
img_data.append(img)
'''
# Method 2
dig_cont = [cv.imread(fl, 0) for fl in files]
#print(len(dig_cont))
img_data = []
for i in range(len(dig_cont)):
img = cv.resize(dig_cont[i], (28, 28))
img = img.reshape(img.shape[0], img.shape[1], 1)
img_data.append(img)
print("Restoring Model ...")
sess = tf.Session()
# Step-1: Recreate the network graph. At this step only graph is created.
tf_saver = tf.train.import_meta_graph('model/model.meta')
# Step-2: Now let's load the weights saved using the restore method.
tf_saver.restore(sess, tf.train.latest_checkpoint('model'))
print("Model restored")
x = tf.get_default_graph().get_tensor_by_name('X:0')
print('x :', x.shape)
y = tf.get_default_graph().get_tensor_by_name('Y:0')
print('y :', y.shape)
dict_data = {x: img_data}
result = sess.run(y, feed_dict=dict_data)
print(result)
print(result.shape)
sess.close()
The problem is fixed, I forgot to pass the value of variable pkeep. I had to make the following changes to make it work.
dict_data = {x: img_data, pkeep: 1.0}
instead of
dict_data = {x: img_data}