Extracting last layers of keras model as a submodel - python

Say we have a convolutional neural network M. I can extract features from images by using
extractor = Model(M.inputs, M.get_layer('last_conv').output)
features = extractor.predict(X)
How can I get the model that will predict classes using features?
I can't use the following lines because it requires the input of the model to be a placeholder.
predictor = Model([M.get_layer('next_layer').input], M.outputs)
pred = predictor.predict(features)
I also can't use K.function because later I want to use it as part of another model, so I will be appliyng predictor to tf.tensor, not np.array.

This is not the nicest solution, but it works:
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Dense, Dropout, Flatten
def cnn():
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=(28, 28, 1), name='l_01'))
model.add(Conv2D(64, (3, 3), activation='relu', name='l_02'))
model.add(MaxPooling2D(pool_size=(2, 2), name='l_03'))
model.add(Dropout(0.25, name='l_04'))
model.add(Flatten(name='l_05'))
model.add(Dense(128, activation='relu', name='l_06'))
model.add(Dropout(0.5, name='l_07'))
model.add(Dense(10, activation='softmax', name='l_08'))
return model
def predictor(input_shape):
model = Sequential()
model.add(Flatten(name='l_05', input_shape=(12, 12, 64)))
model.add(Dense(128, activation='relu', name='l_06'))
model.add(Dropout(0.5, name='l_07'))
model.add(Dense(10, activation='softmax', name='l_08'))
return model
cnn_model = cnn()
cnn_model.save('/tmp/cnn_model.h5')
predictor_model = predictor(cnn_model.output.shape)
predictor_model.load_weights('/tmp/cnn_model.h5', by_name=True)

Every layer in the model is indexed. So if you know which layers you need, you could loop through them, copying them into a new model. This operation should copy the weights inside the layer as well.
Here's a model (from Oli Blum's answer):
model = Sequential()
# add some layers
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=(28, 28, 1), name='l_01'))
model.add(Conv2D(64, (3, 3), activation='relu', name='l_02'))
model.add(MaxPooling2D(pool_size=(2, 2), name='l_03'))
model.add(Dropout(0.25, name='l_04'))
model.add(Flatten(name='l_05'))
model.add(Dense(128, activation='relu', name='l_06'))
model.add(Dropout(0.5, name='l_07'))
model.add(Dense(10, activation='softmax', name='l_08'))
Say you wanted the last three layers:
def extract_layers(main_model, starting_layer_ix, ending_layer_ix):
# create an empty model
new_model = Sequential()
for ix in range(starting_layer_ix, ending_layer_ix + 1):
curr_layer = main_model.get_layer(index=ix)
# copy this layer over to the new model
new_model.add(curr_layer)
return new_model

It depends on what you want to do.
If you are going to throw away the feature extractor afterwards
If you plan on training the feature extractor later
If you are going to use the extracted features but you don't intend on training the model used to generate them, you could use the predict method to get the features as you did:
features = extractor.predict(X)
then save its output to a file (np.save or cPickle or whatever).
After that you could use that new dataset as the input to a new model.
If you plan on training the feature extractor later you'll need to stack the two networks as seen here with vgg as feature extractor https://github.com/fchollet/keras/issues/4576:
img_width, img_height = 150, 150
vgg16_model = VGG16(include_top=False, weights='imagenet')
input = Input(batch_shape=vgg16_model.output_shape)
x = GlobalAveragePooling2D()(input)
x = Dense(256, activation='relu')(x)
x = Dropout(0.5)(x)
predict = Dense(1, activation='sigmoid')(x)
top_model = Model(input, predict)
top_model.load_weights(os.path.join(data_path, 'VGG16Classifier.hdf5'))
input = Input(shape=(3, img_width, img_height))
x = vgg16_model(input)
predict = top_model(x)
model = Model(input, predict)
PS: This example uses channels first ordering. If you are using tensorflow you should change the shape to shape=(img_width, img_height,3 )

Related

raise TypeError('The added layer must be ' TypeError: The added layer must be an instance of class Layer. Found: tf.Tensor

I would like to know what is it that I am doing wrong here.
imports:
import glob
import rasterio as rs
import numpy as np
from matplotlib import pyplot as plt
import tensorflow as tf
from tensorflow import keras as kr
from tensorflow.keras import layers
from tensorflow.python.keras.layers.convolutional import Conv2D
from tensorflow.python.keras.layers.convolutional import MaxPooling2D
from tensorflow.python.keras.metrics import Accuracy, accuracy
The code that worked:
model = kr.Sequential([
kr.layers.Conv2D(32, (2,2), activation='relu', input_shape=(10,10,8)), # output is a 9x9x32 image
kr.layers.Conv2D(64, (4,4), activation='relu'), # output is a 6x6x64 image
kr.layers.MaxPooling2D(3,3), # output is a 2x2x64 image
kr.layers.Conv2D(1024, (2,2), activation='relu'), # output is a 1x1x1024 layer
kr.layers.Dropout(0.5),
kr.layers.Conv2D(3, (1,1), activation=tf.nn.softmax)
])
model.compile(optimizer='Adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.fit(train, train_labels, epochs=50)
print("results=",model.evaluate(test, test_labels))
The code throwing an error:
in_shape = (10,10,8)
conv1 = kr.layers.Conv2D(32, (2,2), activation='relu', input_shape=in_shape)(train)
print('shape of 2nd convolution', np.array(conv1).shape)
conv2 = kr.layers.Conv2D(64, (4,4), activation='relu')(conv1)
print('shape of 2nd convolution', np.array(conv2).shape)
pool1 = kr.layers.MaxPooling2D((3,3))(conv2)
print('shape of 1st pool', np.array(pool1).shape)
conv3 = kr.layers.Conv2D(1024, (2,2), activation='relu')(pool1)
print('shape of 3rd convolution', np.array(conv3).shape)
conv4 = kr.layers.Conv2D(3, (1,1), activation=tf.nn.softmax)(conv3)
print('shape of 4th convolution', np.array(conv4).shape)
# M O D E L
model = kr.Sequential([conv1, conv2, pool1, conv3, conv4])
model.compile(optimizer='Adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
model.fit(train, train_labels, epochs=50)
print("results=",model.evaluate(test, test_labels))
The error:
TypeError: The added layer must be an instance of class Layer. Found: tf.Tensor
This happens because you are mixing Keras Sequential model with Keras Functional API.
Briefly, when you do this:
foo = kr.layers.Conv2D(32, (2,2), activation='relu', input_shape=in_shape)(train)
because you are using the layer on a specified value (that is train), the result stored in foo will be a Tensor, as specified here:
A tensor of rank 4+ representing activation(conv2d(inputs, kernel) + bias).
This is indeed the typical way when you try to create a model using the Functional API.
However, as the error you get suggests:
The added layer must be an instance of Layer class. Found: tf.Tensor
a Keras Sequential model accepts something of Layer class. The problem is that when you do this:
model = kr.Sequential([conv1, conv2, pool1, conv3, conv4])
you are adding to the Sequential model a series of Tensors.
On the contrary, what you want to add to the Sequential model is a variable storing the Layer, so you should do this:
foo = kr.layers.Conv2D(32, (2, 2), activation="relu", input_shape=in_shape)
[...]
model = kr.Sequential([foo, ...])
note that there is no train at the end of the kr.layers.Conv2D layer, and the foo variable is actually storing the Layer,
or this:
model = kr.Sequential()
model.add(kr.layers.Conv2D(32, (2, 2), activation="relu", input_shape=in_shape))
[...]
The other piece of code you posted is actually working because you are indeed adding directly the Layer to the sequential model:
model = kr.Sequential([
kr.layers.Conv2D(32, (2,2), activation='relu', input_shape=(10,10,8)), # output is a 9x9x32 image
kr.layers.Conv2D(64, (4,4), activation='relu'), # output is a 6x6x64 image
kr.layers.MaxPooling2D(3,3), # output is a 2x2x64 image
kr.layers.Conv2D(1024, (2,2), activation='relu'), # output is a 1x1x1024 layer
kr.layers.Dropout(0.5),
kr.layers.Conv2D(3, (1,1), activation=tf.nn.softmax)
])
I was able to correct myself with the help of https://www.youtube.com/watch?v=pAhPiF3yiXI, what I wanted to do was to implement functional api in keras and the equivalent code for the same is as follows:
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
inp = layers.Input(shape=(10,10,8))
conv1 = layers.Conv2D(32, (2,2), activation='relu')(inp)
conv2 = layers.Conv2D(64, (4,4), activation='relu')(conv1)
pool1 = layers.MaxPooling2D(3,3)(conv2)
conv3 = layers.Conv2D(1024, (2,2), activation='relu')(pool1)
dr_out = layers.Dropout(0.5)(conv3)
out = layers.Conv2D(3, (1,1), activation='softmax')(dr_out)
model = keras.Model(inputs=inp, outputs=out)

How do i access data after model.add(Flatten()) layer?

I am trying to use CNN for feature extraction and XGboost for classification of a image data. I researched and found that it could be done by extracting the data after the convolution layers. I found some source code for similar problem and tried on my own.
model = Sequential()
model.add(Conv2D(32, kernel_size=(3,3), strides=(1,1), padding='same', activation="relu", input_shape = data.shape[1:]))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
model.add(Conv2D(64, kernel_size=(3,3), padding='same', strides=(1,1), activation="relu") )
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2))) #max pool window 2x2
model.add(Conv2D(128, kernel_size=(3,3), padding='same', strides=(1,1), activation="relu"))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2))) #max pool window 2x2
model.add(Conv2D(256, kernel_size=(3,3), padding='same', strides=(1,1), activation="relu"))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2))) #max pool window 2x2
model.add(Flatten())
model.add(Dense(128, activation="relu", name='firstDenseLayer'))
model.add(Dense(1, activation="sigmoid"))
# model.summary()
# print(model)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
model.fit(data, label, batch_size=16, epochs=10, validation_data=(val_data, val_label))
Below i accessed the dense layer named "firstDenseLayer".
import xgboost as xgb
from keras.models import Model
layerName = 'firstDenseLayer'
intermediate_layer_model = Model(inputs=model.input,
outputs=model.get_layer(layerName).output)
intermediate_output = intermediate_layer_model.predict(data)
from xgboost import XGBClassifier
xgbmodel = XGBClassifier(objective='multi:softmax', num_class= 2)
xgbmodel.fit(intermediate_output, label)
xgbmodel.score(intermediate_output, label)
As i am new in this, i have several confusions.
How the data is being flowed. After i extract the features of the pictures via convolution layers, how do i actually access the data from there?
What is this line of code doing? What data is it extracting?
intermediate_output = intermediate_layer_model.predict(data)
When i omit(keep commented out) the below line,
model.fit(data, label, batch_size=16, epochs=10, validation_data=(val_data, val_label))
from the first snippet and run the XGboost model directly the XGboost gives low accuracy and when i don't it gives higher accuracy. Why is it being like that?
Kindly help me out. I am stuck with this for quite a while. I am just trying to access the extracted features data from the last convolution layer and use that data to do classification using XGboost. As i tried to follow the method that i found from online, i am not sure if it is the the only way of doing it. If there is another way kindly let me know.
The model.fit(...) line does what you would expect, it trains the convnet defined by model on some data and labels. Your classifier yielding lower accuracy when you're using randomly initialized weights (i.e. without running fit) is not surprising.
intermediate_layer_model is constructed as a keras model whose output is the dense layer just before the output of model. Note the name parameter given to the dense layer in the construction of model.
You could just as easily give a name to one of the Conv2D layers and access it the same way. Alternatively, you could store the layer in a python variable, i.e. instead of
model.add(Conv2D(256, kernel_size=(3,3), padding='same', strides=(1,1), activation="relu"))
in the model construction it could say
last_conv_layer = Conv2D(256, kernel_size=(3,3), padding='same', strides=(1,1), activation="relu")
model.add(last_conv_layer)
Then for the intermediate_layer_model you put
intermediate_layer_model = Model(inputs=model.input, outputs=last_conv_layer.output)

Configuration of CNN model for recognition of sequential data - Architecture of the top of the CNN - Parallel Layers

I am trying to configure a network for character recognition of sequential data like license plates.
Now I would like to use the architecture which is noted in Table 3 in Deep Automatic Licence Plate Recognition system (link: http://www.ee.iisc.ac.in/people/faculty/soma.biswas/Papers/jain_icgvip2016_alpr.pdf).
The architecture the authors presented is this one:
The first layers are very common, but where I was stumbling was the top (the part in the red frame) of the architecture. They mention 11 parallel layers and I am really unsure how to get this in Python. I coded this architecture but it does not seem to be right to me.
model = Sequential()
model.add(Conv2D(64, kernel_size=(5, 5), input_shape = (32, 96, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=(3, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(256, kernel_size=(3, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1024, activation = "relu"))
model.add(Dense(11*37, activation="Softmax"))
model.add(keras.layers.Reshape((11, 37)))
Could someone help? How do I have to code the top to get an equal architecture like the authors?
The code below can build the architecture described in the image.
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Conv2D, Flatten, MaxPooling2D, Dense, Input, Reshape, Concatenate, Dropout
def create_model(input_shape = (32, 96, 1)):
input_img = Input(shape=input_shape)
'''
Add the ST Layer here.
'''
model = Conv2D(64, kernel_size=(5, 5), input_shape = input_shape, activation = "relu")(input_img)
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Dropout(0.25)(model)
model = Conv2D(128, kernel_size=(3, 3), input_shape = input_shape, activation = "relu")(model)
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Dropout(0.25)(model)
model = Conv2D(256, kernel_size=(3, 3), input_shape = input_shape, activation = "relu")(model)
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Dropout(0.25)(model)
model = Flatten()(model)
backbone = Dense(1024, activation="relu")(model)
branches = []
for i in range(11):
branches.append(backbone)
branches[i] = Dense(37, activation = "softmax", name="branch_"+str(i))(branches[i])
output = Concatenate(axis=1)(branches)
output = Reshape((11, 37))(output)
model = Model(input_img, output)
return model
From my understanding, your implementation is almost correct. The authors train 11 individual classifiers taking as input the output from the Fully Connected Layer. Here, you can think of "parallel" as "independent".
However, you cannot apply the Softmax activation right after the Fully Connected Layer. Since all the classifiers are independent, we want each of them to output a probability for each possible character. Putting things differently, we want the sum of the outputs of each classifier to be 1. Hence, the correct implementation would be:
...
model.add(Dense(1024, activation = "relu"))
# Feeding every neuron with the previous layer's output
model.add(Dense(11*37))
model.add(keras.layers.Reshape((11, 37)))
model.add(keras.activations.softmax(x, axis=1))

Facing issue while calling model summary keras

I am new to neural networks. I am trying to build CON2D + LSTM to recognize gestures from images. I have written below code:
input_shape = (120, 120, 3) // Having 120x120 images in RGB 3 channel
num_classes = 5 //Have to detect 5 gestures
model = Sequential()
# First conv, 32
model.add(TimeDistributed(Conv2D((32), (3,3), input_shape = input_shape)))
model.add(Activation('relu'))
model.add(TimeDistributed(Conv2D((32), (3,3))))
model.add(Activation('relu'))
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2))))
# Second conv, 64
model.add(TimeDistributed(Conv2D((64), (3, 3))))
model.add(Activation('relu'))
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2))))
model.add(TimeDistributed(Flatten()))
model.add(TimeDistributed(Dense(20)))
#configure LSTM
model.add(LSTM(20, activation='relu', return_sequences=False))
model.add(TimeDistributed(Dense(num_classes)))
model.add(Activation('softmax'))
model.compile(optimizer="adam", loss='categorical_crossentropy', metrics=['categorical_accuracy'])
print (model.summary())
I am getting below error after calling model.summary()
ValueError: This model has never been called, thus its weights have not yet been created, so no summary can be displayed. Build the model first (e.g. by calling it on some test data).
That's funny... What versions of TF and Keras are you using?
This would be common in a subclassed model, but sounds new to me in a Sequential model.
Just follow the instructions in the error:
test_predictions = model.predict(numpy.zeros((1,) + input_shape))
model.summary()

Keras CNN get output from the Convolutional steps

I am beginning to build CNN models using Keras.
I have built a CNN with a fairly accurate results using the following architecture.
classifier = Sequential()
classifier.add(Convolution2D(32, (3,3), input_shape = (64, 64, 3), activation='relu'))
classifier.add(MaxPool2D(pool_size = (2,2)))
classifier.add(Convolution2D(32, (3,3), activation='relu'))
classifier.add(MaxPool2D(pool_size = (2,2)))
classifier.add(Convolution2D(32, (3,3), activation='relu'))
classifier.add(MaxPool2D(pool_size = (2,2)))
classifier.add(Convolution2D(32, (3,3), activation='relu'))
classifier.add(MaxPool2D(pool_size = (2,2)))
classifier.add(Flatten())
classifier.add(Dense(units=128, activation='relu'))
classifier.add(Dropout(rate = 0.25))
classifier.add(Dense(units=128, activation='relu'))
classifier.add(Dropout(rate = 0.25))
classifier.add(Dense(units=1, activation='sigmoid'))
classifier.compile(optimizer = 'sgd', loss = 'binary_crossentropy', metrics=['accuracy'])
What I want to do is to run my images through the model, but only the convolutional steps. I am interested in the output of the Flattening process (i.e. get the features from the convolutional steps).
Can someone help me how I can get it in Keras?
Thanks in advance
Here is one solution. If you are interested in the output of layer 'max_pooling2d_4' (You can get the layer name by classifier.summary(), but I suggest you to put names for each layer by e.g. classifier.add(MaxPool2D(pool_size=(2,2), name='pool1'))):
layer_dict = dict([(layer.name, layer) for layer in classifier.layers])
# input tensor
input_tensor = classifier.input
# output tensor of the given layer
layer_output = layer_dict['max_pooling2d_4'].output
# get the output with respect to the input
func = K.function([input_tensor], [layer_output])
# test image: [64, 64, 3]
image = np.ones((64,64,3))
# get activation for the test image
activation = func([image[np.newaxis, :, :, :]])

Categories

Resources