The network consist of encoder, lstm and decoder.
The encoder take an input image sequence (TimeDistribution) and extract 2 outputs with different shape.
one use for lstm and pass it to decoder and another one use directly for decoder.
The decoder takes two inputs and output one average image with the same shape with input sequence.
How can I write a keras functional api for this structure?
This is my sample code I have tried:
class CustomEncoder(Architecture):
def __init__(self, inputShape=(128, 128, 1), batchSize=None,
latentSize=1000, training=None):
self.training=training
super().__init__(inputShape, batchSize )
def Build(self):
inLayer = Input(self.inputShape, self.batchSize)
net = Conv2D(64, kernelSize=15, strides=4)(inLayer, training=self.training) #
net = Conv2D(64, kernelSize=11, strides=2)(net, training=self.training)
net = Conv2D(128, kernelSize=7, strides=2)(net, training=self.training)
...
sample = ...
return Model(inputs=inLayer, outputs=[sample, net])
class CustomDecoder(Architecture):
def __init__(self, inputShape=(128, 128, 1), batchSize=None, latentSize=1000, training=None):
self.training=training
super().__init__(inputShape, batchSize, latentSize)
def Build(self):
# input layer is from GlobalAveragePooling:
inLayerA = Input([self.latentSize], self.batchSize)
inLayerB = Input([8, 8, 64], self.batchSize)
neta = Reshape((1, 1, self.latentSize))(inLayerA)
neta = UpSampling2D((self.inputShape[0]//16, self.inputShape[1]//16))(neta)
neta = DeconvBeta(128, kernelSize=7)(neta, training=self.training)
neta = DeconvBeta(64, kernelSize=11)(neta, training=self.training)
neta = DeconvBeta(64, kernelSize=15, strides=4)(neta, training=self.training)
netb = DeconvBeta(128, kernelSize=7)(inLayerB, training=self.training)
netb = DeconvBeta(64, kernelSize=11)(netb, training=self.training)
netb = DeconvBeta(64, kernelSize=15, strides=4)(netb, training=self.training)
neta = Conv2DTranspose(filters=1, kernel_size=15,
padding='same', activation="tanh")(neta)
netb = Conv2DTranspose(filters=1, kernel_size=15,
padding='same', activation="tanh")(netb)
net = Average([neta, netb])
return Model([inLayerA, inLayerB], net)
class ConvLSTMBeta(Architecture):
def __init__(self, inputShape=(None, 8, 8, 64), batchSize=None, latentSize=1000, training=None):
self.training=training
super().__init__(inputShape, batchSize, latentSize)
def Build(self):
# create the input layer for feeding the netowrk
inLayer = Input(self.inputShape, self.batchSize)
net = (ConvLSTM2D(64, (3, 3), padding="same",dropout=0.2, recurrent_dropout=0.2, return_sequences=True))(inLayer) # 1
return Model(inputs=inLayer, outputs=net)
Related
Is there a possibility to create my own block using keras
Maybe something like this:
import tensorflow as tf
class NewLayer(tf.keras.layers.Layer):
def __init__(self):
super(NewLayer, self).__init__()
self.dense1 = tf.keras.layers.Dense(64, activation='relu')
self.dense2 = tf.keras.layers.Dense(16, activation='relu')
self.dense3 = tf.keras.layers.Dense(3)
self._name = 'My_model'
def call(self, inputs):
x = tf.keras.layers.Flatten()(inputs)
x = self.dense1(x)
x = self.dense2(x)
output = self.dense3(x)
return output
inputs = tf.keras.layers.Input((128, 128, 3))
x = tf.keras.layers.Conv2D(256, 67)(inputs)
new_layer = NewLayer()
outputs = new_layer(x)
model = tf.keras.Model(inputs, outputs)
dot_img_file = 'model.png'
tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True)
I am trying to apply channel attention and position attention layer on last convolutional layer of VGG16. But stuck badly at the above error. I am new to coding in python and deep learning.
I am confused about shape values in Class CAM and class PAM.
Here is my code:
from keras import initializers
from keras import regularizers
from keras import constraints
class PAM(Layer):**strong text**
def __init__(self,
gamma_initializer=tf.zeros_initializer(),
gamma_regularizer=None,
gamma_constraint=None,
**kwargs):
super(PAM, self).__init__(**kwargs)
self.gamma_initializer = gamma_initializer
self.gamma_regularizer = gamma_regularizer
self.gamma_constraint = gamma_constraint
def build(self, input_shape):
self.gamma = self.add_weight(shape=(512,),
initializer=self.gamma_initializer,
name='gamma',
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
self.built = True
def compute_output_shape(self, input_shape):
return input_shape
def call(self, input):
input_shape = input.get_shape().as_list()
_, h, w, filters = input_shape
b = Conv2D(512, 3, use_bias=False, kernel_initializer='he_normal')(att_input)
c = Conv2D(512, 3, use_bias=False, kernel_initializer='he_normal')(att_input)
d = Conv2D(512, 3, use_bias=False, kernel_initializer='he_normal')(att_input)
vec_b = K.reshape(b, (-1, h * w, 512))
vec_cT = tf.transpose(K.reshape(c, (-1, h * w,512)), (0, 2, 1))
bcT = K.batch_dot(vec_b, vec_cT)
softmax_bcT = Activation('softmax')(bcT)
vec_d = K.reshape(d, (-1, h * w, 512))
bcTd = K.batch_dot(softmax_bcT, vec_d)
bcTd = K.reshape(bcTd, (-1, h, w, 512))
out = self.gamma*bcTd + att_input
return out
class CAM(Layer):
def __init__(self,
gamma_initializer=tf.zeros_initializer(),
gamma_regularizer=None,
gamma_constraint=None,
**kwargs):
super(CAM, self).__init__(**kwargs)
self.gamma_initializer = gamma_initializer
self.gamma_regularizer = gamma_regularizer
self.gamma_constraint = gamma_constraint
def build(self, input_shape):
self.gamma = self.add_weight(shape=(512,),
initializer=self.gamma_initializer,
name='gamma',
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
self.built = True
def compute_output_shape(self, input_shape):
return input_shape
def call(self, input):
input_shape = input.get_shape().as_list()
_, h, w, filters = input_shape
vec_a = K.reshape(input, (-1, h * w, 512))
vec_aT = tf.transpose(vec_a, (0, 2, 1))
aTa = K.batch_dot(vec_aT, vec_a)
softmax_aTa = Activation('softmax')(aTa)
aaTa = K.batch_dot(vec_a, softmax_aTa)
aaTa = K.reshape(aaTa, (-1, h, w, 512))
out = self.gamma*aaTa + att_input
return out
pam = PAM()(att_input)
pam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(pam)
pam = BatchNormalization(axis=3)(pam)
pam = Activation('relu')(pam)
pam = Dropout(0.5)(pam)
pam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(pam)
cam = CAM()(att_input)
cam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(cam)
cam = BatchNormalization(axis=3)(cam)
cam = Activation('relu')(cam)
cam = Dropout(0.5)(cam)
cam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(cam)
Make sure you have the input dimensions correct. Can't say exactly where the error is without looking at your code but whenever I've had an error like that it has almost always been a case where I overlooked one of the dimensions. Think carefully about how your input changes through the layers. Printing out the model summary might help. Good luck
I'm writting a custom keras model here is my code:
class Model(tf.keras.Model):
def __init__(self, first_layer, num_classes):
super(Model, self).__init__()
self.layer_1 = tf.keras.layers.Dense(first_layer, activation='relu')
self.layer_2 = tf.keras.layers.Dense(num_classes, activation='softmax')
def call(self,inp):
output = self.layer_1(inp)
output = self.layer_2(output)
return output
but I get this error:
ValueError: The last dimension of the inputs to `Dense` should be defined. Found `None`.
I found the tf.keras.input, but all the examples with that are with sequential models for example with a sequential model this is the solution with keras.input:
encoder_input = keras.Input(shape=(28, 28, 1), name="img")
x = layers.Conv2D(16, 3, activation="relu")(encoder_input)
x = layers.Conv2D(32, 3, activation="relu")(x)
but how I introduce this in a custom keras model?
please
version: TF 2.6
Edited code:
import tensorflow as tf
class Model(tf.keras.Model):
def __init__(self, first_layer, num_classes):
super(Model, self).__init__()
self.layer_1 = tf.keras.layers.Dense(first_layer, activation='relu')
self.layer_2 = tf.keras.layers.Dense(num_classes, activation='softmax')
def call(self,inp):
output = self.layer_1(inp)
output = self.layer_2(output)
return output
encoder_input = tf.keras.Input(shape=(28, 28, 1))
x = tf.keras.layers.Conv2D(16, 3, activation="relu")
#x = layers.Conv2D(32, 3, activation="relu")(x)
model = Model(5,3) #instantiated the model
y=model(encoder_input)#took input
print(y)
Output:
KerasTensor(type_spec=TensorSpec(shape=(None, 28, 28, 3),
dtype=tf.float32, name=None), name='model_4/dense_6/Softmax:0',
description="created by layer 'model_4'")
Reference:
https://www.tensorflow.org/guide/keras/custom_layers_and_models
I want to add attention layer before transfer learning, something like below, however, in the function create_model_tl I do not suppose to pass x yet, how to write a function to create a model so later on I can call the function to create model myModel = create_model_tl(input_shape, mdl_type, weights) and pass for compile and fit?
def attention(v, k, q, d_model, num_heads, mask):
......
return output
def create_model_tl(input_shape, mdl_type, weights):
print("start creating model - transfer learning ...")
conv_layer = Conv2D(16, kernel_size=(1, 1), activation='relu', input_shape=input_shape)
x = conv_layer(x)
x = attention(x, x, x, 16, 4, None) # k, q, v, d_model, num_heads, mask
base_model = vgg16.VGG16(include_top=False, input_shape=(x.shape[1], x.shape[2], x.shape[3], weights=weights)
flat1 = Flatten()(base_model.layers[-1].output)
class1 = Dense(1024, activation='relu')(flat1)
dropout1 = Dropout(0.2)(class1)
class2 = Dense(512, activation='relu')(dropout1)
dropout2 = Dropout(0.2)(class2)
output = Dense(2, activation='softmax')(dropout2)
#output = Dense(1, activation='sigmoid')(dropout2)
model = Model(inputs=base_model.inputs, outputs=output)
return model
this is my model :
# basic LeNet5 network
class LeNet5_mode0 (nn.Module) :
# constructor
def __init__(self):
super(LeNet5_mode0, self).__init__() # call to super constructor
# define layers
# 6 # 28x28
self.conv1 = nn.Sequential(
# Lenet's first conv layer is 3x32x32, squeeze color channels into 1 and pad 2
nn.Conv2d(in_channels = 1, out_channels = 6, kernel_size = 5, stride = 1, padding = 2),
nn.ReLU(),
nn.MaxPool2d(kernel_size = 2, stride = 2)
)
# 16 # 10x10
self.conv2 = nn.Sequential(
nn.Conv2d(in_channels = 6, out_channels = 16, kernel_size = 5, stride = 1, padding = 0),
nn.ReLU(),
nn.MaxPool2d(kernel_size =2, stride = 2)
)
self.fc1 = nn.Sequential(
nn.Linear(in_features = 16*5*5, out_features = 120),
nn.ReLU()
)
self.fc2 = nn.Sequential(
nn.Linear(in_features = 120, out_features = 84),
nn.ReLU()
)
self.classifier = nn.Sequential(
nn.Linear(in_features = 84,out_features = 10),
nn.Softmax(dim = 1) # dim =1 meaning do softmax on the colums of 84x10
)
# define forward function
def forward(self, x):
x = self.conv1(x)
x = self.conv2(x)
x = x.view(-1, 16*5*5) # reshape the tensor to [-1,16*5*5]
x = self.fc1(x)
x = self.fc2(x)
x = self.classifier(x)
return x
and I train this model once with :
criterion = nn.CrossEntropyLoss() # aka, LogLoss
optimizer = torch.optim.Adam(model.parameters(), lr=0.0005)
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[5,10,15], gamma=0.5)
and then save with with
torch.save(model.state_dict(), savepath)
and load it with
model.load_state_dict(torch.load(loadpath))
so far no problem . but when i change the optimizer a little to
optimizer = torch.optim.Adam(model.parameters(), lr=0.0005, weight_decay = 0.0005)
and use the same save & load method
I receive the following error:
in loading state_dict for LeNet5_mode0:
Unexpected key(s) in state_dict: "conv1.1.weight", "conv1.1.bias", "conv1.1.running_mean", "conv1.1.running_var", "conv1.1.num_batches_tracked", "conv2.1.weight", "conv2.1.bias", "conv2.1.running_mean", "conv2.1.running_var", "conv2.1.num_batches_tracked".
how can it be fixed? why different optimizer have that effect on the saving of the trained network?