I'm using keras to build a convolutional neural network for image segmentation and I want to use "reflection padding" instead of padding "same" but I cannot find a way to to do it in keras.
inputs = Input((num_channels, img_rows, img_cols))
conv1=Conv2D(32,3,padding='same',kernel_initializer='he_uniform',data_format='channels_first')(inputs)
Is there a way to implement a reflection layer and insert it in a keras model ?
The accepted answer above is not working in the current Keras version. Here is the version that's working:
class ReflectionPadding2D(Layer):
def __init__(self, padding=(1, 1), **kwargs):
self.padding = tuple(padding)
self.input_spec = [InputSpec(ndim=4)]
super(ReflectionPadding2D, self).__init__(**kwargs)
def compute_output_shape(self, s):
""" If you are using "channels_last" configuration"""
return (s[0], s[1] + 2 * self.padding[0], s[2] + 2 * self.padding[1], s[3])
def call(self, x, mask=None):
w_pad,h_pad = self.padding
return tf.pad(x, [[0,0], [h_pad,h_pad], [w_pad,w_pad], [0,0] ], 'REFLECT')
Found the solution! We have only to create a new class that takes a layer as input and use tensorflow predefined function to do it.
import tensorflow as tf
from keras.engine.topology import Layer
from keras.engine import InputSpec
class ReflectionPadding2D(Layer):
def __init__(self, padding=(1, 1), **kwargs):
self.padding = tuple(padding)
self.input_spec = [InputSpec(ndim=4)]
super(ReflectionPadding2D, self).__init__(**kwargs)
def get_output_shape_for(self, s):
""" If you are using "channels_last" configuration"""
return (s[0], s[1] + 2 * self.padding[0], s[2] + 2 * self.padding[1], s[3])
def call(self, x, mask=None):
w_pad,h_pad = self.padding
return tf.pad(x, [[0,0], [h_pad,h_pad], [w_pad,w_pad], [0,0] ], 'REFLECT')
# a little Demo
inputs = Input((img_rows, img_cols, num_channels))
padded_inputs= ReflectionPadding2D(padding=(1,1))(inputs)
conv1 = Conv2D(32, 3, padding='valid', kernel_initializer='he_uniform',
data_format='channels_last')(padded_inputs)
import tensorflow as tf
from keras.layers import Lambda
inp_padded = Lambda(lambda x: tf.pad(x, [[0,0], [27,27], [27,27], [0,0]], 'REFLECT'))(inp)
The solution from Akihiko did not work with the new keras version, so I came up with my own. The snippet pads a batch of 202x202x3 images to 256x256x3
As you can check in the documentation there is no such 'reflect' padding. Only 'same' and 'valid' are implemented in keras.
You maybe try to implemented on your own or find if somebody already did it. You should base yourself in the Conv2D class and check where self.padding member variable is used.
The accepted answer does not work if we have undefined dimensions! There will be an error when compute_output_shape function is called. Here is the simple work around to that.
class ReflectionPadding2D(Layer):
def __init__(self, padding=(1, 1), **kwargs):
self.padding = tuple(padding)
self.input_spec = [InputSpec(ndim=4)]
super(ReflectionPadding2D, self).__init__(**kwargs)
def compute_output_shape(self, s):
if s[1] == None:
return (None, None, None, s[3])
return (s[0], s[1] + 2 * self.padding[0], s[2] + 2 * self.padding[1], s[3])
def call(self, x, mask=None):
w_pad, h_pad = self.padding
return tf.pad(x, [[0, 0], [h_pad, h_pad], [w_pad, w_pad], [0, 0]], 'REFLECT')
def get_config(self):
config = super(ReflectionPadding2D, self).get_config()
print(config)
return config
Related
Recently, I want to implement CDCN in CVPR2020 using tensorflow2.8 + python3.9. This requires my custom layer acquire current conv2d layer's weight.
However, when I try to add my custom layer to the sequential model, error occurred:NotImplementedError: numpy() is only available when eager execution is enabled.
This is my code. Can anyone helps me? I have tried to add tf.compat.v1.enable_eager_execution(), but it doesn't work.
import numpy as np
import tensorflow.keras as tfk
import tensorflow as tf
class CDC(tfk.layers.Layer):
def __init__(self, output_dim, kernel_size=(3, 3), padding='same', activation=None, theta=0.7, **kwargs):
super(CDC, self).__init__()
self.theta = theta
self.activation = None
self.output_dim = output_dim
self.kernel_size = kernel_size
self.padding = padding
if activation is not None:
self.activation = tfk.activations.get(activation)
def build(self, input_shape):
self.conv = tfk.layers.Conv2D(self.output_dim, self.kernel_size, padding=self.padding, input_shape=input_shape)
self.conv.build(input_shape=input_shape)
self._kernel = self.conv.kernel
super(CDC, self).build(input_shape)
self.built = True
def call(self, inputs, training=None, mask=None):
vanillaOutput = self.conv(inputs)
weightSum = self.conv.kernel.numpy().sum(axis=0).sum(axis=0).sum(axis=0)
weightSum = np.reshape(weightSum, (1, 1, 1, self.output_dim))
weightSum = tf.constant(weightSum, dtype=tf.float32)
cDiff = tf.nn.conv2d(inputs, filters=weightSum, strides=self.conv.strides, padding=self.conv.padding.upper())
result = vanillaOutput - self.theta * cDiff
if self.activation is not None:
return self.activation(result)
return vanillaOutput
If you just want the sum of all elements in kernel
tf.math.reduce_sum()
Also replace the lines
weightSum = self.conv.kernel.numpy().sum(axis=0).sum(axis=0).sum(axis=0)
weightSum = np.reshape(weightSum, (1, 1, 1, self.output_dim))
weightSum = tf.constant(weightSum, dtype=tf.float32)
...
weightSum = tf.math.reduce_sum(tf.math.reduce_sum(tf.math.reduce_sum(self.conv.kernel,axis=0),axis=0),axis=0)
weightSum = tf.reshape(weightSum, (1, 1, 1, self.output_dim))
Models implemented as subclasses of keras. Model can generally not be visualized with plot_model. There is a workaround as described here. However, it only applies to simple models. As soon as a model is enclosed by another model, the nestings will not be resolved.
I am looking for a way to resolve nested models implemented as subclasses of the keras. Model. As an example, I have created a minimal GAN model:
import keras
from keras import layers
from tensorflow.python.keras.utils.vis_utils import plot_model
class BaseModel(keras.Model):
def __init__(self, *args, **kwargs):
super(BaseModel, self).__init__(*args, **kwargs)
def call(self, inputs, training=None, mask=None):
super(BaseModel, self).call(inputs=inputs, training=training, mask=mask)
def get_config(self):
super(BaseModel, self).get_config()
def build_graph(self, raw_shape):
""" Plot models that subclass `keras.Model`
Adapted from https://stackoverflow.com/questions/61427583/how-do-i-plot-a-keras-tensorflow-subclassing-api-model
:param raw_shape: Shape tuple not containing the batch_size
:return:
"""
x = keras.Input(shape=raw_shape)
return keras.Model(inputs=[x], outputs=self.call(x))
class GANModel(BaseModel):
def __init__(self, generator, discriminator):
super(GANModel, self).__init__()
self.generator = generator
self.discriminator = discriminator
def call(self, input_tensor, training=False, mask=None):
x = self.generator(input_tensor)
x = self.discriminator(x)
return x
class DiscriminatorModel(BaseModel):
def __init__(self, name="Critic"):
super(DiscriminatorModel, self).__init__(name=name)
self.l1 = layers.Conv2D(64, 2, activation=layers.ReLU())
self.flat = layers.Flatten()
self.dense = layers.Dense(1)
def call(self, inputs, training=False, mask=None):
x = self.l1(inputs, training=training)
x = self.flat(x)
x = self.dense(x, training=training)
return x
class GeneratorModel(BaseModel):
def __init__(self, name="Generator"):
super(GeneratorModel, self).__init__(name=name)
self.dense = layers.Dense(128, activation=layers.ReLU())
self.reshape = layers.Reshape((7, 7, 128))
self.out = layers.Conv2D(1, (7, 7), activation='tanh', padding="same")
def call(self, inputs, training=False, mask=None):
x = self.dense(inputs, training=training)
x = self.reshape(x)
x = self.out(x, training=training)
return x
g = GeneratorModel()
d = DiscriminatorModel()
plot_model(g.build_graph((7, 7, 1)), to_file="generator_model.png",
expand_nested=True, show_shapes=True)
gan = GANModel(generator=g, discriminator=d)
plot_model(gan.build_graph((7, 7, 1)), to_file="gan_model.png",
expand_nested=True, show_shapes=True)
Edit
Using the functional keras API I get the desired result (see here). The nested models are correctly resolved within the GAN model.
from keras import Model, layers, optimizers
from tensorflow.python.keras.utils.vis_utils import plot_model
def get_generator(input_dim):
initial = layers.Input(shape=input_dim)
x = layers.Dense(128, activation=layers.ReLU())(initial)
x = layers.Reshape((7, 7, 128))(x)
x = layers.Conv2D(1, (7, 7), activation='tanh', padding="same")(x)
return Model(inputs=initial, outputs=x, name="Generator")
def get_discriminator(input_dim):
initial = layers.Input(shape=input_dim)
x = layers.Conv2D(64, 2, activation=layers.ReLU())(initial)
x = layers.Flatten()(x)
x = layers.Dense(1)(x)
return Model(inputs=initial, outputs=x, name="Discriminator")
def get_gan(input_dim, latent_dim):
initial = layers.Input(shape=input_dim)
x = get_generator(input_dim)(initial)
x = get_discriminator(latent_dim)(x)
return Model(inputs=initial, outputs=x, name="GAN")
m = get_generator((7, 7, 1))
m.compile(optimizer=optimizers.Adam())
plot_model(m, expand_nested=True, show_shapes=True, to_file="generator_model_functional.png")
gan = get_gan((7, 7, 1), (7, 7, 1))
plot_model(gan, expand_nested=True, show_shapes=True, to_file="gan_model_functional.png")
Whenever you pass each generator and discriminator to GANModel, they act like an encompassed child layer consisting of n times layers. So, if you plot only the generator model by the GANModel instances, it will show as follows (same goes to discriminator) unlike plots while using them separately.
The fact is while we pass data at this point using the call() method of GANModel, the input passes implicitly all internal layers (generator, discriminator) according to its design. Here I will show you two workaround for this to get your desired plot.
Option 1
I believe you probably guess the method. In the GANModel model, we will pass the input very explicitly to each internal layer of those child layers (generator, discriminator).
class GANModel(BaseModel):
def __init__(self, generator, discriminator):
super(GANModel, self).__init__()
self.generator = generator
self.discriminator = discriminator
def call(self, input_tensor, training=False, mask=None):
x = input_tensor
for gen_lyr in self.generator.layers:
print(gen_lyr) # checking
x = gen_lyr(x)
for disc_lyr in self.discriminator.layers:
print(disc_lyr) # checking
x = disc_lyr(x)
return x
If you plot now, you will get
# All Internal Layers of self.generator, self.discriminator
<tensorflow.python.keras.layers.core.Dense object at 0x7f2a472a3710>
<tensorflow.python.keras.layers.core.Reshape object at 0x7f2a461e8f50>
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x7f2a44591f90>
<tensorflow.python.keras.layers.convolutional.Conv2D object at 0x7f2a47317290>
<tensorflow.python.keras.layers.core.Flatten object at 0x7f2a47317ed0>
<tensorflow.python.keras.layers.core.Dense object at 0x7f2a57f42910>
Option 2
I think it's a bit ugly approach. First, we take each internal layer and build a Sequential model with them. Then use .build to create its input layer. BOOM.
gan = GANModel(generator=g, discriminator=d)
all_layer = []
for layer in gan.layers:
all_layer.extend(layer.layers)
gan_plot = tf.keras.models.Sequential(all_layer)
gan_plot.build((None,7,7,1))
list(all_layer)
[<tensorflow.python.keras.layers.core.Dense at 0x7f2a461ab390>,
<tensorflow.python.keras.layers.core.Reshape at 0x7f2a46156110>,
<tensorflow.python.keras.layers.convolutional.Conv2D at 0x7f2a461fedd0>,
<tensorflow.python.keras.layers.convolutional.Conv2D at 0x7f2a461500d0>,
<tensorflow.python.keras.layers.core.Flatten at 0x7f2a4613ea10>,
<tensorflow.python.keras.layers.core.Dense at 0x7f2a462cae10>]
tf.keras.utils.plot_model(gan_plot, expand_nested=True, show_shapes=True)
Main objective
PyTorch equivalent for SeparableConv2D with padding = 'same':
from tensorflow.keras.layers import SeparableConv2D
x = SeparableConv2D(64, (1, 16), use_bias = False, padding = 'same')(x)
What is the PyTorch equivalent for SeparableConv2D?
This source says:
If groups = nInputPlane, kernel=(K, 1), (and before is a Conv2d layer with groups=1 and kernel=(1, K)), then it is separable.
While this source says:
Its core idea is to break down a complete convolutional acid into a two-step calculation, Depthwise Convolution and Pointwise.
This is my attempt:
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, depth, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, out_channels*depth, kernel_size=kernel_size, groups=in_channels, bias=bias)
self.pointwise = nn.Conv2d(out_channels*depth, out_channels, kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
Is this correct? Is this equivalent to tensorflow.keras.layers.SeparableConv2D?
What about padding = 'same'?
How to ensure that my input and output size is the same while doing this?
My attempt:
x = F.pad(x, (8, 7, 0, 0), )
Because the kernel size is (1,16), I added left and right padding, 8 and 7 respectively. Is this the right way (and best way) to achieve padding = 'same'? How can I place this inside my SeparableConv2d class, and calculate on the fly given the input data dimension size?
All together
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, depth, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, out_channels*depth, kernel_size=kernel_size, groups=in_channels, bias=bias)
self.pointwise = nn.Conv2d(out_channels*depth, out_channels, kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.separable_conv = SeparableConv2d(
in_channels=32,
out_channels=64,
depth=1,
kernel_size=(1,16)
)
def forward(self, x):
x = F.pad(x, (8, 7, 0, 0), )
x = self.separable_conv(x)
return x
Any problem with these codes?
The linked definitions are generally agreeing. The best one is in the article.
"Depthwise" (not a very intuitive name since depth is not involved) - is a series of regular 2d convolutions, just applied to layers of the data separately. - "Pointwise" is same as Conv2d with 1x1 kernel.
I suggest a few correction to your SeparableConv2d class:
no need to use depth parameter - it is same as out_channels
I set padding to 1 to ensure same output size with kernel=(3,3). if kernel size is different - adjust padding accordingly, using same principles as with regular Conv2d. Your example class Net() is no longer needed - padding is done in SeparableConv2d.
This is the updated code, should be similar to tf.keras.layers.SeparableConv2D implementation:
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, in_channels, kernel_size=kernel_size,
groups=in_channels, bias=bias, padding=1)
self.pointwise = nn.Conv2d(in_channels, out_channels,
kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
I have a model written in TF 1.x code using tf-slim API as well. Is it possible to convert that to tf.keras in TF 2.0 EXACTLY the way it is? For instance, have exactly the same amount of parameters and training?
In my case, I've tried doing so, but my model in tf.keras actually has about 5% LESS parameters than the one in TF 1.x. I also noticed my model in tf.keras has a much less smooth training stage too. Any thoughts? Thanks
Maybe I'm setting some of the parameters to initialize the layers differently? Any other suggestions would be greatly appreciated
This isn't my full model, but I use a lot of the components below:
Original TF.1x model:
import tensorflow as tf
from tensorflow.contrib import slim
def batch_norm_relu(inputs, is_training):
net = slim.batch_norm(inputs, is_training=is_training)
net = tf.nn.relu(net)
return net
def conv2d_transpose(inputs, output_channels, kernel_size):
upsamp = tf.contrib.slim.conv2d_transpose(
inputs,
num_outputs=output_channels,
kernel_size=kernel_size,
stride=2,
)
return upsamp
def conv2d_fixed_padding(inputs, filters, kernel_size, stride, rate):
net = slim.conv2d(inputs,
filters,
kernel_size,
stride=stride,
rate = rate,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
return net
def block(inputs, filters, is_training, projection_shortcut, stride):
inputs = batch_norm_relu(inputs, is_training)
shortcut = inputs
if projection_shortcut is not None:
shortcut = projection_shortcut(inputs)
conv_k1_s1_r1 = shortcut
conv_k3_s1_r1 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 1,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
conv_k3_s1_r3 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 3,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
conv_k3_s1_r5 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 5,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
net = conv_k1_s1_r1 + conv_k3_s1_r1 + conv_k3_s1_r3 + conv_k3_s1_r5
net = batch_norm_relu(net, is_training)
net = conv2d_fixed_padding(inputs=net, filters=filters, kernel_size=1, stride=1, rate = 1)
outputs = shortcut + net
return outputs
Attempted TF 2.x.keras model same component:
import tensorflow as tf
class BatchNormRelu(tf.keras.layers.Layer):
"""Batch normalization + ReLu"""
def __init__(self, name=None):
super(BatchNormRelu, self).__init__(name=name)
self.bnorm = tf.keras.layers.BatchNormalization(momentum=0.999,
scale=False)
self.relu = tf.keras.layers.ReLU()
def call(self, inputs, is_training):
x = self.bnorm(inputs, training=is_training)
x = self.relu(x)
return x
class Conv2DTranspose(tf.keras.layers.Layer):
"""Conv2DTranspose layer"""
def __init__(self, output_channels, kernel_size, name=None):
super(Conv2DTranspose, self).__init__(name=name)
self.tconv1 = tf.keras.layers.Conv2DTranspose(
filters=output_channels,
kernel_size=kernel_size,
strides=2,
padding='same',
activation=tf.keras.activations.relu
)
def call(self, inputs):
x = self.tconv1(inputs)
return x
class Conv2DFixedPadding(tf.keras.layers.Layer):
"""Conv2D Fixed Padding layer"""
def __init__(self, filters, kernel_size, stride, rate, name=None):
super(Conv2DFixedPadding, self).__init__(name=name)
self.conv1 = tf.keras.layers.Conv2D(filters,
kernel_size,
strides=stride,
dilation_rate=rate,
padding=('same' if stride==1 else 'valid'),
activation=None
)
def call(self, inputs):
x = self.conv1(inputs)
return x
class block(tf.keras.layers.Layer):
def __init__(self,
filters,
stride,
projection_shortcut=True,
name=None):
super(block, self).__init__(name=name)
self.projection_shortcut = projection_shortcut
self.brelu1 = BatchNormRelu()
self.brelu2 = BatchNormRelu()
self.conv1 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=1,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv2 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=3,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv3 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=5,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv4 = Conv2DFixedPadding(filters, 1, 1, 1)
self.conv_sc = Conv2DFixedPadding(filters, 1, stride, 1)
def call(self, inputs, is_training):
x = self.brelu1(inputs, is_training)
shortcut = x
if self.projection_shortcut:
shortcut = self.conv_sc(x)
conv_k1_s1_r1 = shortcut
conv_k3_s1_r1 = self.conv1(shortcut)
conv_k3_s1_r3 = self.conv2(shortcut)
conv_k3_s1_r5 = self.conv3(shortcut)
x = conv_k1_s1_r1 + conv_k3_s1_r1 + conv_k3_s1_r3 + conv_k3_s1_r5
x = self.brelu2(x, is_training)
x = self.conv4(x)
outputs = shortcut + x
return outputs
I have created a custom Keras Conv2D layer as follows:
class CustConv2D(Conv2D):
def __init__(self, filters, kernel_size, kernelB=None, activation=None, **kwargs):
self.rank = 2
self.num_filters = filters
self.kernel_size = conv_utils.normalize_tuple(kernel_size, self.rank, 'kernel_size')
self.kernelB = kernelB
self.activation = activations.get(activation)
super(CustConv2D, self).__init__(self.num_filters, self.kernel_size, **kwargs)
def build(self, input_shape):
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
if input_shape[channel_axis] is None:
raise ValueError('The channel dimension of the inputs '
'should be defined. Found `None`.')
input_dim = input_shape[channel_axis]
num_basis = K.int_shape(self.kernelB)[-1]
kernel_shape = (num_basis, input_dim, self.num_filters)
self.kernelA = self.add_weight(shape=kernel_shape,
initializer=RandomUniform(minval=-1.0,
maxval=1.0, seed=None),
name='kernelA',
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
self.kernel = K.sum(self.kernelA[None, None, :, :, :] * self.kernelB[:, :, :, None, None], axis=2)
# Set input spec.
self.input_spec = InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim})
self.built = True
super(CustConv2D, self).build(input_shape)
I use the CustomConv2D as the first Conv layer of my model.
img = Input(shape=(width, height, 1))
l1 = CustConv2D(filters=64, kernel_size=(11, 11), kernelB=basis_L1, activation='relu')(img)
The model compiles fine; but gives me the following error while training.
ValueError: An operation has None for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.
Is there a way to figure out which operation is throwing the error? Also, is there any implementation error in the way I am writing the custom layer?
You're destroying your build by calling the original Conv2D build (your self.kernel will be replaced, then self.kernelA will never be used, thus backpropagation will never reach it).
It's also expecting biases and all the regular stuff:
class CustConv2D(Conv2D):
def __init__(self, filters, kernel_size, kernelB=None, activation=None, **kwargs):
#...
#...
#don't use bias if you're not defining it:
super(CustConv2D, self).__init__(self.num_filters, self.kernel_size,
activation=activation,
use_bias=False, **kwargs)
#bonus: don't forget to add the activation to the call above
#it will also replace all your `self.anything` defined before this call
def build(self, input_shape):
#...
#...
#don't use bias:
self.bias = None
#consider the layer built
self.built = True
#do not destroy your build
#comment: super(CustConv2D, self).build(input_shape)
It may be because there are some weights in your code that are defined by not used in the calculation of the output. Thus its gradient wrt the loss is None/undefined.
A coded out example can be found here: https://github.com/keras-team/keras/issues/12521#issuecomment-496743146