Main objective
PyTorch equivalent for SeparableConv2D with padding = 'same':
from tensorflow.keras.layers import SeparableConv2D
x = SeparableConv2D(64, (1, 16), use_bias = False, padding = 'same')(x)
What is the PyTorch equivalent for SeparableConv2D?
This source says:
If groups = nInputPlane, kernel=(K, 1), (and before is a Conv2d layer with groups=1 and kernel=(1, K)), then it is separable.
While this source says:
Its core idea is to break down a complete convolutional acid into a two-step calculation, Depthwise Convolution and Pointwise.
This is my attempt:
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, depth, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, out_channels*depth, kernel_size=kernel_size, groups=in_channels, bias=bias)
self.pointwise = nn.Conv2d(out_channels*depth, out_channels, kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
Is this correct? Is this equivalent to tensorflow.keras.layers.SeparableConv2D?
What about padding = 'same'?
How to ensure that my input and output size is the same while doing this?
My attempt:
x = F.pad(x, (8, 7, 0, 0), )
Because the kernel size is (1,16), I added left and right padding, 8 and 7 respectively. Is this the right way (and best way) to achieve padding = 'same'? How can I place this inside my SeparableConv2d class, and calculate on the fly given the input data dimension size?
All together
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, depth, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, out_channels*depth, kernel_size=kernel_size, groups=in_channels, bias=bias)
self.pointwise = nn.Conv2d(out_channels*depth, out_channels, kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.separable_conv = SeparableConv2d(
in_channels=32,
out_channels=64,
depth=1,
kernel_size=(1,16)
)
def forward(self, x):
x = F.pad(x, (8, 7, 0, 0), )
x = self.separable_conv(x)
return x
Any problem with these codes?
The linked definitions are generally agreeing. The best one is in the article.
"Depthwise" (not a very intuitive name since depth is not involved) - is a series of regular 2d convolutions, just applied to layers of the data separately. - "Pointwise" is same as Conv2d with 1x1 kernel.
I suggest a few correction to your SeparableConv2d class:
no need to use depth parameter - it is same as out_channels
I set padding to 1 to ensure same output size with kernel=(3,3). if kernel size is different - adjust padding accordingly, using same principles as with regular Conv2d. Your example class Net() is no longer needed - padding is done in SeparableConv2d.
This is the updated code, should be similar to tf.keras.layers.SeparableConv2D implementation:
class SeparableConv2d(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, bias=False):
super(SeparableConv2d, self).__init__()
self.depthwise = nn.Conv2d(in_channels, in_channels, kernel_size=kernel_size,
groups=in_channels, bias=bias, padding=1)
self.pointwise = nn.Conv2d(in_channels, out_channels,
kernel_size=1, bias=bias)
def forward(self, x):
out = self.depthwise(x)
out = self.pointwise(out)
return out
Related
Recently, I want to implement CDCN in CVPR2020 using tensorflow2.8 + python3.9. This requires my custom layer acquire current conv2d layer's weight.
However, when I try to add my custom layer to the sequential model, error occurred:NotImplementedError: numpy() is only available when eager execution is enabled.
This is my code. Can anyone helps me? I have tried to add tf.compat.v1.enable_eager_execution(), but it doesn't work.
import numpy as np
import tensorflow.keras as tfk
import tensorflow as tf
class CDC(tfk.layers.Layer):
def __init__(self, output_dim, kernel_size=(3, 3), padding='same', activation=None, theta=0.7, **kwargs):
super(CDC, self).__init__()
self.theta = theta
self.activation = None
self.output_dim = output_dim
self.kernel_size = kernel_size
self.padding = padding
if activation is not None:
self.activation = tfk.activations.get(activation)
def build(self, input_shape):
self.conv = tfk.layers.Conv2D(self.output_dim, self.kernel_size, padding=self.padding, input_shape=input_shape)
self.conv.build(input_shape=input_shape)
self._kernel = self.conv.kernel
super(CDC, self).build(input_shape)
self.built = True
def call(self, inputs, training=None, mask=None):
vanillaOutput = self.conv(inputs)
weightSum = self.conv.kernel.numpy().sum(axis=0).sum(axis=0).sum(axis=0)
weightSum = np.reshape(weightSum, (1, 1, 1, self.output_dim))
weightSum = tf.constant(weightSum, dtype=tf.float32)
cDiff = tf.nn.conv2d(inputs, filters=weightSum, strides=self.conv.strides, padding=self.conv.padding.upper())
result = vanillaOutput - self.theta * cDiff
if self.activation is not None:
return self.activation(result)
return vanillaOutput
If you just want the sum of all elements in kernel
tf.math.reduce_sum()
Also replace the lines
weightSum = self.conv.kernel.numpy().sum(axis=0).sum(axis=0).sum(axis=0)
weightSum = np.reshape(weightSum, (1, 1, 1, self.output_dim))
weightSum = tf.constant(weightSum, dtype=tf.float32)
...
weightSum = tf.math.reduce_sum(tf.math.reduce_sum(tf.math.reduce_sum(self.conv.kernel,axis=0),axis=0),axis=0)
weightSum = tf.reshape(weightSum, (1, 1, 1, self.output_dim))
I have a simple autoencoder architecture in PyTorch, which I train to do feature compression and reconstruction. My goal is to use the latent space of the autoencoder to reduce the initial dimensionality of my data and compress it in the test phase.
To perform this, I would need to pass my test data only to my encoder, not the whole autoencoder. Would you have any idea of how to do this ? Something like model = Autoencoder.encoder() or else?
My complete architecture is below:
class Autoencoder(nn.Module):
def __init__(self, n_features):
super(Autoencoder, self).__init__()
self.n_sensors = n_sensors
self.encoder = nn.Sequential(
nn.Linear(self.n_features, 1),
nn.ReLU(True))
self.decoder = nn.Sequential(
nn.Linear(1, self.n_features),
nn.ReLU(True))
def forward(self, x):
x = self.encoder(x)
x = self.decoder(x)
return x
Base on your model definition, you can call forward on the encoder submodule directly:
class Autoencoder(nn.Module):
def __init__(self, n_features):
super(Autoencoder, self).__init__()
self.n_features = n_features
self.encoder = nn.Sequential(
nn.Linear(self.n_features, 1),
nn.ReLU(True))
self.decoder = nn.Sequential(
nn.Linear(1, self.n_features),
nn.ReLU(True))
def forward(self, x):
x = self.encoder(x)
x = self.decoder(x)
return x
Keep in mind you first need to initialize the model:
>>> ae = Autoencoder(n_features=10)
>>> x = torch.empty(16, 10)
>>> ae.encoder(x).shape
(16, 1)
I’m dealing with a strange issue where the gradients after backward pass have different shapes depending on whether CUDA or CPU is used. The model used is relatively simple:
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 6, 5)
self.pool1 = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(6, 16, 5)
self.pool2 = nn.MaxPool2d(2, 2)
self.fc1 = nn.Linear(16 * 5 * 5, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 10)
self.relu1 = nn.ReLU()
self.relu2 = nn.ReLU()
self.relu3 = nn.ReLU()
self.relu4 = nn.ReLU()
def forward(self, x):
x = self.pool1(self.relu1(self.conv1(x)))
x = self.pool2(self.relu2(self.conv2(x)))
x = x.view(-1, 16 * 5 * 5)
x = self.relu3(self.fc1(x))
x = self.relu4(self.fc2(x))
x = self.fc3(x)
return x
The input tensor has shape (1, 3, 32, 32), and the relevant section of code is as follows, with the method generate_gradients being of particular importance:
class VanillaBackprop():
"""
Produces gradients generated with vanilla back propagation from the image
"""
def __init__(self, model):
self.model = model
self.gradients = None
# Put model in evaluation mode
self.model.eval()
# Hook the first layer to get the gradient
self.hook_layers()
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model.to(self.device)
def hook_layers(self):
def hook_function(module, grad_in, grad_out):
self.gradients = grad_in[0]
# Register hook to the first layer
try:
first_layer = list(self.model.features._modules.items())[0][1]
except:
first_layer = list(self.model._modules.items())[0][1]
first_layer.register_backward_hook(hook_function)
def generate_gradients(self, input_image, target_class):
# Forward
model_output = self.model(input_image.to(self.device))
# Zero grads
self.model.zero_grad()
# Target for backprop
one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_()
one_hot_output[0][target_class] = 1
# Backward pass
model_output.backward(gradient=one_hot_output.to(self.device))
# Convert Pytorch variable to numpy array
gradients_as_arr = self.gradients.data.cpu().numpy()[0]
return gradients_as_arr
When on CPU, self.gradients has shape (1, 3, 32, 32), while on CUDA it has shape (1, 6, 28, 28). How is that possible, and how do I fix this? Any help is much appreciated.
It looks like the issue stems from the register_backward_hook() function. As pointed out in the PyTorch forums:
You might want to double check the register_backward_hook() doc. But
it is known to be kind of broken at the moment and can have this
behavior.
I would recommend you use autograd.grad() for this though. That will
make it simpler than backward+access to the .grad field.
I, however, opted to use register_hook() instead of register_backward_hook() (as opposed to autograd.grad() as suggested), which seems to work as well:
class VanillaBackprop():
"""
Produces gradients generated with vanilla back propagation from the image
"""
def __init__(self, model):
self.model = model
self.gradients = None
# Put model in evaluation mode
self.model.eval()
# Hook the first layer to get the gradient
def hook_input(self, input_tensor):
def hook_function(grad_in):
self.gradients = grad_in
input_tensor.register_hook(hook_function)
def generate_gradients(self, input_image, target_class):
# Register input hook
self.hook_input(input_image)
# Forward
model_output = self.model(input_image)
# Zero grads
self.model.zero_grad()
# Target for backprop
device = next(self.model.parameters()).device
one_hot_output = torch.FloatTensor(1, model_output.size()[-1]).zero_()
one_hot_output[0][target_class] = 1
one_hot_output = one_hot_output.to(device)
# Backward pass
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model_output.backward(gradient=one_hot_output.to(device))
# Convert Pytorch variable to numpy array
# [0] to get rid of the first channel (1,3,224,224)
gradients_as_arr = self.gradients.data.cpu().numpy()[0]
return gradients_as_arr
class DeConv2d(nn.Module):
def __init__(self, in_channel, out_channel, kernel_size, stride, padding, dilation):
super().__init__()
self.up = nn.Upsample(scale_factor=2, mode='nearest')
self.conv = nn.Conv2d(in_channel, out_channel, kernel_size=kernel_size, \
stride=stride, padding=padding, dilation=dilation)
def forward(self, x):
output = self.up(x)
output = self.conv(output)
return output
class EncoderDecoder(nn.Module):
def __init__(self, pretrained_net, n_class):
super().__init__()
self.n_class = n_class
self.pretrained_net = pretrained_net
self.relu = nn.ReLU(inplace=True)
self.deconv1 = DeConv2d(512, 512, kernel_size=3, stride=1, padding=1, dilation=1)
self.bn1 = nn.BatchNorm2d(512)
self.deconv2 = DeConv2d(512, 256, kernel_size=3, stride=1, padding=1, dilation=1)
self.bn2 = nn.BatchNorm2d(256)
self.deconv3 = DeConv2d(256, 128, kernel_size=3, stride=1, padding=1, dilation=1)
self.bn3 = nn.BatchNorm2d(128)
self.deconv4 = DeConv2d(128, 64, kernel_size=3, stride=1, padding=1, dilation=1)
self.bn4 = nn.BatchNorm2d(64)
self.classifier = nn.Conv2d(64, n_class, kernel_size=1)
def forward(self, x):
output=self.pretrained_net.layers(x)
output=self.relu(self.deconv1(output))
output=self.bn1(output)
output=self.relu(self.deconv2(output))
output=self.bn2(output)
output=self.relu(self.deconv3(output))
output=self.bn3(output)
output=self.relu(self.deconv4(output))
output=self.bn4(output)
output=self.classifier(output)
return output
this is my code and I don't know why the type error exist . Does somebody know how to fix these problems?
When you create a class, and define a function with self args within the class, self is autofilled with the class.Ex:
class item():# create a random class
self.var = 0
def fun(self,x):
self.var +=x
n = item()
And you can try add:
n.fun(3)
print(n.var)
returns 3
self argument is autofilled with class itself
In my case, I used transformers.Trainer.train() to train my model. And then I used predict_scores = model(predict_dataset) to get the prediction result. But I got the same error as "forward() takes 1 positional argument but 2 were given...".
Then I changed to predict_scores = trainer.predict(predict_dataset), then I got the correct result. Hope this is helpful to others. I am a newbie to PyTorch.
I have a model written in TF 1.x code using tf-slim API as well. Is it possible to convert that to tf.keras in TF 2.0 EXACTLY the way it is? For instance, have exactly the same amount of parameters and training?
In my case, I've tried doing so, but my model in tf.keras actually has about 5% LESS parameters than the one in TF 1.x. I also noticed my model in tf.keras has a much less smooth training stage too. Any thoughts? Thanks
Maybe I'm setting some of the parameters to initialize the layers differently? Any other suggestions would be greatly appreciated
This isn't my full model, but I use a lot of the components below:
Original TF.1x model:
import tensorflow as tf
from tensorflow.contrib import slim
def batch_norm_relu(inputs, is_training):
net = slim.batch_norm(inputs, is_training=is_training)
net = tf.nn.relu(net)
return net
def conv2d_transpose(inputs, output_channels, kernel_size):
upsamp = tf.contrib.slim.conv2d_transpose(
inputs,
num_outputs=output_channels,
kernel_size=kernel_size,
stride=2,
)
return upsamp
def conv2d_fixed_padding(inputs, filters, kernel_size, stride, rate):
net = slim.conv2d(inputs,
filters,
kernel_size,
stride=stride,
rate = rate,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
return net
def block(inputs, filters, is_training, projection_shortcut, stride):
inputs = batch_norm_relu(inputs, is_training)
shortcut = inputs
if projection_shortcut is not None:
shortcut = projection_shortcut(inputs)
conv_k1_s1_r1 = shortcut
conv_k3_s1_r1 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 1,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
conv_k3_s1_r3 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 3,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
conv_k3_s1_r5 = slim.conv2d(shortcut,
filters,
kernel_size = 3,
stride = 1,
rate = 5,
padding=('SAME' if stride == 1 else 'VALID'),
activation_fn=None
)
net = conv_k1_s1_r1 + conv_k3_s1_r1 + conv_k3_s1_r3 + conv_k3_s1_r5
net = batch_norm_relu(net, is_training)
net = conv2d_fixed_padding(inputs=net, filters=filters, kernel_size=1, stride=1, rate = 1)
outputs = shortcut + net
return outputs
Attempted TF 2.x.keras model same component:
import tensorflow as tf
class BatchNormRelu(tf.keras.layers.Layer):
"""Batch normalization + ReLu"""
def __init__(self, name=None):
super(BatchNormRelu, self).__init__(name=name)
self.bnorm = tf.keras.layers.BatchNormalization(momentum=0.999,
scale=False)
self.relu = tf.keras.layers.ReLU()
def call(self, inputs, is_training):
x = self.bnorm(inputs, training=is_training)
x = self.relu(x)
return x
class Conv2DTranspose(tf.keras.layers.Layer):
"""Conv2DTranspose layer"""
def __init__(self, output_channels, kernel_size, name=None):
super(Conv2DTranspose, self).__init__(name=name)
self.tconv1 = tf.keras.layers.Conv2DTranspose(
filters=output_channels,
kernel_size=kernel_size,
strides=2,
padding='same',
activation=tf.keras.activations.relu
)
def call(self, inputs):
x = self.tconv1(inputs)
return x
class Conv2DFixedPadding(tf.keras.layers.Layer):
"""Conv2D Fixed Padding layer"""
def __init__(self, filters, kernel_size, stride, rate, name=None):
super(Conv2DFixedPadding, self).__init__(name=name)
self.conv1 = tf.keras.layers.Conv2D(filters,
kernel_size,
strides=stride,
dilation_rate=rate,
padding=('same' if stride==1 else 'valid'),
activation=None
)
def call(self, inputs):
x = self.conv1(inputs)
return x
class block(tf.keras.layers.Layer):
def __init__(self,
filters,
stride,
projection_shortcut=True,
name=None):
super(block, self).__init__(name=name)
self.projection_shortcut = projection_shortcut
self.brelu1 = BatchNormRelu()
self.brelu2 = BatchNormRelu()
self.conv1 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=1,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv2 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=3,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv3 = tf.keras.layers.Conv2D(filters,
kernel_size=3,
strides=1,
dilation_rate=5,
padding=('same' if stride==1 else 'valid'),
activation=None
)
self.conv4 = Conv2DFixedPadding(filters, 1, 1, 1)
self.conv_sc = Conv2DFixedPadding(filters, 1, stride, 1)
def call(self, inputs, is_training):
x = self.brelu1(inputs, is_training)
shortcut = x
if self.projection_shortcut:
shortcut = self.conv_sc(x)
conv_k1_s1_r1 = shortcut
conv_k3_s1_r1 = self.conv1(shortcut)
conv_k3_s1_r3 = self.conv2(shortcut)
conv_k3_s1_r5 = self.conv3(shortcut)
x = conv_k1_s1_r1 + conv_k3_s1_r1 + conv_k3_s1_r3 + conv_k3_s1_r5
x = self.brelu2(x, is_training)
x = self.conv4(x)
outputs = shortcut + x
return outputs