I am putting the (NMT Tensorflow code) inside a main class. The code base has two classes - 'Encoder' and 'Decoder'. They are referenced in their respective 'init' methods. However it raises an error - 'Undefined named Encoder'.
class TranslationModel(ModelBase):
pathToZip = tf.keras.utils.get_file('spa-eng.zip', origin='http://download.tensorflow.org/data/spa-eng.zip', extract=True)
pathToFile = os.path.dirname(pathToZip)+"/spa-eng/spa.txt"
def __init__(self,
batchSize = 64,
bufferSize = None,
numberOfBatches = None,
units = 1024,
vocabInputSize = None,
vocabTargetSize = None,
optimizer = tf.train.AdamOptimizer(),
dataSetPath = None,
inputTensor = None,
targetTensor = None,
inputLanguage = None,
targetLanguage = None,
maxLengthInput = None,
maxLengthTarget = None,
embeddingDimension = 256, *arg, **kwargs):
self.batchSize = 64
self.bufferSize = None
self.numberOfBatches = None
self.units = units
self.vocabInputSize = None
self.vocabTargetSize = None
self.optimizer = optimizer
self.dataSetPath = dataSetPath
self.targetTensor = targetTensor
self.inputTensor = inputTensor
self.inputLanguage = inputLanguage
self.targetLanguage = targetLanguage
self.maxLengthInput = maxLengthInput
self.maxLengthTarget = maxLengthTarget
self.embeddingDimension = embeddingDimension
super().__init__(*arg, **kwargs)
#OTHER FUNCTIONS HERE
class Encoder(tf.keras.Model):
def __init__(self, vocabSize, embeddingDimension, encoderUnits, batchSize):
super(Encoder, self).__init__() # Raises error - 'Undefined named Encoder'
#Other code here
class Decoder(tf.keras.Model):
def __init__(self, vocabSize, embeddingDimension, dec_units, batchSize):
super('Decoder', self).__init__() # Raises error - 'Undefined named Decoder'
## Other code
It's because when you have a class inside of another and you want to identify it, you should do it this way : OutterClass.InnerClass
it won't work if you just use : InnerClass
for your case it's TranslationModel.Encoder
Related
I can create a FasterRCNN object using
model = fasterrcnn_resnet50_fpn(...)
which I want to inherit from, as
class MyDetector(FasterRCNN):
...
but overwrite the superclass instance from the fasterrcnn_resnet50_fpn() factory. I have tried using __new__, as:
class MyDetector(FasterRCNN):
def __new__(cls):
return fasterrcnn_resnet50_fpn(weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT)
def __init__(self):
num_features_in = self.roi_heads.box_predictor.cls_score.in_features
self.roi_heads.box_predictor = FastRCNNPredictor(num_features_in, num_classes=2)
def some_func(self):
pass
so that I can add custom methods to the child class and so forth. What is the correct way of doing this?
I guess you'd be better to make your own factory function.
import libraries
from typing import Optional, Any
import torch
from torch import nn
import torchvision
from torchvision.models.resnet import resnet50, ResNet50_Weights
from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights, FasterRCNN
from torchvision.models._utils import _ovewrite_value_param
from torchvision.models.detection.backbone_utils import (
_validate_trainable_layers,
_resnet_fpn_extractor,
)
from torchvision.models.detection._utils import overwrite_eps
from torchvision.ops import misc as misc_nn_ops
class MyDetector
class MyDetector(FasterRCNN):
def __init__(self, backbone, num_classes=None, **kwarg):
super().__init__(backbone=backbone, num_classes=num_classes, **kwarg)
def some_func(self):
pass
MyDetector factory function
# https://github.com/pytorch/vision/blob/main/torchvision/models/detection/faster_rcnn.py#L459
def mydetector_resnet50_fpn(
*,
weights: Optional[FasterRCNN_ResNet50_FPN_Weights] = None,
progress: bool = True,
num_classes: Optional[int] = None,
weights_backbone: Optional[ResNet50_Weights] = ResNet50_Weights.IMAGENET1K_V1,
trainable_backbone_layers: Optional[int] = None,
**kwargs: Any,
) -> MyDetector:
weights = FasterRCNN_ResNet50_FPN_Weights.verify(weights)
weights_backbone = ResNet50_Weights.verify(weights_backbone)
if weights is not None:
weights_backbone = None
num_classes = _ovewrite_value_param(
"num_classes", num_classes, len(weights.meta["categories"])
)
elif num_classes is None:
num_classes = 91
is_trained = weights is not None or weights_backbone is not None
trainable_backbone_layers = _validate_trainable_layers(
is_trained, trainable_backbone_layers, 5, 3
)
norm_layer = misc_nn_ops.FrozenBatchNorm2d if is_trained else nn.BatchNorm2d
backbone = resnet50(
weights=weights_backbone, progress=progress, norm_layer=norm_layer
)
backbone = _resnet_fpn_extractor(backbone, trainable_backbone_layers)
model = MyDetector(backbone, num_classes=num_classes, **kwargs)
if weights is not None:
model.load_state_dict(weights.get_state_dict(progress=progress))
if weights == FasterRCNN_ResNet50_FPN_Weights.COCO_V1:
overwrite_eps(model, 0.0)
return model
utility for checking
# https://discuss.pytorch.org/t/check-if-models-have-same-weights/4351/6
def compare_models(model_1, model_2):
models_differ = 0
for key_item_1, key_item_2 in zip(
model_1.state_dict().items(), model_2.state_dict().items()
):
if torch.equal(key_item_1[1], key_item_2[1]):
pass
else:
models_differ += 1
if key_item_1[0] == key_item_2[0]:
print("Mismtach found at", key_item_1[0])
else:
raise Exception
if models_differ == 0:
print("Models match perfectly! :)")
test
model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT
)
my_model = mydetector_resnet50_fpn(weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT)
compare_models(model, my_model)
output
Models match perfectly! :)
And I also tried to make hard coded version. But as you know, customizing settings of FPN is somewhat complicated.
from torchvision.models.resnet import resnet50, ResNet50_Weights
from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights, FasterRCNN
from torchvision.models.detection.backbone_utils import _resnet_fpn_extractor
from torchvision.ops import misc as misc_nn_ops
class MyDetector(FasterRCNN):
def __init__(self, **kwarg):
weights = FasterRCNN_ResNet50_FPN_Weights.DEFAULT
backbone = resnet50(
weights=ResNet50_Weights.IMAGENET1K_V1,
norm_layer=misc_nn_ops.FrozenBatchNorm2d,
)
backbone = _resnet_fpn_extractor(backbone, trainable_layers=3)
# default of num_classes is 91
# this num_classes is used for setting FastRCNNPreditcor
# https://github.com/pytorch/vision/blob/main/torchvision/models/detection/faster_rcnn.py#L257
num_classes = len(weights.meta["categories"])
super().__init__(backbone=backbone, num_classes=num_classes, **kwarg)
self.load_state_dict(weights.get_state_dict(progress=True))
def some_func(self):
pass
m = MyDetector()
Here is the error about deepcopy, how should I do it.
error:
target_encoder = copy.deepcopy(self.online_encoder)
RuntimeError: Only Tensors created explicitly by the user (graph leaves) support the deepcopy protocol at the moment
class Model(nn.Module):
def __init__(
self,
model, # byol
projection_size=256,
pred_size = 256,
projection_hidden_size=4096,
moving_average_decay=0.99,
use_momentum=True,
):
super(SSL, self).__init__()
self.online_encoder = Pre_model(model) # 256
self.use_momentum = use_momentum
self.target_encoder = None
self.target_ema_updater = EMA(moving_average_decay)
def _get_target_encoder(self):
target_encoder = copy.deepcopy(self.online_encoder)
set_requires_grad(target_encoder, False)
return target_encoder
def forward(self, x):
anchors = x['anchor'].cuda(non_blocking = True)
neighbors = x['neighbor'].cuda(non_blocking = True)
online_anchor_proj = self.online_encoder(anchors)
online_neighbor_proj = self.online_encoder(neighbors)
with torch.no_grad():
target_online = self._get_target_encoder() if self.use_momentum else self.online_encoder
target_anchor_proj= target_online(anchors)
target_neighbor_proj = target_online(neighbors)
I have the following class:
class DataGenerator(keras.utils.Sequence):
__slots__ = 'path','batch_size','dim','mode','split_proportion','indices','processes'
def __init__(self, path: str,
batch_size: int,
dim: Tuple[int] = (12,86,98),
mode: str = 'train',
split_proportion: float = None,
indices: List[List[int]] = None,
processes: Optional[int] = None):
self._path = path
self._mode = mode
self._split_proportion = split_proportion
self._k_indices = indices
self._dim = dim
self._batch_size = batch_size
self._processes = processes
# Change mode to retrieve from folders
if mode == 'validation':
mode = 'train'
self._im_path = os.path.join(self._path, mode,'image')
self._msk_path = os.path.join(self._path, mode,'mask')
If I instantiate it, it should not have the attribute dict since it contains slots. However:
training_data = DataGenerator(path = '/path', batch_size = 1,
mode = 'train', split_proportion = 0.1)
training_data.__dict__
{'_path': '/path',
'_mode': 'train',
'_split_proportion': 0.1,
'_k_indices': None,
'_dim': (12, 86, 98),
'_batch_size': 1,
'_processes': None,
'_im_path': '/path/train/image',
'_msk_path': '/path/train/mask'}
Additionally, if I check memory requirements, they seem to be higher than without the slots.
# with __slots__
sys.getsizeof(training_data)
112
sys.getsizeof(training_data.__dict__)
152
# without __slots__
sys.getsizeof(training_data)
56
sys.getsizeof(training_data.__dict__)
152
I'm trying to quantize a model which uses PReLU. Replacing PReLU with ReLU is not possible as it drastically affects the network performance to the point its useless.
As far as I know, PReLU is not supported in Pytorch when it comes to quantization. So I tried to rewrite this module manually and implement the multiplication and additions using torch.FloatFunctional() to get around this limitation.
This is what I have come up so far:
class PReLU_Quantized(nn.Module):
def __init__(self, prelu_object):
super().__init__()
self.weight = prelu_object.weight
self.quantized_op = nn.quantized.FloatFunctional()
self.quant = torch.quantization.QuantStub()
self.dequant = torch.quantization.DeQuantStub()
def forward(self, inputs):
# inputs = torch.max(0, inputs) + self.weight * torch.min(0, inputs)
self.weight = self.quant(self.weight)
weight_min_res = self.quantized_op.mul(self.weight, torch.min(inputs)[0])
inputs = self.quantized_op.add(torch.max(inputs)[0], weight_min_res).unsqueeze(0)
self.weight = self.dequant(self.weight)
return inputs
and for the replacement :
class model(nn.Module):
def __init__(self)
super().__init__()
....
self.prelu = PReLU()
self.prelu_q = PReLU_Quantized(self.prelu)
....
Basically, I read the learned parameter of the existing prelu module, and run the calculation myself in a new module. The module seems to be working in the sense, its not failing the whole application.
However, in order to assess whether my implementation is actually correct and yields the same result as the original module, I tried to test it.
Here is a counterpart for normal models (i.e. not quantized model):
For some reason, the error between the actual PReLU and my implementation is very large!
Here are sample diffs in different layers:
diff : 1.1562038660049438
diff : 0.02868632599711418
diff : 0.3653906583786011
diff : 1.6100226640701294
diff : 0.8999372720718384
diff : 0.03773299604654312
diff : -0.5090572834014893
diff : 0.1654307246208191
diff : 1.161868691444397
diff : 0.026089997962117195
diff : 0.4205571115016937
diff : 1.5337920188903809
diff : 0.8799554705619812
diff : 0.03827812895178795
diff : -0.40296515822410583
diff : 0.15618863701820374
and the diff is calculated like this in the forward pass:
def forward(self, x):
residual = x
out = self.bn0(x)
out = self.conv1(out)
out = self.bn1(out)
out = self.prelu(out)
out2 = self.prelu2(out)
print(f'diff : {( out - out2).mean().item()}')
out = self.conv2(out)
...
This is the normal implementation which I used on ordinary model (i.e. not quantized!) to asses whether it produces correct result and then move on to quantized version:
class PReLU_2(nn.Module):
def __init__(self, prelu_object):
super().__init__()
self.prelu_weight = prelu_object.weight
self.weight = self.prelu_weight
def forward(self, inputs):
x = self.weight
tmin, _ = torch.min(inputs,dim=0)
tmax, _ = torch.max(inputs,dim=0)
weight_min_res = torch.mul(x, tmin)
inputs = torch.add(tmax, weight_min_res)
inputs = inputs.unsqueeze(0)
return inputs
What am I missing here?
I figured it out! I made a huge mistake in the very begining. I needed to calculate
PReLU(x)=max(0,x)+a∗min(0,x)
or
and not the actual torch.min! or torch.max! which doesn't make any sense!
Here is the final solution for normal models (i.e not quantized)!:
class PReLU_2(nn.Module):
def __init__(self, prelu_object):
super().__init__()
self.prelu_weight = prelu_object.weight
self.weight = self.prelu_weight
def forward(self, inputs):
pos = torch.relu(inputs)
neg = -self.weight * torch.relu(-inputs)
inputs = pos + neg
return inputs
and this is the quantized version :
class PReLU_Quantized(nn.Module):
def __init__(self, prelu_object):
super().__init__()
self.prelu_weight = prelu_object.weight
self.weight = self.prelu_weight
self.quantized_op = nn.quantized.FloatFunctional()
self.quant = torch.quantization.QuantStub()
self.dequant = torch.quantization.DeQuantStub()
def forward(self, inputs):
# inputs = max(0, inputs) + alpha * min(0, inputs)
self.weight = self.quant(self.weight)
weight_min_res = self.quantized_op.mul(-self.weight, torch.relu(-inputs))
inputs = self.quantized_op.add(torch.relu(inputs), weight_min_res)
inputs = self.dequant(inputs)
self.weight = self.dequant(self.weight)
return inputs
Side note:
I also had a typo where I was calculating the diff :
out = self.prelu(out)
out2 = self.prelu2(out)
print(f'diff : {( out - out2).mean().item()}')
out = self.conv2(out)
needs to be
out1 = self.prelu(out)
out2 = self.prelu2(out)
print(f'diff : {( out1 - out2).mean().item()}')
out = self.conv2(out1)
Update:
In case you face issues in quantization, you may try this version :
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.quantized as nnq
from torch.quantization import fuse_modules
class QPReLU(nn.Module):
def __init__(self, num_parameters=1, init: float = 0.25):
super(QPReLU, self).__init__()
self.num_parameters = num_parameters
self.weight = nn.Parameter(torch.Tensor(num_parameters).fill_(init))
self.relu1 = nn.ReLU()
self.relu2 = nn.ReLU()
self.f_mul_neg_one1 = nnq.FloatFunctional()
self.f_mul_neg_one2 = nnq.FloatFunctional()
self.f_mul_alpha = nnq.FloatFunctional()
self.f_add = nnq.FloatFunctional()
self.quant = torch.quantization.QuantStub()
self.dequant = torch.quantization.DeQuantStub()
self.quant2 = torch.quantization.QuantStub()
self.quant3 = torch.quantization.QuantStub()
# self.dequant2 = torch.quantization.QuantStub()
self.neg_one = torch.Tensor([-1.0])
def forward(self, x):
x = self.quant(x)
# PReLU, with modules only
x1 = self.relu1(x)
neg_one_q = self.quant2(self.neg_one)
weight_q = self.quant3(self.weight)
x2 = self.f_mul_alpha.mul(
weight_q, self.f_mul_neg_one2.mul(
self.relu2(
self.f_mul_neg_one1.mul(x, neg_one_q),
),
neg_one_q)
)
x = self.f_add.add(x1, x2)
x = self.dequant(x)
return x
m1 = nn.PReLU()
m2 = QPReLU()
# check correctness in fp
for i in range(10):
data = torch.randn(2, 2) * 1000
assert torch.allclose(m1(data), m2(data))
# toy model
class M(nn.Module):
def __init__(self):
super(M, self).__init__()
self.prelu = QPReLU()
def forward(self, x):
x = self.prelu(x)
return x
# quantize it
m = M()
m.qconfig = torch.quantization.default_qconfig
torch.quantization.prepare(m, inplace=True)
# calibrate
m(torch.randn(4, 4))
# convert
torch.quantization.convert(m, inplace=True)
# run some data through
res = m(torch.randn(4, 4))
print(res)
and make sure to read the ralted notes here
I'm trying override a str method in Person() class:
'''class Person(object):
def __init__(self, Nose = None, Neck = None, RShoulder = None, RElbow = None, RWrist = None, LShoulder = None, LElbow = None, LWrist = None, MidHip = None, RHip = None, RKnee = None, RAnkle = None, LHip = None, LKnee = None, LAnkle = None, REye = None, LEye = None, REar = None, LEar = None, LBigToe = None, LSmallToe = None, LHeel = None, RBigToe = None, RSmallToe = None, RHeel = None):
self.Nose = Nose
self.Neck = Neck
self.RShoulder = RShoulder
self.RElbow = RElbow
self.RWrist = RWrist
self.LShoulder = LShoulder
self.LElbow = LElbow
self.LWrist = LWrist
self.MidHip = MidHip
self.RHip = RHip
self.RKnee = RKnee
self.RAnkle = RAnkle
self.LHip = LHip
self.LKnee = LKnee
self.LAnkle = LAnkle
self.REye = REye
self.LEye = LEye
self.REar = REar
self.LEar = LEar
self.LBigToe = LBigToe
self.LSmallToe = LSmallToe
self.LHeel = LHeel
self.RBigToe = RBigToe
self.RSmallToe = RSmallToe
self.RHeel = RHeel
def __str__(self):
return 'Nose = %s\nNeck = \n%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s'%(self.Nose,self.Neck,self.RShoulder,self.RElbow,self.RWrist,self.LShoulder,self.LElbow,self.LWrist,self.MidHip,self.RHip,self.RKnee,self.RAnkle,self.LHip,self.LKnee,self.LAnkle,self.REye,self.LEye,self.REar,self.LEar,self.LBigToe,self.LSmallToe,self.LHeel,self.RBigToe,self.RSmallToe,self.RHeel)'''
And I want to find more elegant way to return a string which will look like that:
Nose = something
Neck = something
...
...
...
Question: elegant way to return a string which will look like ...
You can use the built-in vars function to get the __dict__ of the class variable and format it using .format(... and .join(....
Reference:
vars([object])
Return the __dict__ attribute for a module, class, instance, or any other object with a __dict__ attribute.
.format(value[, format_spec])
Convert a value to a “formatted” representation, as controlled by a standard formatting syntax that is used by most built-in types: Format Specification Mini-Language.
<str>.join(iterable)
Return a string which is the concatenation of the strings in iterable.
class Person:
def __init__(self, **kwargs):
self.Nose = kwargs.get('Nose', None)
self.Neck = kwargs.get('Neck', None)
self.RShoulder = kwargs.get('RShoulder', None)
def __str__(self):
return '\n'.join(('{} = {}'
.format(k, v) for k, v in vars(self).items()))
p = Person(Nose=1, Neck=1)
print(p)
Output:
Nose = 1
Neck = 1
RShoulder = None
Tested with Python: 3.6