I've been coding a VAE to process one-hot encoded strings (as 248x46 arrays, where 248 = the length of a padded string and 46 = the number of possible characters). I've been trying to regenerate input strings using the following model architecture. It works well enough on strings with a variety of characters, but on strings that are just repeating characters (e.g. "ccc...ccc" or "ababab...ababab") it does really poorly, and will just output random characters. This only happens once the repeating string gets to be around 80+ characters.
I'm using BCE loss and have been testing using a single string as input with 200 epochs worth of training. So I would think that the model should be overfitting, but I will consistently get random outputs e.g. "ccc...ccc" will become something like "#I4ccci4#c44Ic45I..."
Has anyone encountered this issue before, where repeating inputs can't seem to be learned properly by a VAE? What are some possible explanations for this?
class Encoder(nn.Module):
def __init__(self, latent_dim):
super(Encoder, self).__init__()
self.latent_dim = latent_dim
self.linear = nn.Linear(in_features=46, out_features=435)
self.linear_mu = nn.Linear(in_features=435, out_features=self.latent_dim)
self.linear_var = nn.Linear(in_features=435, out_features=self.latent_dim)
def forward(self, x):
x = self.linear(x)
mu = self.linear_mu(x)
log_var = self.linear_var(x)
return mu, log_var
class Decoder(nn.Module):
def __init__(self, latent_dim):
super(Decoder, self).__init__()
self.latent_dim = latent_dim
self.linear_1 = nn.Linear(in_features=self.latent_dim, out_features=435)
self.linear_2 = nn.Linear(in_features=435, out_features=46)
def forward(self, z):
x = self.linear_1(z)
x = self.linear_2(x)
out = F.softmax(x.view(-1, 248, 46), dim=1)
return out
class VAE(nn.Module):
def __init__(self, latent_dim=100):
super(VAE, self).__init__()
self.encoder = Encoder(latent_dim)
self.decoder = Decoder(latent_dim)
def sample(self, mu, log_var):
std = torch.exp(0.5 * log_var)
eps = torch.randn_like(std)
return eps * std + mu
def forward(self, x):
mu, log_var = self.encoder(x)
z = self.sample(mu, log_var)
out = self.decoder(z)
return out, mu, log_var
Related
I am trying to train a genetic algorithm but for some reason it does not work when it's stored inside of a class. I have two equivalent pieces of code but the one stored inside of a class fails. It returns this..
raise ValueError("The fitness function must accept 2 parameters:
1) A solution to calculate its fitness value.
2) The solution's index within the population.
The passed fitness function named '{funcname}' accepts {argcount} parameter(s).".format(funcname=fitness_func.__code__.co_name, argcount=fitness_func.__code__.co_argcount))
ValueError: The fitness function must accept 2 parameters:
1) A solution to calculate its fitness value.
2) The solution's index within the population.
The passed fitness function named 'fitness_func' accepts 3 parameter(s).
Here is the simplified version of the one that doesnt work.
import torch
import torch.nn as nn
import pygad.torchga
import pygad
class NN(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super().__init__()
self.linear1 = nn.Linear(input_size, hidden_size)
self.linear2 = nn.Linear(hidden_size, hidden_size)
self.linear3 = nn.Linear(hidden_size, hidden_size)
self.linear4 = nn.Linear(hidden_size, output_size)
def forward(self, x):
x = self.linear1(x)
x = self.linear2(x)
x = self.linear3(x)
x = self.linear4(x)
return x
class Coin:
def __init__(self):
self.NeuralNet = NN(1440, 1440, 3)
def fitness_func(self, solution, solution_idx):
return 0
def trainModel(self):
torch_ga = pygad.torchga.TorchGA(model=self.NeuralNet, num_solutions=10)
ga_instance = pygad.GA(num_generations=10,
num_parents_mating=2,
initial_population=torch_ga.population_weights,
fitness_func=self.fitness_func)
ga_instance.run()
if __name__ == "__main__":
coin = Coin()
coin.trainModel()
Here is the simplified version of the one that does work.
import torch
import torch.nn as nn
import pygad.torchga
import pygad
class NN(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super().__init__()
self.linear1 = nn.Linear(input_size, hidden_size)
self.linear2 = nn.Linear(hidden_size, hidden_size)
self.linear3 = nn.Linear(hidden_size, hidden_size)
self.linear4 = nn.Linear(hidden_size, output_size)
def forward(self, x):
x = self.linear1(x)
x = self.linear2(x)
x = self.linear3(x)
x = self.linear4(x)
return x
def fitness_func(solution, solution_idx):
return 0
def trainModel():
NeuralNet = NN(1440, 1440, 3)
torch_ga = pygad.torchga.TorchGA(model=NeuralNet, num_solutions=10)
ga_instance = pygad.GA(num_generations=10,
num_parents_mating=2,
initial_population=torch_ga.population_weights,
fitness_func=fitness_func)
ga_instance.run()
if __name__ == "__main__":
trainModel()
Both of these should work the same but they don't
When you look at the pygad code you can see it's explicitly checking that the fitness function has exactly two parameters:
# Check if the fitness function accepts 2 paramaters.
if (fitness_func.__code__.co_argcount == 2):
self.fitness_func = fitness_func
else:
self.valid_parameters = False
raise ValueError("The fitness function must accept 2 parameters:\n1) A solution to calculate its fitness value.\n2) The solution's index within the population.\n\nThe passed fitness function named '{funcname}' accepts {argcount} parameter(s).".format(funcname=fitness_func.__code__.co_name, argcount=fitness_func.__code__.co_argcount))
So if you want to use it in a class you'll need to make it a static method so you aren't required to pass in self:
#staticmethod
def fitness_func(solution, solution_idx):
return 0
I have a simple model:
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
self.fc1 = nn.Linear(3, 10)
self.fc2 = nn.Linear(10, 30)
self.fc3 = nn.Linear(30, 2)
def forward(self, x):
x = torch.relu(self.fc1(x))
x = torch.relu(self.fc2(x))
x = torch.tanh(self.fc3(x))
return x
net = Model()
How can I keep the weights to always be between a certain value (eg -1,1)?
I tried the following:
self.fc1 = torch.tanh(nn.Linear(3, 10))
Which I'm not entirely sure that will always keep them in that value (even if the gradient update is trying to push them farther).
But got the following error:
TypeError: tanh(): argument 'input' (position 1) must be Tensor, not Linear
According to the discuss.pytorch you can create extra class to clip weights between a given range. Link to the discussion.
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
self.fc1 = nn.Linear(3, 10)
self.fc2 = nn.Linear(10, 30)
self.fc3 = nn.Linear(30, 2)
def forward(self, x):
x = torch.relu(self.fc1(x))
x = torch.relu(self.fc2(x))
x = torch.tanh(self.fc3(x))
return x
You should add weight clipper:
class WeightClipper(object):
def __call__(self, module):
# filter the variables to get the ones you want
if hasattr(module, 'weight'):
w = module.weight.data
w = w.clamp(-1,1)
module.weight.data = w
model = Model()
clipper = WeightClipper()
model.apply(clipper)
Here is a simple example:
class Net(nn.Module):
def __init__(self):
super(Net,self).__init__()
A = Variable(torch.randn((1)), requires_grad=True)
self.A = nn.Parameter(A)
self.B = nn.Linear(2,2)
self.register_parameter("Ablah",self.A)
def forward(self,x):
self.B.weight.data = self.B.weight * self.A
return self.B(x)
net = Net()
input = torch.ones(2,2)
input.requires_grad_(True)
output = net(input)
optim = torch.optim.Adam(net.parameters(), lr = 0.01)
loss = torch.norm(output)
print(net.A)
loss.backward()
optim.step()
print(net.A)
I am trying to modify a PyTorch model's parameter self.B with another parameter self.A by using the code self.B.weight.data = self.B.weight * self.A, but it seems that A has no grad and it can not be updated.
It has confused me for a long time. Is there anything wrong?
Please give me some advice. Thank you!
With Pytorch I am attempting to use ModuleList to ensure model parameters are detected, and can be optimized. When calling the SGD optimizer I get the following error:
ValueError: optimizer got an empty parameter list
Can you please review the code below and advise?
class LR(nn.Module):
def ___init___(self):
super(LR, self).___init___()
self.linear = nn.ModuleList()
self.linear.append(nn.Linear(in_features=28*28, out_features=128, bias=True))
def forward(self, x):
y_p = torch.sigmoid(self.linear(x))
return y_p
LR_model = LR()
optimizer = torch.optim.SGD(params = LR_model.parameters(), lr=learn_rate)
This seems to be a copy-paste issue: your __init__ has 3 underscores instead of 2, both at __init__(self) and super(LR, self).__init__(). Thus the init itself failed. Delete the extra underscores and try again or try the below code:
class LR(nn.Module):
def __init__(self):
super(LR, self).__init__()
self.linear = nn.ModuleList()
self.linear.append(nn.Linear(in_features=28*28,
out_features=128,
bias=True))
def forward(self, x):
y_p = torch.sigmoid(self.linear(x))
return y_p
LR_model = LR()
optimizer = torch.optim.SGD(params = list(LR_model.parameters()),
lr=learn_rate)
I am trying to get into Neural Networks and wanted to test some code from a video I am watching, but I keep getting the error, " Neural_Network object has no attribute w1", and I can't seem to figure it out. I went through many related stackoverflows, but they don't seem to answer the question, and since I haven't done any object-oriented programming for python before, I do not understand what is going on.
When I looked through the code, I thought that self.w1 was being set as a local variable so I tried to instantiate it and set is as a global variable above the constructor declaration, but that didn't work.
import numpy as np
class Neural_Network(object):
def _init_(self):
self.inputLayerSize = 2
self.outputLayerSize = 1
self.hiddenLayerSize = 3
self.w1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)
self.w2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)
def forward(self, x):
self.z2= np.dot(x, self.w1)
self.a2 = self.sigmoid(self.layer1)
self.z3 = np.dot(self.a2,self.w2)
yhat = self.sigmoid(self.z3)
return yhat
def sigmoid(self, x):
return 1 / (1 + np.exp(-x))
x = np.array(([3,5],[5,1],[10,2]),dtype=float)
y = np.array((([75],[82],[93])),dtype=float)
n1 = Neural_Network()
yhat = n1.forward(x)
print(yhat)
The code should produce a matrix that states possible grades given a specified input 'x'.
something like: x = [[2,3],[5,2]]
output: [[82],[93],[100]]
init should have two underscores surrounding it like:
def __init__(self):
Without two underscores on each side, Python does not call the function when the object is instantiated. In general, all such special functions and attributes have a pair of double underscores.
This noob errors just kills me, i also found it not instantly:
init with one _ is not cound as init so while Neural_Network() nothing was executed
Inheriting from Object is always, dont write that
then you ll notice the hidden layer self.layer1 was not defined
then, do not give variable names one letter + digit, like w1, its very confusing with wl, etc
Ahh also its important, you may not have output [75],[82],[93], because sigmoid as you remember have range from 0 to 1, the same for input. Make sigmoida for input and choose clear outputs, means only 0 or 1
import numpy as np
class Neural_Network:
def __init__(self):
self.inputLayerSize = 2
self.outputLayerSize = 1
self.hiddenLayerSize = 3
self.w_h = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)
self.w_o = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)
self.layer_hidden = np.random.randn(self.hiddenLayerSize)
def forward(self, x):
self.z2= np.dot(x, self.w_h)
self.a2 = self.sigmoid(self.layer_hidden)
self.z3 = np.dot(self.a2,self.w_o)
yhat = self.sigmoid(self.z3)
return yhat
def sigmoid(self, x):
return 1 / (1 + np.exp(-x))
x = np.array(([3,5],[5,1],[10,2]),dtype=float)
y = np.array((([75],[82],[93])),dtype=float)
n1 = Neural_Network()
yhat = n1.forward(x)
print(yhat)