Develop Reference

can't classify the inputs of an formula by output.(celcius-fahrenheit)

I am learning deep learning for a while by myself. However, I try to build a classification model in pytorch. The input and output can be taken from formula of celcius-fahrenheit.
C = (F-32)/1.8
The inputs are value of fahrenheit and the outputs are classified as negative value-0 or positive value-1.
Input
Output
...
1
34
1
33
1
32
0
31
0
...
0
I tried the following pipeline but I can't configure the model to predict the tests with %100 accuracy. How to reach this prediction accuracy rate?
import torch
import numpy as np
x = np.arange(-100,100)
y = np.where(((x-32)*1.8)>0, 1., 0.)
x = torch.from_numpy(x).to(torch.float32).unsqueeze(1)
y = torch.from_numpy(y).to(torch.float32).unsqueeze(1)
class BasicModel(torch.nn.Module):
def __init__(self,in_features:int,out_features:int):
super(BasicModel,self).__init__()
self.in_features = in_features
self.out_features = out_features
self.linear = torch.nn.Linear(in_features=self.in_features,out_features=self.out_features)
self.sigmoid = torch.nn.Sigmoid()
def forward(self,input):
out = self.linear(input)
out = self.sigmoid(out)
return out
model = BasicModel(1,1)
loss_func = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(),lr=0.1)
epochs = 1000
model.train()
for epoch in range(epochs):
losses = []
for value,target in zip(x,y):
optimizer.zero_grad()
prediction = model(value)
loss = loss_func(prediction,target)
loss.backward()
optimizer.step()
losses.append(loss)
if epoch%100 ==0:
for name, param in model.named_parameters():
print(name, param.data,end = " ")
print(f"Epoch:{epoch} loss:{sum(losses)/len(losses)}")
x_test = torch.tensor([[33.],
[32.]]) # 2 sample test input
model.eval()
print(f"Test:{model(x_test)}") # should be [[>0.5],[<0.5]]
I expect weight and bias to be 0.5555 and -17.7777 respectively but both keep increasing.With these expected values,output of the sigmoid should give 1 or 0. Isn't that correct? How to solve this problem?

Sigmoid can neither get 1 nor 0 exactly. It gets closer and closer to 1 for large inputs, which is probably why your weights keep increasing and increasing. However, it never reaches 1.
To solve this issue, remove the sigmoid layer or add a linear layer after the sigmoid layer.
For statically building your classifier, Yyou might actually want to use the heaviside function as activation function: https://en.wikipedia.org/wiki/Heaviside_step_function.
However, as the gradient of this function is zero always, this will not
allow you to train the classifier.

NN from scratch works with simple problems but not with MNIST numbers

I have been working on a NN with one hidden layer with a flexible amount of nodes in each of the three layers. Here is the code:
import time
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.keras.datasets import mnist
class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
class NeuralNetwork():
correct = 0
num_predictions = 10
epochs = 100
sizeOfEpoch = 5000
Lambda = 10
learningRate = 0.00001
def __init__(self, sizes):
self.dimensions = sizes
self.x = np.arange(1,self.epochs+1)
self.y = np.empty(self.epochs)
self.secondLayerNeurons = np.empty(sizes[1])
self.outputNeurons = np.empty(sizes[2])
self.firstLayerWeights = np.random.rand(sizes[1], sizes[0])
self.secondLayerWeights = np.random.rand(sizes[2], sizes[1])
self.firstLayerBiases = np.random.rand(sizes[1])
self.secondLayerBiases = np.random.rand(sizes[2])
self.firstLayerWeightsSummations = np.zeros([sizes[1], sizes[0]])
self.secondLayerWeightsSummations = np.zeros([sizes[2], sizes[1]])
self.firstLayerBiasesSummations = np.zeros([sizes[1]])
self.secondLayerBiasesSummations = np.zeros([sizes[2]])
self.hiddenLayerErrors = np.empty(sizes[1])
self.outputLayerErrors = np.empty(sizes[2])
def sigmoid(self, x):
return 1/(1+np.exp(-x))
def sigmoidDerivative(self, x):
return np.multiply(x,(1-x))
def forwardProp(self, inputs):
for i in range (self.dimensions[1]):
self.secondLayerNeurons[i] = self.sigmoid(np.dot(self.firstLayerWeights[i], inputs)+self.firstLayerBiases[i])
for i in range (self.dimensions[2]):
self.outputNeurons[i] = self.sigmoid(np.dot(self.secondLayerWeights[i], self.secondLayerNeurons)+self.secondLayerBiases[i])
def backProp(self, inputs, correct_output):
self.outputLayerErrors = np.subtract(self.outputNeurons, correct_output)
self.hiddenLayerErrors = np.multiply(np.dot(self.secondLayerWeights.T, self.outputLayerErrors), self.sigmoidDerivative(self.secondLayerNeurons))
for i in range (self.dimensions[2]):
for j in range (self.dimensions[1]):
if j==0:
self.secondLayerBiasesSummations[i] += self.outputLayerErrors[i]
self.secondLayerWeightsSummations[i][j] += self.outputLayerErrors[i]*self.secondLayerNeurons[j]
for i in range (self.dimensions[1]):
for j in range (self.dimensions[0]):
if j==0:
self.firstLayerBiasesSummations[i] += self.hiddenLayerErrors[i]
self.firstLayerWeightsSummations[i][j] += self.hiddenLayerErrors[i]*inputs[j]
def train(self, trainImages, trainLabels):
size = str(self.sizeOfEpoch)
greatestError = 0.0
start_time2 = time.time()
for m in range (self.sizeOfEpoch):
correct_output = np.zeros([self.dimensions[2]])
correct_output[int(class_names[trainLabels[m]])] = 1.0
self.forwardProp(trainImages[m].flatten())
self.backProp(trainImages[m].flatten(), correct_output)
if np.argmax(self.outputNeurons) == int(trainLabels[m]):
self.correct+=1
if m%200 == 0:
error = np.amax(np.absolute(self.outputLayerErrors))
if error > greatestError:
greatestError = error
accuracy = str(int((self.correct/(m+1))*100)) + '%'
percent = str(int((m/self.sizeOfEpoch)*100)) + '%'
print ("Progress: " + percent + " -- Accuracy: " + accuracy + " -- Error: " + str(greatestError), end="\r")
self.change()
time2 = str(round((time.time() - start_time2), 2))
print (size + '/' + size + " -- " + time2 + "s" + " -- Accuracy: " + accuracy + " -- Error: " + str(greatestError), end="\r")
return greatestError
def change(self):
for i in range (self.dimensions[2]):
for j in range (self.dimensions[1]):
if j == 0:
self.secondLayerBiases[i] -= self.learningRate*self.secondLayerBiasesSummations[i]
self.secondLayerWeights[i][j] -= self.learningRate*(self.secondLayerWeightsSummations[i][j]+self.Lambda*self.secondLayerWeights[i][j])
for i in range (self.dimensions[1]):
for j in range (self.dimensions[0]):
if j == 0:
self.firstLayerBiases[i] -= self.learningRate*self.firstLayerBiasesSummations[i]
self.firstLayerWeights[i][j] -= self.learningRate*(self.firstLayerWeightsSummations[i][j]+self.Lambda*self.firstLayerWeights[i][j])
self.firstLayerSummations = np.zeros([self.dimensions[1], self.dimensions[0]])
self.secondLayerSummations = np.zeros([self.dimensions[2], self.dimensions[1]])
self.firstLayerBiasesSummations = np.zeros(self.dimensions[1])
self.secondLayerBiasesSummations = np.zeros(self.dimensions[2])
self.correct = 0
def predict(self, testImage):
secondLayerAnsNodes = np.empty([self.dimensions[1]])
outputAns = np.empty([self.dimensions[2]])
for i in range (self.dimensions[1]):
secondLayerAnsNodes[i] = self.sigmoid(np.dot(self.firstLayerWeights[i], testImage)+self.firstLayerBiases[i])
for i in range (self.dimensions[2]):
outputAns[i] = self.sigmoid(np.dot(self.secondLayerWeights[i], secondLayerAnsNodes)+self.secondLayerBiases[i])
return np.argmax(outputAns)
if __name__ == "__main__":
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images/255.0
test_images = test_images/255.0
neural_network = NeuralNetwork([784, 16, 10])
start_time = time.time()
for i in range (neural_network.epochs):
print ("\nEpoch", str(i+1) + "/" + str(neural_network.epochs))
neural_network.y[i]=neural_network.train(train_images, train_labels)
time = time.time() - start_time
plt.plot(neural_network.x, neural_network.y, 'b')
plt.ylabel('Error Change')
plt.xlabel('Epochs')
plt.show()
print("\n\n\nTotal Time Used")
if time/60 < 60:
print("Minutes: %s" % round((time/60),2))
else:
print("Seconds: %s" % round(time,2))
for i in range (neural_network.num_predictions):
prediction = neural_network.predict(test_images[i].flatten())
plt.grid(False)
plt.imshow(test_images[i], cmap=plt.cm.binary)
plt.title("Prediction: " + str(prediction) + " -- Actual: " + class_names[test_labels[i]] + "\n" + str(i+1) + "/" + str(neural_network.num_predictions))
plt.show()
For some reason, this code does not work with more complicated problems. The error doesn't get minimized and the accuracy stays the same. This exact code works for the xor problem and another one similar to it. When I try and give it the MNIST dataset of numbers, it doesn't work. The only difference is that there are more nodes in each layer, the algorithm is the same.
What could be the problem here?
This is a graph after running 20 epochs with a learning rate of 0.000001 and a lambda of 10. It shows the error per epoch. The y label is supposed to say error, not error change.
https://i.stack.imgur.com/fLXzz.png

There is nothing technically wrong with your implementation. However, there are a few things to note, all of which have significant impact on the performance you are seeing. This is a long answer, but each part reflects an important change that I made to your code to make it work as intended, so read it all carefully.
Firstly, you shouldn't initialize your weights in (0, 1), which is what np.random.randn does by default. Specifically, if you're going to choose uniform random weights, the uniform distribution should be centered at zero. For instance, choose random numbers in the range (-1, 1), or (-.1, .1). Otherwise, your MLP is immediately biased; many of the hidden layer neurons will immediately map to nearly 1 through the sigmoid activation. Afterall, the sigmoid activation is centered (along the x axis) at zero, so your default inputs should be as well. This problem can very easily prevent your MLP from converging at all (and, in fact, it does in your case). There are better weight initialization methods than sampling from a uniform random distribution, but that is not to say that this method will not work if done properly.
Secondly, you should probably normalize the image data. Neural networks don't do great with inputs between 0 and 255, which is how the image data is exported from keras by default. You can fix this by just dividing each input feature by 255. The reason for this is that the sigmoid curve has very small derivatives at high-magnitude subdomains. In other words, when x is very large or very small (very negative), the derivative of sigmoid(x) w.r.t x is very nearly zero. When you're multiplying some weights by very large values (such as 255), then you are likely to get into this high-magnitude domain of the sigmoid curve right off the bat. This won't necessarily prevent your net from converging, but it'll certainly slow it down in the beginning since small derivatives result in small gradients, which in turn result in small weight updates. You could increase the learning rate, but that might result in the neural network overstepping (and possibly diverging) once it gets out of the low-derivative areas of the sigmoid curve. Again, I have tested (and fixed) this problem in your specific program, and it does make a significant difference (final accuracies around .8 as opposed to .6).
Next, your way of calculating an "error" is a bit strange. It computes the largest error over the entire epoch and prints it. The largest error of an epoch is hardly a useful error metric; even a very well-designed, well-trained, deep convolutional neural network will occasionally perform poorly on at least one datapoint throughout an epoch. Your accuracy measure is probably decent enough for getting a gauge of how well your model is converging. However, I also added an "average error" by simply adapting your current error calculation. Since you're using cross entropy loss (at least, this is true given your method of calculating gradients), I'd recommend writing a function which actually computes a cross entropy loss (sum of negative log likelihoods in your case). Keep in mind when interpreting such a loss that the negative log likelihood over a sigmoid is bounded in (0, infinity), and so cross entropy loss is as well.
Of course, another issue is likely the learning rate. In fact, most would argue that the learning rate is the most important hyperparameter to tune. I ended up using 0.00001, though I didn't do much of a grid search.
Next, you are using full batch learning. This means that you compute a sum of gradients over every single data point, and then you update your weights once. In other words, you only perform one weight update per epoch. If that's the case, you're going to have to do a lot of epochs to get decent results. If you have the time and computational resources, that's probably fine. However, if you don't, you might consider mini-batch. Mini-batch is still fairly robust to sample order (though you theoretically should still shuffle the data for each epoch), at least compared to online / stochastic learning. It involves dividing your full dataset into "batches" of some predefined size. For each batch, you compute the sum of the model gradients over each data point in the batch. Then, you do a weight update (by calling change()). Once you've gone over every batch, that constitutes a single epoch. I used minibatch and a batch size of 1,000.
Lastly (and I want to say most importantly, but the other things I've mentioned are also preventing convergence), you are not training on all the training data (8,000 / 60,000); you are not training for enough epochs (5 is not likely to be enough, especially when you're only training on a fraction of the data); and your model is likely too simple (not enough hidden layer nodes). However, the overarching issue is that the implementation does not always make use of vectorized operations when appropriate, and thus it is far too slow to actually train on all of the training data with a sufficient number epochs and model complexity.
I updated your implementation (most notably backprop() and change()) to make use of numpy's vectorized operations whenever possible. This sped the implementation up by several orders of magnitude. However, I don't believe it inherently changed the semantics of your code at all. I've also implemented the other changes I suggested throughout this post. I'm getting about 85% training accuracy on average (though it varies +/- 6% by batch) after just 20 epochs and only 32 hidden nodes in the hidden layer. I did not run it against the test set, and so I did not mess with the regularization parameter either (I simply set Lambda to zero). Here is the updated code (I redacted parts, such as the predict() function, for brevity):
import numpy as np
from tensorflow.keras.datasets import mnist
class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
class NeuralNetwork():
correct = 0
epochs = 20
Lambda = 0
learningRate = 0.00001
def __init__(self, sizes, batchSize):
self.batchSize = batchSize
self.dimensions = sizes
self.secondLayerNeurons = np.empty(sizes[1])
self.outputNeurons = np.empty(sizes[2])
# Draw weights and biases from (-1, 1) by multiplying the (0, 1)
# values by 2 and subtracting 1. There are better ways of doing this,
# but this works just fine.
self.firstLayerWeights = np.random.rand(sizes[1], sizes[0]) * 2 - 1
self.secondLayerWeights = np.random.rand(sizes[2], sizes[1]) * 2 - 1
self.firstLayerBiases = np.random.rand(sizes[1]) * 2 - 1
self.secondLayerBiases = np.random.rand(sizes[2]) * 2 - 1
self.firstLayerWeightsSummations = np.zeros([sizes[1], sizes[0]])
self.secondLayerWeightsSummations = np.zeros([sizes[2], sizes[1]])
self.firstLayerBiasesSummations = np.zeros([sizes[1]])
self.secondLayerBiasesSummations = np.zeros([sizes[2]])
self.hiddenLayerErrors = np.empty(sizes[1])
self.outputLayerErrors = np.empty(sizes[2])
def sigmoid(self, x):
return 1/(1+np.exp(-x))
def sigmoidDerivative(self, x):
return np.multiply(x,(1-x))
def forwardProp(self, inputs):
for i in range (self.dimensions[1]):
self.secondLayerNeurons[i] = self.sigmoid(np.dot(self.firstLayerWeights[i], inputs)+self.firstLayerBiases[i])
for i in range (self.dimensions[2]):
self.outputNeurons[i] = self.sigmoid(np.dot(self.secondLayerWeights[i], self.secondLayerNeurons)+self.secondLayerBiases[i])
def backProp(self, inputs, correct_output):
self.outputLayerErrors = np.subtract(self.outputNeurons, correct_output)
self.hiddenLayerErrors = np.multiply(np.dot(self.secondLayerWeights.T, self.outputLayerErrors), self.sigmoidDerivative(self.secondLayerNeurons))
self.secondLayerBiasesSummations += self.outputLayerErrors
self.secondLayerWeightsSummations += np.outer(self.outputLayerErrors, self.secondLayerNeurons)
self.firstLayerBiasesSummations += self.hiddenLayerErrors
self.firstLayerWeightsSummations += np.outer(self.hiddenLayerErrors, inputs)
def train(self, trainImages, trainLabels):
size = str(self.batchSize)
err_sum = 0.0
err_count = 0
avg_err = 0.0
for m in range (self.batchSize):
correct_output = np.zeros([self.dimensions[2]])
correct_output[trainLabels[m]] = 1.0
self.forwardProp(trainImages[m].flatten())
self.backProp(trainImages[m].flatten(), correct_output)
if np.argmax(self.outputNeurons) == int(trainLabels[m]):
self.correct+=1
if m%150 == 0:
error = np.amax(np.absolute(self.outputLayerErrors))
err_sum += error
err_count += 1
avg_err = err_sum / err_count
accuracy = str(int((self.correct/(m+1))*100)) + '%'
percent = str(int((m/self.batchSize)*100)) + '%'
print ("Progress: " + percent + " -- Accuracy: " + accuracy + " -- Error: " + str(avg_err), end="\r")
self.change()
print (size + '/' + size + " -- " + " -- Accuracy: " + accuracy + " -- Error: " + str(avg_err), end="\r")
self.correct = 0
def change(self):
self.secondLayerBiases -= self.learningRate * self.secondLayerBiasesSummations
self.secondLayerWeights -= self.learningRate * self.secondLayerWeightsSummations
self.firstLayerBiases -= self.learningRate * self.firstLayerBiasesSummations
self.firstLayerWeights -= self.learningRate * self.firstLayerWeightsSummations
self.firstLayerSummations = np.zeros([self.dimensions[1], self.dimensions[0]])
self.secondLayerSummations = np.zeros([self.dimensions[2], self.dimensions[1]])
self.firstLayerBiasesSummations = np.zeros(self.dimensions[1])
self.secondLayerBiasesSummations = np.zeros(self.dimensions[2])
if __name__ == "__main__":
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images / 255 # Normalize image data
num_using = 60000 # Amount of data points to use. It's fast now, so we may as well use the full 60,000
bs = 1000 # Batch size. 60,000 is full batch. Consider trying mini-batch
neural_network = NeuralNetwork([784, 32, 10], bs)
for i in range (neural_network.epochs):
print ("\nEpoch", str(i+1) + "/" + str(neural_network.epochs))
for j in range(int(num_using / bs)):
print("Batch", str(j+1) + "/" + str(int(60000 / bs)))
neural_network.train(train_images[int(j * bs):int(j * bs) + bs], train_labels[int(j * bs):int(j * bs) + bs])
For further improvements requiring minimal effort, I'd suggest trying even more hidden nodes (perhaps even 128), further tweaking the learning rate and regularization parameter, trying different batch sizes, and tweaking the number of epochs.
Let me know if you have any questions.

Why does my neural network give the correct outputs after backpropagation, but doesn't on new inputs?

I've been trying my hand in neural networks, and below is a simple one which uses a sigmoid function to output 1 (or a number REALLY close) if the number is odd, and 0 if the number is even. After training the neural network the output is correct. When new values are introduced the output is zero. Why is the output zero?
import numpy as np
inputs = np.array([9, 45, 62, 87, 88, 49])
outputs = np.array([1, 1, 0, 1, 0, 1]) # 1 for odd 0 for even
weights = np.random.random(()) # weights are initialized as random.
lr = 0.1 # learning rate
mw = np.dot(inputs, weights)
def sigmoid(x, deriv=False):
if deriv == True:
return sigmoid(x) * (1 - sigmoid(x)) # derivative of sigmoid
else:
return 1 / (1 + np.exp(-x))
z = sigmoid(mw)
print("Results before training: {}".format(z))
# Results before backpropagation with random value as weight
for x in range(20000): # training loop
error = (z - outputs)
adjustments = sigmoid(z, deriv=True) * error * inputs
weights = weights - lr * adjustments
# readjusting the weights to minimize error
# After the training loop with the readjusted weights
new_mw = (weights * inputs)
new_z = sigmoid(new_mw) # sigmoid of new weights * input
print("New results after training:{}".format(new_z)) # -> [1, 1, 0, 1, 0, 1]
def think(x, weights):
print("New situation: {}".format(x))
xw = np.dot(x, weights)
print("New results after thinking: {}".format(sigmoid(xw)))
x = np.array([2, 4, 6, 7, 17, 53]) #array of new test data
think(x, weights) # -> 0.0

Impossible solution
There is no possible weight (single! at least in your code) that would result in your function being able to output 1 (or a number REALLY close) if the number is odd, and 0 if the number is even.
It's plausible that during training it learns a relationship that it should output close to 1 for large numbers and 0 for smaller numbers, as that's probably the best it can do with a single multiplicative weight. Looking at your training data, it may be that the boundary is somewhere around 50 or 60, which would result in all your test samples being 0, because they're small-ish - but you could (and perhaps should!) draw a graph for all the values of your learned function from, say, 1 to 100 to illustrate this.
It's worth spending some time thinking why you believe(d) that some weight in f(x) = sigmoid(x * [trainable weight]) would result in a function that distinguishes even numbers from odd numbers. If this is not obvious, plotting the learned function might be informative.

Always same output for tensorflow autoencoder

At the moment I try to build an Autoencoder for timeseries data in tensorflow. I have nearly 500 days of data where each day have 24 datapoints. Since this is my first try my architecture is very simple. After my input of size 24 the hidden layers are of size: 10; 3; 10 with an output of again 24. I normalized the data (datapoints are in range [-0.5; 0.5]), use the sigmoid activation function and the RMSPropOptimizer.
After training (loss function in picture) the output is the same for every timedata i give into the network. Does someone know what is the reason for that? Is it possible that my Dataset is the issue (code below)?
class TimeDataset:
def __init__(self,data):
self._index_in_epoch = 0
self._epochs_completed = 0
self._data = data
self._num_examples = data.shape[0]
pass
#property
def data(self):
return self._data
def next_batch(self, batch_size, shuffle=True):
start = self._index_in_epoch
# first call
if start == 0 and self._epochs_completed == 0:
idx = np.arange(0, self._num_examples) # get all possible indexes
np.random.shuffle(idx) # shuffle indexe
self._data = self.data[idx] # get list of `num` random samples
if start + batch_size > self._num_examples:
# not enough samples left -> go to the next batch
self._epochs_completed += 1
rest_num_examples = self._num_examples - start
data_rest_part = self.data[start:self._num_examples]
idx0 = np.arange(0, self._num_examples) # get all possible indexes
np.random.shuffle(idx0) # shuffle indexes
self._data = self.data[idx0] # get list of `num` random samples
start = 0
self._index_in_epoch = batch_size - rest_num_examples #avoid the case where the #sample != integar times of batch_size
end = self._index_in_epoch
data_new_part = self._data[start:end]
return np.concatenate((data_rest_part, data_new_part), axis=0)
else:
# get next batch
self._index_in_epoch += batch_size
end = self._index_in_epoch
return self._data[start:end]
*edit: here are some examples of the output (red original, blue reconstructed):
**edit: I just saw an autoencoder example with a more complicant luss function than mine. Someone know if the loss function self.loss = tf.reduce_mean(tf.pow(self.X - self.decoded, 2)) is sufficient?
***edit: some more code to describe my training
This is my Autoencoder Class:
class AutoEncoder():
def __init__(self):
# Training Parameters
self.learning_rate = 0.005
self.alpha = 0.5
# Network Parameters
self.num_input = 24 # one day as input
self.num_hidden_1 = 10 # 2nd layer num features
self.num_hidden_2 = 3 # 2nd layer num features (the latent dim)
self.X = tf.placeholder("float", [None, self.num_input])
self.weights = {
'encoder_h1': tf.Variable(tf.random_normal([self.num_input, self.num_hidden_1])),
'encoder_h2': tf.Variable(tf.random_normal([self.num_hidden_1, self.num_hidden_2])),
'decoder_h1': tf.Variable(tf.random_normal([self.num_hidden_2, self.num_hidden_1])),
'decoder_h2': tf.Variable(tf.random_normal([self.num_hidden_1, self.num_input])),
}
self.biases = {
'encoder_b1': tf.Variable(tf.random_normal([self.num_hidden_1])),
'encoder_b2': tf.Variable(tf.random_normal([self.num_hidden_2])),
'decoder_b1': tf.Variable(tf.random_normal([self.num_hidden_1])),
'decoder_b2': tf.Variable(tf.random_normal([self.num_input])),
}
self.encoded = self.encoder(self.X)
self.decoded = self.decoder(self.encoded)
# Define loss and optimizer, minimize the squared error
self.loss = tf.reduce_mean(tf.pow(self.X - self.decoded, 2))
self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate).minimize(self.loss)
def encoder(self, x):
# sigmoid, tanh, relu
en_layer_1 = tf.nn.sigmoid (tf.add(tf.matmul(x, self.weights['encoder_h1']),
self.biases['encoder_b1']))
en_layer_2 = tf.nn.sigmoid (tf.add(tf.matmul(en_layer_1, self.weights['encoder_h2']),
self.biases['encoder_b2']))
return en_layer_2
def decoder(self, x):
de_layer_1 = tf.nn.sigmoid (tf.add(tf.matmul(x, self.weights['decoder_h1']),
self.biases['decoder_b1']))
de_layer_2 = tf.nn.sigmoid (tf.add(tf.matmul(de_layer_1, self.weights['decoder_h2']),
self.biases['decoder_b2']))
return de_layer_2
and this is how I train my network (input data have shape (number_days, 24)):
model = autoencoder.AutoEncoder()
num_epochs = 3
batch_size = 50
num_batches = 300
display_batch = 50
examples_to_show = 16
loss_values = []
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#training
for e in range(1, num_epochs+1):
print('starting epoch {}'.format(e))
for b in range(num_batches):
# get next batch of data
batch_x = dataset.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
l = sess.run([model.loss], feed_dict={model.X: batch_x})
sess.run(model.optimizer, feed_dict={model.X: batch_x})
# Display logs
if b % display_batch == 0:
print('Epoch {}: Batch ({}) Loss: {}'.format(e, b, l))
loss_values.append(l)
# testing
test_data = dataset.next_batch(batch_size)
decoded_test_data = sess.run(model.decoded, feed_dict={model.X: test_data})

Just a suggestion, I have had some issues with autoencoders using the sigmoid function.
I switched to tanh or relu and those improved the results.
With the autoencoder it is basically learning to recreate the output from the input, by encoding and decoding. If you mean it's the same as the input, then you are getting what you want. It has learned the data set.
Ultimately you can compare by reviewing the Mean Squared Error between the input and output and see if it is exactly the same. If you mean that the output is exactly the same regardless of the input, that isn't something I've run into. I guess if your input doesn't vary much from day to day, then I could imagine that would have some impact. Are you looking for anomalies?
Also, if you have a time series for training, I wouldn't shuffle the data in this particular case. If the temporal order is significant, you introduce data leakage (basically introducing future data into the training set) depending on what you are trying to achieve.
Ah, I didn't initially see your post with the graph results.. thanks for adding.

The sigmoid output is floored at 0, so it cannot reproduce your data that is below 0.
If you want to use a sigmoid output, then rescale your data between ]0;1[ (0 and 1 excluded).

I know this is a very old post, so this is just an attempt to help whoever wonders here again with the same problem.... If the autoencoder is converging to the same encoding for all the different instances, there may be a problem in the loss function.... Check the size and shape of the return of the loss function, as it may be getting confused and evaluating the wrong tensors (i.e. you may need to transpose something somewhere) Basically, assuming you are using the autoencoder to encode M features of N training instances, your loss function should return N values. the size of your loss tensor should be the amount of instances in your training set. I found that the hard way.....

reshape Error/ValueError: total size of new array must be unchanged

I have a code for image classification using CNN, so there are training data set and testing dataset. When I perform the system I have this error:
ValueError Traceback (most recent call last)
<ipython-input-44-cb7ec1a13881> in <module>()
1 optimize(num_iterations=1)
2
----> 3 print_validation_accuracy()
<ipython-input-43-7f1a17e48e41> in print_validation_accuracy(show_example_errors, show_confusion_matrix)
21
22 # Get the images from the test-set between index i and j.
---> 23 images = data.valid.images[i:j, :].reshape(batch_size, img_size_flat)
24 #images = data.valid.images[i:j, :].reshape(1, 128)
25
ValueError: total size of new array must be unchanged
and the steps of the code that precessed this error are:
def print_validation_accuracy(show_example_errors=False,
show_confusion_matrix=False):
# Number of images in the test-set.
num_test = len(data.valid.images)
# Allocate an array for the predicted classes which
# will be calculated in batches and filled into this array.
cls_pred = np.zeros(shape=num_test, dtype=np.int)
# Now calculate the predicted classes for the batches.
# We will just iterate through all the batches.
# There might be a more clever and Pythonic way of doing this.
# The starting index for the next batch is denoted i.
i = 0
while i < num_test:
# The ending index for the next batch is denoted j.
j = min(i + batch_size, num_test)
# Get the images from the test-set between index i and j.
images = data.valid.images[i:j, :].reshape(batch_size, img_size_flat)
# Get the associated labels.
labels = data.valid.labels[i:j, :]
# Create a feed-dict with these images and labels.
feed_dict = {x: images,
y_true: labels}
# Calculate the predicted class using TensorFlow.
cls_pred[i:j] = session.run(y_pred_cls, feed_dict=feed_dict)
# Set the start-index for the next batch to the
# end-index of the current batch.
i = j
cls_true = np.array(data.valid.cls)
cls_pred = np.array([classes[x] for x in cls_pred])
# Create a boolean array whether each image is correctly classified.
correct = (cls_true == cls_pred)
# Calculate the number of correctly classified images.
# When summing a boolean array, False means 0 and True means 1.
correct_sum = correct.sum()
# Classification accuracy is the number of correctly classified
# images divided by the total number of images in the test-set.
acc = float(correct_sum) / num_test
# Print the accuracy.
msg = "Accuracy on Test-Set: {0:.1%} ({1} / {2})"
print(msg.format(acc, correct_sum, num_test))
# Plot some examples of mis-classifications, if desired.
if show_example_errors:
print("Example errors:")
plot_example_errors(cls_pred=cls_pred, correct=correct)
# Plot the confusion matrix, if desired.
if show_confusion_matrix:
print("Confusion Matrix:")
plot_confusion_matrix(cls_pred=cls_pred)
Can anyone help me please?

As the error message shows, there is a mismatch in your reshape in this statement.
images = data.valid.images[i:j, :].reshape(batch_size, img_size_flat)
What is happening is this equation is not equal i.e
(j - i) * (column_size of data.valid.images) is not equal to batch_size * img_size_flat.
Make it equal and the problem will be solved.