Develop Reference

How to forecast a univariate time series 20/30 days ahead using tensorflow LSTM?

I have used below code for training and validation. It gives decent result but I don't know the code to forecast n periods ahead (like 30/50 days ahead) using the trained model.
GitHub Link for the code with data output is here.
Import the libraries:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import math
from keras.models import Sequential
from keras.layers import Dense,Dropout,Conv1D,Bidirectional
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')
import tensorflow
np.random.seed(1)
tensorflow.random.set_seed(1)
Load the univariate time series data and normalize the values:
dataframe=pd.read_sas('train.sas7bdat')
dataframe['Datetime']=pd.to_datetime(dataframe['Datetime'],format='%d%b%Y:%H:%M:%S')
dataframe.set_index('Datetime',inplace=True)
data=dataframe
#filter input data according to datetime i.e: 01th May 2020
dataset=data[data.index>='2021-05-01 00:00:00']
# Replcae null value with previous 15 minute value
dataset.ffill(axis ='rows',inplace=True)
dataset.shape
def normalize_cols(df,cols):
"""Scale the values of each feature
according to the columns max value"""
data = df.loc[:,cols]
for col in cols:
scaler = lambda x: x / data[col].max()
data[col] = data[col].apply(scaler)
print(data[cols].head())
return data[cols].values
features = df.columns.values # columns to train model on
X = normalize_cols(df,features)
Turn each signal into a labeled dataset:
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
window_size = 15 #15 # num. days per training sample
batch_size = 64 # num. of samples per epoch
buffer_size = 1000 # num of samples in memory for random selection
split_time = 400 # where to split the data for training/validation
forecast_length=10*24*4
def window_dataset(series, window_size, batch_size, shuffle_buffer):
"""Funtion to turn time series data into set of sequences
where the last value is the intended output of our model"""
ser = tf.expand_dims(series, axis=-1)
data = tf.data.Dataset.from_tensor_slices(series)
data = data.window(window_size + 1, shift=1, drop_remainder=True)
data = data.flat_map(lambda w: w.batch(window_size + 1))
data = data.shuffle(shuffle_buffer)
data = data.map(lambda w: (w[:-1], w[1:]))
return data.batch(batch_size).prefetch(1)
x_train = X[:split_time]
x_test = X[split_time:]
print(f"Training data shape: {x_train.shape}")
print(f"Validation data shape: {x_test.shape}")
train_set = window_dataset(x_train,window_size,batch_size,buffer_size)
keras.backend.clear_session()
Choose and connect the model components:
# 1D convolution layers
conv1 = layers.Conv1D(
filters=60,kernel_size=15,strides=1,
padding="causal",activation="relu",
input_shape=[None,len(features)])
conv2 = layers.Conv1D(
filters=60,kernel_size=5,strides=1,
padding="causal",activation="tanh")
# Bidirectional LSTM layers
lstm1 = layers.Bidirectional(layers.LSTM(50,return_sequences=True))
lstm2 = layers.Bidirectional(layers.LSTM(20,return_sequences=True))
# Model construction
inputs = layers.Input(shape=(None,len(features)))
x = conv1(inputs)
x = lstm1(x)
x = lstm2(x)
x = conv2(x)
x = layers.Dense(60,activation='relu')(x)
x = layers.Dropout(.1)(x)
x = layers.Dense(1,activation='tanh')(x)
outputs = layers.Lambda(lambda x: 25*abs(x))(x)
#outputs = layers.Lambda(lambda x: 1*abs(x))(x)
# SGD optimizer and Huber loss
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
loss = keras.losses.Huber()
model = keras.Model(inputs=inputs,outputs=outputs)
model.compile(optimizer,loss,
metrics=["mae"])
model.summary()
"""
### Train model
"""
epochs = 100
history = model.fit(train_set, epochs=epochs, verbose=1)
print(f"Model trained for {epochs} epochs")
Inspect training results:
def model_forecast(model, X, window_size):
"""Takes in numpy array, creates a windowed tensor
and predicts the following value on each window"""
data = tf.data.Dataset.from_tensor_slices(X)
data = data.window(window_size, shift=1, drop_remainder=True)
data = data.flat_map(lambda w: w.batch(window_size))
data = data.batch(32).prefetch(1)
forecast = model.predict(data)
return forecast
train_window = [i for i in range(split_time-window_size)]
forecast = model_forecast(model,x_train,window_size)
import seaborn as sns
plt.figure(figsize=(8,5),dpi=120)
sns.lineplot(train_window,forecast[:-1,1,0].reshape(-1),label='Forecast') #forecast[:-1,1,0]
sns.lineplot(train_window,X[:split_time-window_size].reshape(-1),label='actual_load')
Make predictions on test data:
val_window = [i for i in range(split_time,len(df)-window_size)]
forecast = model_forecast(model,x_test,window_size)
plt.figure(figsize=(8,5),dpi=120)
sns.lineplot(val_window,forecast[:-1,1,0].reshape(-1),label='Forecast')
sns.lineplot(val_window,X[split_time:-window_size].reshape(-1),label='actual_load')

RuntimeError: shape '[265, 265]' is invalid for input of size 265

I am trying to train a model using Gaussian Process regression in Python, and I have some problems in predicting. The training set is of size 265, and after having trained the model, I am trying to predict on the test set.
I get the error "RuntimeError: shape '[265, 265]' is invalid for input of size 265" and I don't really understand why.
I also get a vector as loss while training (instead than a scalar). I solved the loss problem by doing loss.mean() (even if I still don't understand why the loss is returned as a vector), but I have not managed to solve the other problem.
265 is the size of the training set (and not of the test set), so I imagine the error is related to that and not to the test set that I give in input.
Thank you really much for your help! I send all my code, the problem is in the function predict. PS: I am sorry but I don't know how to upload the training and test data here.
'''
import os
import typing
#from sklearn.gaussian_process.kernels import *
import numpy as np
#from sklearn.gaussian_process import GaussianProcessRegressor
#from sklearn.model_selection import cross_val_score
#from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
from matplotlib import cm
import gpytorch
import matplotlib.pyplot as plt
from matplotlib import rcParams
import numpy as np
import torch
import pandas as pd
class Model(object): # were we allowed to change here?
"""
Model for this task.
You need to implement the fit_model and predict methods
without changing their signatures, but are allowed to create additional methods.
"""
def __init__(self): # are we allowed to change here?
"""
Initialize your model here.
We already provide a random number generator for reproducibility.
"""
self.likelihood = gpytorch.likelihoods.GaussianLikelihood()
# self.mean_module = gpytorch.means.ZeroMean() #prior mean
# self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel()) #covariance (which kernel to use?)
self.model = None
self.rng = np.random.default_rng(seed=0) # it was already present
# TODO: Add custom initialization for your model here if necessary
def predict(self, x: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""
Predict the pollution concentration for a given set of locations.
:param x: Locations as a 2d NumPy float array of shape (NUM_SAMPLES, 2)
#TODO: for us it is 1-d!!!
:return:
Tuple of three 1d NumPy float arrays, each of shape (NUM_SAMPLES,),
containing your predictions, the GP posterior mean, and the GP posterior stddev (in that order)
"""
# TODO: Use your GP to estimate the posterior mean and stddev for each location here
gp_mean = np.zeros(x.shape[0], dtype=float)
gp_std = np.zeros(x.shape[0], dtype=float)
x = torch.Tensor(x)
# Turn the model into evaluation mode
self.model.eval()
self.likelihood.eval()
# The gpytorch.settings.fast_pred_var flag activates LOVE (for fast variances)
# See https://arxiv.org/abs/1803.06058
with torch.no_grad(), gpytorch.settings.fast_pred_var():
# Obtain the predictive mean and covariance matrix
#print(x.size())
f_preds = self.model(x) #TODO: check the input!!!
print(type(f_preds))
gp_mean = np.array(f_preds.mean)
gp_std = torch.sqrt(f_preds.variance) # check if it is right
gp_std = np.array(gp_std)
# Make predictions by feeding model through likelihood
# observed_pred = self.likelihood(f_preds) #I think it is useless
# TODO: Use the GP posterior to form your predictions here
predictions = gp_mean
# check if we want to change the predictions for the asymmetric cost
'''for i in range(0, len(predictions)):
if (predictions[i] + 3 * gp_std[i] > 35.5) and (predictions[i] < 35.5):
predictions[i] = 35.51
elif ((predictions[i] - gp_std[i]) > 35.5):
predictions[i] = predictions[i] - gp_std[i]'''
return predictions, gp_mean, gp_std
def fit_model(self, train_x: np.ndarray, train_y: np.ndarray):
"""
Fit your model on the given training data.
:param train_x: Training features as a 2d NumPy float array of shape (NUM_SAMPLES, 2) #TODO: 1-d for us!!!
:param train_y: Training pollution concentrations as a 1d NumPy float array of shape (NUM_SAMPLES,)
"""
# TODO: Fit your model here
# Put the model into training mode
# initialize likelihood and model (necessary?)
train_x = torch.Tensor(train_x)
train_y = torch.Tensor(train_y)
# torch.Tensor prende in input matrici di numpy e le trasforma in elementi di torch,
# dentro pytorch hai di default un algoritmo di propagation
# clustering
# samplesize=5000
# train_x = train_x[0:samplesize, :]
# train_y = train_y[0:samplesize]
#
self.model = ExactGPModel(train_x, train_y, self.likelihood)
self.model.train()
self.likelihood.train()
# understand how to manage these last 4 rows
# Use the SGD optimizer
optimizer = torch.optim.Adam([
{'params': self.model.parameters()}, # Includes GaussianLikelihood parameters
],
lr=0.1) # if instead of "Adam" we put "SGD" it uses stochastic gradient descent! Oppure minibatch gradient descent
#TODO: choose which optimizer to use
# we can always use stochastic, minibatch or standard(batch)
# we can use gridsearch, change parameters like lr,beta1,beta2,... in order to have a better approximation
# "Loss" for GPs - the marginal log likelihood
mll = gpytorch.mlls.ExactMarginalLogLikelihood(self.likelihood, self.model)
training_iter = 1200 # numero di epochs
for i in range(training_iter):
# Zero gradients from previous iteration
optimizer.zero_grad()
# Output from model
output = self.model(train_x)
# Calc loss and backprop gradients
loss = -mll(output, train_y)
loss = torch.mean(loss) #TODO: understand how to deal with this loss, we shouldn't do the mean!!!
#print(loss)
#print(type(loss))
loss.backward()
print('Iter %d/%d - Loss: %.3f lengthscale: %.3f noise: %.3f' % (
i + 1, training_iter, loss.item(),
self.model.covar_module.base_kernel.lengthscale.item(),
self.model.likelihood.noise.item()))
optimizer.step()
# Auxiliary class
class ExactGPModel(gpytorch.models.ExactGP):
# every object of ExactGPmodel will have every module of models.ExactGP
def __init__(self, train_x, train_y, likelihood):
super(ExactGPModel, self).__init__(train_x, train_y, likelihood)
self.mean_module = gpytorch.means.ZeroMean()
self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=1 / 2))
def forward(self, x):
mean_x = self.mean_module(x)
covar_x = self.covar_module(x)
return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
# Load the training dateset and test features
trainfilename="TrainingData.txt"
testfilename="TestingData.txt"
'''train_set = pd.read_csv(trainfilename)
test_set = pd.read_csv(testfilename)
# Convert Pandas DataFrame to NumpyArray
train_set = train_set.to_numpy()
test_set = test_set.to_numpy()
meant = np.mean(train_set[:,0:1])
sdt = np.std(train_set[:,0:1])
train_set[:,0:1] = (train_set[:,0:1]-meant)/sdt
train_set = torch.from_numpy(train_set)
meant = np.mean(test_set[:,0:1])
sdt = np.std(test_set[:,0:1])
test_set[:,0:1] = (test_set[:,0:1]-meant)/sdt
test_set = torch.from_numpy(test_set)
train_x = train_set[:,0:1]
train_y = train_set[:,1:2]
test_x = test_set[:,0:1]'''
train_set = np.loadtxt(trainfilename, delimiter=',', skiprows=1)
test_x = np.loadtxt(testfilename, delimiter=',', skiprows=1)
train_x = train_set[:,0:1]
train_y = train_set[:,1:2]
#print(type(train_x)) #the type looks fine
# Fit the model
print('Fitting model')
model = Model()
model.fit_model(train_x, train_y) #TODO: how to give the input here???
# Predict on the test features
print('Predicting on test features')
#print(test_x.size())
print(test_x.shape)
predicted_y = model.predict(test_x) #TODO: how to give the input here???
print(predicted_y)
#TODO: RuntimeError: shape '[265, 265]' is invalid for input of size 265
#TODO: I think it does not get the covariance matrix
'''

trying to make prediction in keras without knowing the correct answer

I have a question. I've got trained model, but the thing is I don't actually know how to make a real prediction with that. Output looks like that. But I wanna have a DataFrame for prediction only for a couple days ahead without knowing the actual future data to compare with.
here is the code
X = dataset.iloc[:, 4:-1]
y = dataset.iloc[:, -1]
#dataset shape = (7069, 13)
split = int(len(dataset)*0.8)
X_train, X_test, y_train, y_test = X[:split], X[split:], y[:split], y[split:]
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
classifier = keras.models.load_model('C:\\model')
y_pred = classifier.predict(X_test)
y_pred = (y_pred > 0.5)
dataset['y_pred'] = np.NaN
dataset.iloc[(len(dataset) - len(y_pred)):,-1:] = y_pred
trade_dataset = dataset.dropna()
trade_dataset['Tomorrows Returns'] = 0.
trade_dataset['Tomorrows Returns'] = np.log(trade_dataset['Close']/trade_dataset['Close'].shift(1))
trade_dataset['Tomorrows Returns'] = trade_dataset['Tomorrows Returns'].shift(-1)
trade_dataset['Strategy Returns'] = 0.
trade_dataset['Strategy Returns'] = np.where(trade_dataset['y_pred'] == True, trade_dataset['Tomorrows Returns'], - trade_dataset['Tomorrows Returns'])
trade_dataset['Cumulative Market Returns'] = np.cumsum(trade_dataset['Tomorrows Returns'])
trade_dataset['Cumulative Strategy Returns'] = np.cumsum(trade_dataset['Strategy Returns'])
import matplotlib.pyplot as plt
plt.figure(figsize=(10,5))
plt.plot(trade_dataset['Cumulative Market Returns'], color='r', label='Market Returns')
plt.plot(trade_dataset['Cumulative Strategy Returns'], color='g', label='Strategy Returns')
plt.legend()
plt.grid()
plt.show()

See if this does what you want.
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout, Bidirectional
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from yahoo_fin import stock_info as si
from collections import deque
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time
import os
import random
# set seed, so we can get the same results after rerunning several times
np.random.seed(314)
tf.random.set_seed(314)
random.seed(314)
def load_data(ticker, n_steps=50, scale=True, shuffle=True, lookup_step=1,
test_size=0.2, feature_columns=['adjclose', 'volume', 'open', 'high', 'low']):
# see if ticker is already a loaded stock from yahoo finance
if isinstance(ticker, str):
# load it from yahoo_fin library
df = si.get_data(ticker)
elif isinstance(ticker, pd.DataFrame):
# already loaded, use it directly
df = ticker
# this will contain all the elements we want to return from this function
result = {}
# we will also return the original dataframe itself
result['df'] = df.copy()
# make sure that the passed feature_columns exist in the dataframe
for col in feature_columns:
assert col in df.columns, f"'{col}' does not exist in the dataframe."
if scale:
column_scaler = {}
# scale the data (prices) from 0 to 1
for column in feature_columns:
scaler = preprocessing.MinMaxScaler()
df[column] = scaler.fit_transform(np.expand_dims(df[column].values, axis=1))
column_scaler[column] = scaler
# add the MinMaxScaler instances to the result returned
result["column_scaler"] = column_scaler
# add the target column (label) by shifting by `lookup_step`
df['future'] = df['adjclose'].shift(-lookup_step)
# last `lookup_step` columns contains NaN in future column
# get them before droping NaNs
last_sequence = np.array(df[feature_columns].tail(lookup_step))
# drop NaNs
df.dropna(inplace=True)
sequence_data = []
sequences = deque(maxlen=n_steps)
for entry, target in zip(df[feature_columns].values, df['future'].values):
sequences.append(entry)
if len(sequences) == n_steps:
sequence_data.append([np.array(sequences), target])
# get the last sequence by appending the last `n_step` sequence with `lookup_step` sequence
# for instance, if n_steps=50 and lookup_step=10, last_sequence should be of 59 (that is 50+10-1) length
# this last_sequence will be used to predict in future dates that are not available in the dataset
last_sequence = list(sequences) + list(last_sequence)
# shift the last sequence by -1
last_sequence = np.array(pd.DataFrame(last_sequence).shift(-1).dropna())
# add to result
result['last_sequence'] = last_sequence
# construct the X's and y's
X, y = [], []
for seq, target in sequence_data:
X.append(seq)
y.append(target)
# convert to numpy arrays
X = np.array(X)
y = np.array(y)
# reshape X to fit the neural network
X = X.reshape((X.shape[0], X.shape[2], X.shape[1]))
# split the dataset
result["X_train"], result["X_test"], result["y_train"], result["y_test"] = train_test_split(X, y, test_size=test_size, shuffle=shuffle)
# return the result
return result
def create_model(sequence_length, units=256, cell=LSTM, n_layers=2, dropout=0.3,
loss="mean_absolute_error", optimizer="rmsprop", bidirectional=False):
model = Sequential()
for i in range(n_layers):
if i == 0:
# first layer
if bidirectional:
model.add(Bidirectional(cell(units, return_sequences=True), input_shape=(None, sequence_length)))
else:
model.add(cell(units, return_sequences=True, input_shape=(None, sequence_length)))
elif i == n_layers - 1:
# last layer
if bidirectional:
model.add(Bidirectional(cell(units, return_sequences=False)))
else:
model.add(cell(units, return_sequences=False))
else:
# hidden layers
if bidirectional:
model.add(Bidirectional(cell(units, return_sequences=True)))
else:
model.add(cell(units, return_sequences=True))
# add dropout after each layer
model.add(Dropout(dropout))
model.add(Dense(1, activation="linear"))
model.compile(loss=loss, metrics=["mean_absolute_error"], optimizer=optimizer)
return model
# Window size or the sequence length
N_STEPS = 100
# Lookup step, 1 is the next day
LOOKUP_STEP = 1
# test ratio size, 0.2 is 20%
TEST_SIZE = 0.2
# features to use
FEATURE_COLUMNS = ["adjclose", "volume", "open", "high", "low"]
# date now
date_now = time.strftime("%Y-%m-%d")
### model parameters
N_LAYERS = 3
# LSTM cell
CELL = LSTM
# 256 LSTM neurons
UNITS = 256
# 40% dropout
DROPOUT = 0.4
# whether to use bidirectional RNNs
BIDIRECTIONAL = False
### training parameters
# mean absolute error loss
# LOSS = "mae"
# huber loss
LOSS = "huber_loss"
OPTIMIZER = "adam"
BATCH_SIZE = 64
EPOCHS = 10
# save the dataframe
from datetime import date
from datetime import datetime
start_date = date(2019, 6,30)
end_date = date(2020, 6, 30)
days = np.busday_count(start_date, end_date)
# Apple stock market
ticker = "AAPL"
ticker_data_filename = os.path.join("data", f"{ticker}_{date_now}.csv")
# model name to save, making it as unique as possible based on parameters
model_name = f"{date_now}_{ticker}-{LOSS}-{OPTIMIZER}-{CELL.__name__}-seq-{N_STEPS}-step-{LOOKUP_STEP}-layers-{N_LAYERS}-units-{UNITS}"
if BIDIRECTIONAL:
model_name += "-b"
# create these folders if they does not exist
if not os.path.isdir("results"):
os.mkdir("results")
if not os.path.isdir("logs"):
os.mkdir("logs")
if not os.path.isdir("data"):
os.mkdir("data")
# load the data
data = load_data(ticker, N_STEPS, lookup_step=LOOKUP_STEP, test_size=TEST_SIZE, feature_columns=FEATURE_COLUMNS)
# save the dataframe
data["df"].to_csv(ticker_data_filename)
# construct the model
model = create_model(N_STEPS, loss=LOSS, units=UNITS, cell=CELL, n_layers=N_LAYERS,
dropout=DROPOUT, optimizer=OPTIMIZER, bidirectional=BIDIRECTIONAL)
# some tensorflow callbacks
checkpointer = ModelCheckpoint(os.path.join("results", model_name + ".h5"), save_weights_only=True, save_best_only=True, verbose=1)
tensorboard = TensorBoard(log_dir=os.path.join("logs", model_name))
history = model.fit(data["X_train"], data["y_train"],
batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_data=(data["X_test"], data["y_test"]),
callbacks=[checkpointer, tensorboard],
verbose=1)
model.save(os.path.join("results", model_name) + ".h5")
# after the model ends running...or during training, run this
# tensorboard --logdir="logs"
# http://localhost:6006/
data = load_data(ticker, N_STEPS, lookup_step=LOOKUP_STEP, test_size=TEST_SIZE,
feature_columns=FEATURE_COLUMNS, shuffle=False)
# construct the model
model = create_model(N_STEPS, loss=LOSS, units=UNITS, cell=CELL, n_layers=N_LAYERS,
dropout=DROPOUT, optimizer=OPTIMIZER, bidirectional=BIDIRECTIONAL)
model_path = os.path.join("results", model_name) + ".h5"
model.load_weights(model_path)
# evaluate the model
mse, mae = model.evaluate(data["X_test"], data["y_test"], verbose=0)
# calculate the mean absolute error (inverse scaling)
mean_absolute_error = data["column_scaler"]["adjclose"].inverse_transform([[mae]])[0][0]
print("Mean Absolute Error:", mean_absolute_error)
def predict(model, data, classification=False):
# retrieve the last sequence from data
last_sequence = data["last_sequence"][:N_STEPS]
# retrieve the column scalers
column_scaler = data["column_scaler"]
# reshape the last sequence
last_sequence = last_sequence.reshape((last_sequence.shape[1], last_sequence.shape[0]))
# expand dimension
last_sequence = np.expand_dims(last_sequence, axis=0)
# get the prediction (scaled from 0 to 1)
prediction = model.predict(last_sequence)
# get the price (by inverting the scaling)
predicted_price = column_scaler["adjclose"].inverse_transform(prediction)[0][0]
return predicted_price
# predict the future price
future_price = predict(model, data)
print(f"Future price after {LOOKUP_STEP} days is ${future_price:.2f}")
These is some more code here.
https://github.com/ASH-WICUS/Notebooks/blob/master/TensorFlow%20-%20Stock%20Price%20Prediction.ipynb

Python LSTM Bitcoin prediction flatlines

I'm currently trying to build a "simple" LSTM model that takes historical Bitcoin data, learns from that and then tries to predict the future X steps in advance.
I've build it on the idea that A + B + C = D so B + C + D should be E. (I think that's a very simple idea behind an LSTM model. I might be wrong however i'm pretty new to it.)
I managed to build the basics in python (I'm fairly new to python) but something seems off by the prediction. For some reason many of the predictions i test / make end up flatlining. I have a theory on why but we have no idea if it's correct and even less idea on how to solve it.
My theory is that within a sequence the model learns to put more importance / weight on the last digit in the sequence because with Bitcoin prices the future price (in 1 minute) is probably pretty close to the price now. That's try the predicted values keeps getting closer to the real value eventually being equal and thus flatlining in a graph. (I don't know if that makes sense but thats what i tought anyway.)
I've also added a screenshot of my graph from a few days ago. Almost all predictions however end similar to this graph. This is just a more extreme example as demonstration.
Here is my code, can someone please explain why it flatlines and what i did wrong?
import numpy as np
from matplotlib import pyplot
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
import yfinance as yf
from sklearn.preprocessing import MinMaxScaler
from math import sqrt
from sklearn.metrics import mean_squared_error
# Create output sets X + Y from given input-set
# with inputset : a 1-dimensional list of floats
# with N : the number of lookback values to use for X
# with Gap : the number of point skipped between X and Y
# Y: is equal to input, (although the first N are missing)
# X: for each y of Y a corresponding set of size N is created
# composed of the N values preceeding y.
def create_lookback(inputset, n=1, gap=0):
print("create_lookback with n=%d gap=%d" % (n,gap))
print(" - length of inputset = %d" % len(inputset))
dataX, dataY = [], []
for i in range(len(inputset) - (n+gap)):
a = inputset[i:(i + n), 0]
dataX.append(a)
dataY.append(inputset[i + n+gap, 0])
print(" - length of dataY = %d" % len(dataY))
data_x = np.array(dataX)
xret = data_x.reshape(data_x.shape[0], 1, data_x.shape[1])
return xret, np.array(dataY)
# Train model based on given training-set + Test-set
def create_model(trainX,trainY,testX,testY):
model = Sequential()
model.add(LSTM(units = 100, input_shape=(trainX.shape[1], trainX.shape[2], )))
model.add(Dropout(0.2))
#model.add(LSTM(30, return_sequences=True))
#model.add(Dropout(0.1))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
history = model.fit(trainX, trainY, epochs=100, batch_size=5, validation_data=(testX, testY), verbose=1, shuffle=False)
return model
# Evaluate given X / Y set.
# - Calculate RMSE
# - Generate visual line-plot to screen
def show_result(scaler,yhat,setY,txt):
print("Show %s result" % txt)
yhat_inverse = scaler.inverse_transform(yhat.reshape(-1, 1))
testY_inverse = scaler.inverse_transform(setY.reshape(-1, 1))
if len(testY_inverse) == len(yhat_inverse):
rmse = sqrt(mean_squared_error(testY_inverse, yhat_inverse))
print(' RMSE %s : %.3f' % (txt,rmse))
pyplot.plot(yhat_inverse, label='predict '+txt)
pyplot.plot(testY_inverse, label='actual '+txt, alpha=0.5)
pyplot.legend()
pyplot.show()
# Extrapoleer is dutch for Extrapolate
def extrapoleer(i,model,tup,toekomst):
if(i == 0):
return
setX = np.array([[tup]])
y = model.predict(setX)
y_float = y[0][0]
tup_new = np.append(tup[1:], y_float)
toekomst.append(y_float)
extrapoleer(i-1, model, tup_new,toekomst)
# --- end of defined functions
# -- start of main flow
data_grid_1 = yf.download('BTC-USD', start="2021-04-14",end="2021-04-15", interval="1m");
data_grid_2 = yf.download('BTC-USD', period="12h", interval="1m");
dataset_1 = data_grid_1.iloc[:, 1:2].values
dataset_2 = data_grid_2.iloc[:, 1:2].values
scaler = MinMaxScaler(feature_range = (0, 1))
scaled = scaler.fit_transform(dataset_1)
# 70% of dataset_1 is used to train ; 30% to test
train_size = int(len(scaled) * 0.7)
test_size = len(scaled) - train_size
train, test = scaled[0:train_size,:], scaled[train_size:len(scaled),:]
print("train: %d test: %d" % (len(train), len(test)))
scaled_2 = scaler.fit_transform(dataset_2)
look_back_n = 3
look_back_gap = 0
trainX, trainY = create_lookback(train, look_back_n, look_back_gap)
testX, testY = create_lookback(test, look_back_n, look_back_gap)
testX_2, testY_2 = create_lookback(scaled_2, look_back_n, look_back_gap)
model = create_model(trainX,trainY,testX,testY)
yhat_1 = model.predict(testX)
yhat_2 = model.predict(testX_2)
show_result(scaler,yhat_1,testY,"test")
show_result(scaler,yhat_2,testY_2,"test2")
last_n = testY_2[-look_back_n:]
#toekomst = Future in dutch
toekomst = []
#aantal = Amount in Dutch, this indicates the amount if steps you want to future predict
aantal = 30
extrapoleer(aantal, model, last_n, toekomst)
print("Resultaat van %d voorspelde punten in de toekomst: " % aantal)
print(toekomst)
yhat_2_plus = np.append(yhat_2,toekomst)
show_result(scaler,yhat_2_plus,testY_2,"test2-plus")

Keras Image Classification: Error when checking input: expected input_1 to have 4 dimensions, but got array with shape (6885, 7500)

I have seen other posts that say just add extra dimensions as it expects. Firstly, I don't know how to do that exactly, but most importantly, I want to know why my dimensions are changing so I can figure this out on my own for future models.
Note, I am restricted to only MLP for training. This means only Fully Connected layers. No Convolutional Layers nor feedbacks are allowed (LSTM or any RNN architecture). No pre-trained models are allowed such as (resnet, densenet, ...). I can use other operations in-between layers, like Dropout, Batch Normalization or other types of layer input/output manipulation. I expect I will have to supply my entire code to get the help I need. Please forgive all my comments in my code, which I have as a reminder to me as to what everything does. I do know I will need data augmentation, but need this to work first.
# **********************************************************************************************************************
# ----------------------------------------------------------------------------------------------------------------------
# Goal: Use MLP to classify blood cells into 4 different categories:
# 1. red blood cell
# 2. ring - malaria
# 3. schizont - malaria
# 4. trophozoite - malaria
# Metric used: macro-averaged F1-score and
# Cohen's Kappa score - mean of these two scores
# **********************************************************************************************************************
# import Packages
# **********************************************************************************************************************
import os
import cv2
from keras.layers import Dense, Input
from sklearn.preprocessing import LabelEncoder
# https://machinelearningmastery.com/how-to-normalize-center-and-standardize-images-with-the-imagedatagenerator-in-keras/
import random
import numpy as np
import tensorflow as tf
from keras.callbacks import ModelCheckpoint
from sklearn.model_selection import train_test_split
from sklearn.metrics import cohen_kappa_score, f1_score
import matplotlib.pyplot as plt
from keras.models import Model
from keras import optimizers
from keras.utils import to_categorical
# *************************************************************
# %% --------------------------------------- Set-Up --------------------------------------------------------------------
print('Set-up')
# np.load("x_train.npy"), np.load("y_train.npy")
# np.load("x_train.npy"), np.load("y_train.npy")
SEED = 42
os.environ['PYTHONHASHSEED'] = str(SEED)
random.seed(SEED)
np.random.seed(SEED)
tf.random.set_seed(SEED)
# **********************************************************************************************************************
# LOAD DATA
# **********************************************************************************************************************
print('Load Data')
if "train" not in os.listdir():
os.system("wget https://storage.googleapis.com/exam-deep-learning/train.zip")
os.system("unzip train.zip")
DATA_DIR = os.getcwd() + "/train/"
RESIZE_TO = 50 # 50 pixels
x, y = [], []
for path in [f for f in os.listdir(DATA_DIR) if f[-4:] == ".png"]: # for .png images in directory
x.append(cv2.resize(cv2.imread(DATA_DIR + path), (RESIZE_TO, RESIZE_TO))) # resizes image to dimensions
with open(DATA_DIR + path[:-4] + ".txt", "r") as s: # opens a .txt file of same name as image
label = s.read() # reads the file's contents
y.append(label) # appends the contents to y data set
x, y = np.array(x), np.array(y) # sets x and y to an array
le = LabelEncoder() # label encoder encodes target labels with values between 0 and n_classes-1, done to y target
le.fit(["red blood cell", "ring", "schizont", "trophozoite"]) # should have 4 values; fit label encoder
y = le.transform(y) # transforms labels to normalized encoding
print(x.shape, y.shape)
# (8607, 50, 50, 3) (8607,)
print(x)
print(y)
# **********************************************************************************************************************
# SPLIT DATA
# **********************************************************************************************************************
print('Split Data')
# This script generates the training set
# and the held out set
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=SEED, test_size=0.2, stratify=y)
np.save("x_train.npy", x_train)
np.save("y_train.npy", y_train) # saves datasets as .npy files
np.save("x_test.npy", x_test)
np.save("y_test.npy", y_test)
# summarize dataset shape
print('Train shape', x_train.shape, y_train.shape)
print('Test shape', x_test.shape, y_test.shape)
# Train shape (6885, 50, 50, 3) (6885,)
# Test shape (1722, 50, 50, 3) (1722,)
# summarize pixel values
print('Train pixels', x_train.min(), x_train.max(), x_train.mean(), x_train.std())
print('Test pixels', x_test.min(), x_test.max(), x_test.mean(), x_test.std())
# Train pixels 0 255 129.69422568869524 68.92910646179355
# Test pixels 0 255 129.9020098335269 69.49813333178977
print('Training data shape : ', x_train.shape, y_train.shape)
print('Testing data shape : ', x_test.shape, y_test.shape)
# Find the unique numbers from the train labels
classes = np.unique(y_train)
nClasses = len(classes)
print('Total number of outputs : ', nClasses)
print('Output classes : ', classes)
# %% ----------------------------------- Hyper Parameters --------------------------------------------------------------
print('Set Hyperparameters')
# a model hyperparameter is a configuration that is external to the model and whose value cannot be estimated from data
LR = 1e-3 # learning rate
N_NEURONS = 4 # number of neurons
N_EPOCHS = 50 # number of epochs - # of passes through the entire training dataset the mL algorithm has completed
BATCH_SIZE = 100 # batch size - defines the scope of our data upfront. Limits the number of samples to be shown
# to the network before a weight update can be performed.
# The same limitation is then imposed when making predictions with the fit model.
# Batch size used when fitting the model controls how many predictions you must make at a time.
# https://machinelearningmastery.com/use-different-batch-sizes-training-predicting-python-keras/
DROPOUT = 0.2 # dropout - term used for a technique which drops out some nodes of the network.
# Dropping out can be seen as temporarily deactivating or ignoring neurons of the network.
# This technique is applied in the training phase to reduce overfitting effects.
# Basic idea behind dropout is to dropout nodes so that the network can concentrate on other features.
# https://www.python-course.eu/neural_networks_with_dropout.php
# Iterations is number of batches needed to complete one epoch.
# %% -------------------------------------- Data Prep ------------------------------------------------------------------
print('Data Prep')
# load training data from the .npy files created by Numpy. loads data faster with .npy
# x_train, y_train, x_test, y_test = np.load("x_train.npy"), np.load("y_train.npy"), np.load("x_test.npy"), np.load(
# "y_test.npy")
print('x_train shape: ', x_train.shape[1:])
print('x_test shape: ', x_test.shape[1:])
print('find shape of input')
# find shape of input image then reshape it into input format for training and testing sets
nRows, nCols, nDims = x_train.shape[1:]
train_data = x_train.reshape(x_train.shape[0], nRows, nCols, nDims)
test_data = x_test.reshape(x_test.shape[0], nRows, nCols, nDims)
input_shape = (nRows, nCols, nDims)
print('train_data shape: ', train_data.shape[1:])
print('test_data shape: ', test_data.shape[1:])
# then change all datatypes into floats
train_data = train_data.astype('float32')
test_data = test_data.astype('float32')
# flatten and normalize data
print('flatten and normalize data')
train_data = train_data.reshape(len(train_data), -1)
test_data = test_data.reshape(len(test_data), -1)
train_data /= 255
test_data /= 255
train_labels_one_hot = to_categorical(y_train)
test_labels_one_hot = to_categorical(y_test)
print('Original label 0 : ', y_train[0])
print('After conversion to categorical ( one-hot ) : ', train_labels_one_hot[0])
# %%-------------------------------will want to eventually do this data augmentation--------------------------
# data augmentation later
# %% -------------------------------------- Training Prep ----------------------------------------------------------
print('Training Prep')
print('Create Model')
# this returns a tensor
inputs = Input(shape=input_shape)
output_1 = Dense(N_NEURONS, activation="relu")(inputs)
output_2 = Dense(N_NEURONS, activation="relu")(output_1)
predictions = Dense(nClasses, activation="softmax")(output_2)
# This creates a model that includes the Input layer and three Dense layers
model = Model(inputs=inputs, outputs = predictions)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Before training a model, you need to configure the learning process, which is done via the compile method.
# It receives three arguments:
# 1. optimizer
# 2. loss function - objective that the model will try to minimize
# 3. list of metrics - for classification problem, set to metrics=['accuracy']
model.compile(optimizers.Adam(lr=LR), loss="categorical_crossentropy", metrics=["accuracy"])
model.summary()
# %% -------------------------------------- Training Loop ----------------------------------------------------------
print('training Loop')
model.fit(
train_data,
train_labels_one_hot,
batch_size=BATCH_SIZE,
epochs=N_EPOCHS,
validation_data=(test_data, test_labels_one_hot),
callbacks=[ModelCheckpoint("mlp_user1.hdf5", monitor="val_loss", save_best_only=True)])
# modelcheckpoint sets check-point in the learning model during training in Python using the Keras library
# a snapshot of the state of the system is taken in case of system failure. It may be used directly, or as the
# starting point for a new run, picking up where it left off.
# The checkpoint is the weights of the model. These weights can be used to make predictions as is,
# or used as the basis for ongoing training.
# https://machinelearningmastery.com/check-point-deep-learning-models-keras/
# %% ------------------------------------------ Final test -------------------------------------------------------------
print('Final Test')
print("Final accuracy on validations set:", 100 * model.evaluate(x_test, y_test)[1], "%")
print("Cohen Kappa", cohen_kappa_score(np.argmax(model.predict(x_test), axis=1), np.argmax(y_test, axis=1)))
print("F1 score", f1_score(np.argmax(model.predict(x_test), axis=1), np.argmax(y_test, axis=1), average='macro'))

The problem here is that you flatten out for normalization and forgot to reshape it to the same old shape (this line train_data = train_data.reshape(len(train_data), -1),test_data = test_data.reshape(len(test_data), -1)) which is you flatten out all dimension except first dimension and then you use its old dimension (before you flatten out) as an input dimension (input_shape = (nRows, nCols, nDims), inputs = Input(shape=input_shape))