I am trying to generate price predictions for cryptocurrencies using keras. I have it working on 1 day intervals using 1 day data from the Kraken API. My problem comes when I try to generate predictions on other time frames. 1hr 4hr and 1week. The prediction timestamps come out as daily regardless of the input data. For hourly the output dataframe is completely scrambled. These are screenshots of the output.This is correctly working
This is the scrambled 1 hour dataframe
This is the 4hr dataframe with wrong timestamps
My code is as follows. Sorry it's not a minimum reproducible example.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense, Dropout, GRU, SimpleRNN, Conv1D
from tensorflow.keras.optimizers import Adam
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler, Normalizer
import os
import tensorflow as tf
import krakenex
from pykrakenapi import KrakenAPI
#Pulling a DataFrame from the API
api = krakenex.API()
k = KrakenAPI(api)
ohlc, last = k.get_ohlc_data("ETHUSD", interval=60)
df = ohlc
last_close = ohlc
last_close = last_close[['close']]
last_close = last_close.iloc[::-1]
last_price = last_close.close[-1]
print(last_price)
#Clipping Dataframe to only containg index anc close prices
df = df[['close']]
#Reversing dataframe order
df = df.iloc[::-1]
df.tail()
#LSTM works better if data is scaled but removing for a moment
scaler = MinMaxScaler()
df = pd.DataFrame(scaler.fit_transform(df), columns = df.columns, index=df.index)
#print(df.columns)
df.tail()
#Visualizing price action
df.plot(figsize=(14,8))
plt.title('ETH prices')
plt.ylabel('normalized prices')
plt.show()
def split_sequence(seq, n_steps_in, n_steps_out):
"""
Splits the unzivariate time sequence
"""
# Creating a list for both variables
X, y = [], []
for i in range(len(seq)):
# Finding the end of the current sequence
end = i + n_steps_in
out_end = end + n_steps_out
# Breaking out of the loop if we have exceeded the dataset's length
if out_end > len(seq):
break
# Splitting the sequences into: x = past prices and indicators, y = prices ahead
seq_x, seq_y = seq[i:end], seq[end:out_end]
X.append(seq_x)
y.append(seq_y)
return np.array(X), np.array(y)
#Some charting for accuracy tests
def visualize_results(results):
history = results.history
plt.figure(figsize=(14,8))
plt.plot(history['val_loss'])
plt.plot(history['loss'])
plt.legend(['val_loss', 'loss'])
plt.title('Loss')
plt.xlabel('epochs')
plt.ylabel('loss')
plt.show()
plt.figure(figsize=(14,8))
plt.plot(history['val_accuracy'])
plt.plot(history['accuracy'])
plt.legend(['val_accuracy', 'accuracy'])
plt.title('Accuracy')
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.show()
#number of periods to look back at
n_per_in = 680
#number of periods to predict
n_per_out = 20
n_features = 1
#making a sequence for predictions
X, y = split_sequence(list(df.close), n_per_in, n_per_out)
#Giving the model the proper dimensions
X = X.reshape(X.shape[0], X.shape[1], n_features)
# Instantiating the model
d = 0.2
activ = 'softsign'
model = Sequential()
#model.add(Conv1D(100, kernel_size=2, input_shape=(n_per_in, n_features), activation=activ))
model.add(LSTM(60, input_shape=(n_per_in, n_features), activation=activ))
model.add(Dense(units=n_per_out))
# Model summary
model.summary()
model.compile(loss = "mse", optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), metrics=['accuracy'])
res = model.fit(X,y, epochs=10, batch_size=4, validation_split=0.2)
plt.figure(figsize=(14, 8))
yhat = model.predict(X[-1].reshape(1,n_per_in,n_features)).tolist()[0]
yhat = np.array(yhat).reshape(-1,1).tolist()
actual = y[-1].reshape(-1,1)
print("predicted ", yhat)
plt.plot(yhat, label='predicted')
print('actuals ', actual.tolist())
plt.plot(actual.tolist(), label='actual')
plt.title("Predicted vs Actual")
plt.ylabel('price')
plt.legend()
plt.show()
yhat = model.predict(np.array(df.head(n_per_in)).reshape(1, n_per_in, n_features)).tolist()[0]
yhat = np.array(yhat).reshape(-1,1).tolist()
preds = pd.DataFrame(yhat, index=pd.date_range(start=df.index[-1], periods=len(yhat)), columns=df.columns)
print(preds)
periods = 30
actual = pd.DataFrame(df[['close']].tail(periods), index = df.close.tail(periods).index, columns= df.columns).append(preds.head(1))
plt.figure(figsize=(14,8))
plt.plot(actual, label='actuals')
plt.plot(preds, label='predictions')
plt.ylabel("price")
plt.xlabel('times')
plt.title(f'Forecasting the next {len(yhat)} days')
plt.legend()
plt.show()
How can I get the correct time frames in my output? I've gone to the docs and scoured google for hours and can't come up with a solution other than switching to prophet rather than Keras, and I don't want to do that.
Related
I've made a predictive model using LSTM which predicts future prices for raw materials like cotton,fibre,yarn etc. At the end of code I used matplotlib library to plot graph which displays the original prices, predicted prices and future predicted prices.
This is the graph which shows future prices according to dates
How do I display this graph on Django framework? Because I need to deploy this model on a web application using Django but the tutorials I've seen so far show predictive models which take user input and don't really show anything related to plots or graphs.
Following is the code:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime as dt
from datetime import datetime
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard
import os
import glob
import pandas
import numpy
from sklearn import preprocessing
import numpy as np
# Importing Training Set
dataset_train = pd.read_csv('201222-yarn-market-price-china--034.csv1.csv')
dataset_train.info()
# Select features (columns) to be involved intro training and predictions
cols = list(dataset_train)[1:5]
# Extract dates (will be used in visualization)
datelist_train = list(dataset_train.iloc[0])
datelist_train = [dt.datetime.strptime(date, '%m/%d/%Y').date() for date in datelist_train]
print('Training set shape == {}'.format(dataset_train.shape))
print('All timestamps == {}'.format(len(datelist_train)))
print('Featured selected: {}'.format(cols))
dataset_train = dataset_train[cols].astype(str)
for i in cols:
for j in range(0, len(dataset_train)):
dataset_train[i][j] = dataset_train[i][j].replace(',', '')
dataset_train = dataset_train.astype(float)
# Using multiple features (predictors)
training_set = dataset_train.values
print('Shape of training set == {}.'.format(training_set.shape))
training_set
# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
training_set_scaled = sc.fit_transform(training_set)
sc_predict = StandardScaler()
sc_predict.fit_transform(training_set[:, 0:1])
# Creating a data structure with 90 timestamps and 1 output
X_train = []
y_train = []
n_future = 60 # Number of days we want top predict into the future
n_past = 90 # Number of past days we want to use to predict the future
for i in range(n_past, len(training_set_scaled) - n_future +1):
X_train.append(training_set_scaled[i - n_past:i, 0:dataset_train.shape[1] - 1])
y_train.append(training_set_scaled[i + n_future - 1:i + n_future, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
print('X_train shape == {}.'.format(X_train.shape))
print('y_train shape == {}.'.format(y_train.shape))
# Import Libraries and packages from Keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from tensorflow.keras.optimizers import Adam
# Initializing the Neural Network based on LSTM
model = Sequential()
# Adding 1st LSTM layer
model.add(LSTM(units=64, return_sequences=True, input_shape=(n_past, dataset_train.shape[1]-1)))
# Adding 2nd LSTM layer
model.add(LSTM(units=10, return_sequences=False))
# Adding Dropout
model.add(Dropout(0.25))
# Output layer
model.add(Dense(units=1, activation='linear'))
# Compiling the Neural Network
model.compile(optimizer = Adam(learning_rate=0.01), loss='mean_squared_error')
es = EarlyStopping(monitor='val_loss', min_delta=1e-10, patience=10, verbose=1)
rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, verbose=1)
mcp = ModelCheckpoint(filepath='weights.h5', monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True)
tb = TensorBoard('logs')
history = model.fit(X_train, y_train, shuffle=True, epochs=30, callbacks=[es, rlr, mcp, tb],
validation_split=0.2, verbose=1, batch_size=256)
# Generate list of sequence of days for predictions
datelist_future = pd.date_range(datelist_train[-1], periods=n_future, freq='1d').tolist()
'''
Remeber, we have datelist_train from begining.
'''
# Convert Pandas Timestamp to Datetime object (for transformation) --> FUTURE
datelist_future_ = []
for this_timestamp in datelist_future:
datelist_future_.append(this_timestamp.date())
# Perform predictions
predictions_future = model.predict(X_train[-n_future:])
predictions_train = model.predict(X_train[n_past:])
# Inverse the predictions to original measurements
# ---> Special function: convert <datetime.date> to <Timestamp>
def datetime_to_timestamp(x):
'''
x : a given datetime value (datetime.date)
'''
return datetime.strptime(x.strftime('%m%d%Y'), '%m%d%Y')
y_pred_future = sc_predict.inverse_transform(predictions_future)
y_pred_train = sc_predict.inverse_transform(predictions_train)
a=dataset_train.iloc[:, 3]
print(a)
PREDICTIONS_FUTURE = pd.DataFrame(y_pred_future, columns=['Cotton
Yarn1']).set_index(pd.Series(datelist_future))
PREDICTION_TRAIN = pd.DataFrame(y_pred_train, columns=['Cotton
Yarn1']).set_index(pd.Series(datelist_train[2 * n_past + n_future -1:]))
# Convert <datetime.date> to <Timestamp> for PREDCITION_TRAIN
PREDICTION_TRAIN.index = PREDICTION_TRAIN.index.to_series().apply(datetime_to_timestamp)
print(PREDICTION_TRAIN.head(3))
#plt.rcParams["figure.figsize"] = (20,3)
#rcParams['figure.figsize'] = 14, 5
# Plot parameters
START_DATE_FOR_PLOTTING = '12/24/2019'
dataset_train = pd.DataFrame(dataset_train, columns=cols)
dataset_train.index = datelist_train
dataset_train.index = pd.to_datetime(dataset_train.index)
plt.plot(PREDICTIONS_FUTURE.index, PREDICTIONS_FUTURE['Cotton Yarn1'], color='r',
label='Predicted Stock Price')
plt.plot(PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:].index,
PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:]['Cotton Yarn1'], color='orange',
label='Training predictions')
plt.plot(dataset_train.loc[START_DATE_FOR_PLOTTING:].index,
dataset_train.loc[START_DATE_FOR_PLOTTING:]['Cotton Yarn1'], color='b', label='Actual Stock
Price')
plt.axvline(x = min(PREDICTIONS_FUTURE.index), color='green', linewidth=2, linestyle='--')
plt.grid(which='major', color='#cccccc', alpha=0.5)
plt.legend(shadow=True)
plt.title('Predcitions and Acutal Stock Prices', family='Arial', fontsize=12)
plt.xlabel('Timeline', family='Arial', fontsize=10)
plt.ylabel('Stock Price Value', family='Arial', fontsize=10)
plt.xticks(rotation=45, fontsize=8)
plt.show()
I've just recently started dabbling with Keras and I'm having a hell of a time trying to understand how to format my data to be used by Keras.
I'm currently trying to use a TimeDistributed ConvLSTM1D layer to predict stock prices. I've used a walk-forward approach to generate my dataset.
In creating the dataset, each data point comprises of 21 features for 5 days. I've arranged these data points into sequences of 10. The shapes for my test data and labels are as follows:
X.shape = (3467, 10, 5, 21)
Y.shape = (3467, 10)
From my understanding of a TimeDistributed(ConvLSTM1D) layer, I would need the following shape: (samples, time, rows, channels)
I assume samples should be 10 and time should be 5. I'm not clear on how I should reshape my array to account for rows and channels. Do I have 21 rows of 1 channel, or 1 row of 21 channels? Does it even matter?
Below is my code:
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
import tensorflow as tf
import sys
import warnings
from datetime import datetime, timedelta
warnings.simplefilter(action='ignore', category=FutureWarning)
from sklearn.model_selection import train_test_split
from keras.models import Sequential, model_from_json
from keras.layers import Conv2D, Conv1D, MaxPooling1D, Bidirectional,LSTM,Dropout,TimeDistributed,Flatten, ConvLSTM1D, ConvLSTM2D
from keras.layers import Dense, RepeatVector, LeakyReLU
from keras.layers import BatchNormalization
from keras.callbacks import TensorBoard
from talib import DEMA, ROCR, ATR, RSI, SMA, ADX, CCI, MACD, OBV, SAR, EMA, T3
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from matplotlib.ticker import AutoMinorLocator, FixedLocator
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif, f_regression
from sklearn import preprocessing
os.environ['CUDA_VISIBLE_DEVICES'] = '-1' #stop using GPU since there's a problem
def setup_TA(dataframe):
#Add these indicators
dataframe['DEMA']=DEMA(dataframe['Close'])
dataframe['ROCR']=ROCR(dataframe['Close'])
dataframe['ATR']=ATR(dataframe['High'], dataframe['Low'], dataframe['Close'])
dataframe['RSI']=RSI(dataframe['Close'])
dataframe['SMA']=SMA(dataframe['Close'])
dataframe['ADX']=ADX(dataframe['High'], dataframe['Low'], dataframe['Close'])
dataframe['CCI']=CCI(dataframe['High'], dataframe['Low'], dataframe['Close'])
dataframe['MACD'], dataframe['MACD Signal'], dataframe['MACD Hist'] = MACD(dataframe['Close'])
dataframe['OBV']=OBV(dataframe['Close'],dataframe['Volume'])
dataframe['SAR']=SAR(dataframe['High'], dataframe['Low'])
dataframe['EMA']=EMA(dataframe['Close'])
dataframe['T3']=T3(dataframe['Close'])
# Lets make everything relative to the close price as far as we can.
#Reduce these scales to between 0-1
dataframe['RSI'] = dataframe['RSI'] / 100
dataframe['CCI'] = dataframe['CCI'] / 100
dataframe['ADX'] = dataframe['ADX'] / 100
dataframe['SAR'] = dataframe['SAR'] / dataframe['Close']
dataframe['ATR'] = dataframe['ATR'] / dataframe['Close']
dataframe['OBV'] = dataframe['OBV'] / dataframe['Volume']
dataframe['Open'] = dataframe['Open'] / dataframe['Close']
dataframe['High'] = dataframe['High'] / dataframe['Close']
dataframe['Low'] = dataframe['Low'] / dataframe['Close']
dataframe['DEMA'] = dataframe['DEMA'] / dataframe['Close']
dataframe['SMA'] = dataframe['SMA'] / dataframe['Close']
dataframe['EMA'] = dataframe['EMA'] / dataframe['Close']
dataframe['T3'] = dataframe['T3'] / dataframe['Close']
dataframe['ROCR'] = dataframe['ROCR'] - 1
#Remove any rows with NaN
rows_with_nan=[]
for index, rows in dataframe.iterrows():
if rows.isnull().any():
rows_with_nan.append(index)
dataframe.drop(rows_with_nan,inplace=True)
dataframe=dataframe.reset_index(drop=True)
return dataframe
def feature_selection(dataframe):
selector = SelectKBest(score_func=f_regression, k='all')
selector.fit_transform(dataframe.drop(['Labels'], 1).to_numpy(), dataframe['Labels'].to_numpy())
sorted = (selector.scores_).argsort()[:]
print(sorted)
print(dataframe.columns[sorted])
print(selector.scores_)
print(list(dataframe))
#Open, High, Low, Close, Volumen, compound, msg count, DEMA, ROCR, ATR, RSI, SMA, ADX, CCI, MACD, MACD Signal, MACD Hist, OBV, SAR, EMA, T3
raw_data = pd.read_csv('./Test/test.csv', index_col=0)
ta_df = setup_TA(raw_data)
ta_df['msg count'] +=1
ta_df['Shifted Close'] = ta_df['Close'].shift(1)
ta_df['Shifted Volume'] = ta_df['Volume'].shift(1)
ta_df['Shifted msg count'] = ta_df['msg count'].shift(1)
ta_df['Close'] = np.log(ta_df['Close']/ta_df['Shifted Close'])
ta_df['Volume'] = np.log(ta_df['Volume']/ta_df['Shifted Volume'])
ta_df['msg count'] = np.log(ta_df['msg count']/ta_df['Shifted msg count'])
ta_df.drop(['Shifted Close', 'Shifted Volume', 'Shifted msg count'], 1, inplace=True)
ta_df = ta_df.drop(0,0).reset_index(drop=True)
ta_df['Close'] = (ta_df['Close'] -ta_df['Close'].mean())/ta_df['Close'].std()
ta_df['Volume'] = (ta_df['Volume'] - ta_df['Volume'].mean())/ta_df['Volume'].std()
ta_df['compound'] = (ta_df['compound'] - ta_df['compound'].mean())/ta_df['compound'].std()
ta_df['msg count'] = (ta_df['msg count'] - ta_df['msg count'].mean())/ta_df['msg count'].std()
ta_df['OBV'] = (ta_df['OBV'] - ta_df['OBV'].mean())/ta_df['OBV'].std()
ta_df['Labels'] = ta_df['Close'].shift(-1)
ta_df=ta_df[:-1]
clean_df = ta_df.drop(['Date','Labels'],1)
#create samples(5 day window, 10 previous weeks?)
X=[]
Y=[]
for k in range(len(clean_df)-10-5):
temp_x=[]
temp_y=[]
for i in range(10):
start_index=k+i
end_index = start_index + 4
temp2_x=[]
for j in range(start_index, end_index+1):
temp2_x.append(clean_df.xs(j))
temp_x.append(temp2_x)
temp_y.append(ta_df.iloc[end_index,ta_df.columns.get_loc('Labels')])
X.append(temp_x)
Y.append(temp_y)
pd.set_option('display.max_rows',None)
X=np.array(X)
Y=np.array(Y)
#X = X.reshape(X.shape[0],X.shape[1],X.shape[2],1,X.shape[3])
#Y = Y.reshape(Y.shape[0],Y.shape[1],1)
print(X.shape)
print(Y.shape)
train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.2, shuffle=False)
model = Sequential()
model.add(TimeDistributed(ConvLSTM1D(filters = 21, kernel_size=(3), padding='same', return_sequences=True, input_shape=(10,5,21,1))))
model.add(TimeDistributed(ConvLSTM1D(filters = 21, kernel_size=(3), padding='same', return_sequences=False)))
model.add(TimeDistributed((Flatten())))
model.add(TimeDistributed(Dense(21)))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam', metrics = ['mse'])
history = model.fit(train_X, train_Y, validation_data=(test_X, test_Y), epochs = 100, batch_size = 64, shuffle=False, verbose=1)
I have followed this tutorial https://www.youtube.com/watch?v=QIUxPv5PJOY to predict the stock price of Apple one day into the future. The code is:
#Import the libraries
import math
import pandas_datareader as web
import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense, LSTM
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
#Get the stock quote
df = web.DataReader('AAPL', data_source='yahoo', start='2012-01-01', end='2020-12-07')
#Show the data
df
#Get the number of rows and columns in the data set
df.shape
#Visualize the closing price history
#We create a plot with name 'Close Price History'
plt.figure(figsize=(16,8))
plt.title('Close Price History')
#We give the plot the data (the closing price of our stock)
plt.plot(df['Close'])
#We label the axis
plt.xlabel('Date', fontsize=18)
plt.ylabel('Close Price USD ($)', fontsize=18)
#We show the plot
plt.show()
#Create a new dataframe with only the 'Close' column
data = df.filter(['Close'])
#Convert the dataframe to a numpy array
dataset = data.values
#Get the number of rows to train the model on
training_data_len = math.ceil( len(dataset) * 0.8 )
training_data_len
#Scale the data
scaler = MinMaxScaler(feature_range=(0,1))
scaled_data = scaler.fit_transform(dataset)
scaled_data
#Create the training data set
#Create the scaled training data set
train_data = scaled_data[0:training_data_len, :]
#Split the data into x_train and y_train data sets
x_train = []
y_train = []
#We create a loop
for i in range(60, len(train_data)):
x_train.append(train_data[i-60:i, 0]) #Will conaint 60 values (0-59)
y_train.append(train_data[i, 0]) #Will contain the 61th value (60)
if i <= 60:
print(x_train)
print(y_train)
print()
#Convert the x_train and y_train to numpy arrays
x_train, y_train = np.array(x_train), np.array(y_train)
#Reshape the data
x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
x_train.shape
#Build the LSTM model
model = Sequential()
model.add(LSTM(50, return_sequences=True, input_shape=(x_train.shape[1], 1)))
model.add(LSTM(50, return_sequences=False))
model.add(Dense(25))
model.add(Dense(1))
#Compile the model
model.compile(optimizer='adam', loss='mean_squared_error')
#Train the model
model.fit(x_train, y_train, batch_size=1, epochs=1)
#Create the testing data set
#Create a new array containing scaled values from index 1738 to 2247
test_data = scaled_data[training_data_len - 60:]
#Create the data set x_test and y_test
x_test = []
y_test = dataset[training_data_len:, :]
for i in range(60, len(test_data)):
x_test.append(test_data[i-60:i, 0])
#Convert the data to a numpy array
x_test = np.array(x_test)
#Reshape the data
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
#Get the model's predicted price values for the x_test data set
predictions = model.predict(x_test)
predictions = scaler.inverse_transform(predictions)
predictions
#Evaluate model (get the root mean quared error (RMSE))
rmse = np.sqrt( np.mean( predictions - y_test )**2 )
rmse
#Plot the data
train = data[:training_data_len]
valid = data[training_data_len:]
valid['Predictions'] = predictions
#Visualize the data
plt.figure(figsize=(16,8))
plt.title('Model')
plt.xlabel('Date', fontsize=18)
plt.ylabel('Close Price USD ($)', fontsize=18)
plt.plot(train['Close'])
plt.plot(valid[['Close', 'Predictions']])
plt.legend(['Train', 'Validation', 'Predictions'], loc='lower right')
plt.show()
Now I want to extend the predicted portion of the graph to show future dates as well ('x' days into the future). I think I could do it by getting the predicted price for the next day and then use that price in the input to get the next day, and then use that day to get the next day, and so on. How can I do it? I thought of appending the next day pred price to the dataset used to train the model, but I wasn't successful at this. Thank you for your help.
Your intuition is correct. I have done what you were thinking of in this way:
X_FUTURE = 100
predictions = np.array([])
last = x_test[-1]
for i in range(X_FUTURE):
curr_prediction = model.predict(np.array([last]))
print(curr_prediction)
last = np.concatenate([last[1:], curr_prediction])
predictions = np.concatenate([predictions, curr_prediction[0]])
predictions = scaler.inverse_transform([predictions])[0]
print(predictions)
I have basically constructed shifting arrays with the new predictions
After that I have constructed the dataframe that contains the new prediction:
import datetime
from datetime import timedelta
dicts = []
curr_date = data.index[-1]
for i in range(X_FUTURE):
curr_date = curr_date + timedelta(days=1)
dicts.append({'Predictions':predictions[i], "Date": curr_date})
new_data = pd.DataFrame(dicts).set_index("Date")
And I plotted the result:
#Plot the data
train = data
#Visualize the data
plt.figure(figsize=(16,8))
plt.title('Model')
plt.xlabel('Date', fontsize=18)
plt.ylabel('Close Price USD ($)', fontsize=18)
plt.plot(train['Close'])
plt.plot(new_data['Predictions'])
plt.legend(['Train', 'Predictions'], loc='lower right')
plt.show()
Why it seems so bad (anyway we don't know the future...)? I did retrain the model on all the dataset, but the problem here is that the further I go the greater would be the uncertain. I am not too expert of time series prediction, but I think that the model has not learned any good pattern under the time series. But as example it does what it needs to do
I wrote a small
"Linear Regression Neural Network Tensorflow Keras Python program"
Input dataset is
y = mx + c straight line data.
Predicted y values are not correct and are giving horizontal line kind of
values, instead of a line with some slope.
I ran this program on Windows laptop with tensorflow, Keras and
Jupyter notebook.
What to do to fix this program please?
Thanks and best regards,
SSJ
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
n2 = 50
count = 20
n4 = n2 + count
p = 100
m = 10
c = 5
x = np.linspace(n2, n4, p)
y = m * x + c
x
y
plt.scatter(x,y)
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers.experimental import preprocessing
x_normalizer = preprocessing.Normalization(input_shape=[1,])
x_normalizer.adapt(x)
x_normalized = x_normalizer(x)
y_normalizer = preprocessing.Normalization(input_shape=[1,])
y_normalizer.adapt(y)
y_normalized = x_normalizer(y)
y_model = tf.keras.Sequential([
y_normalizer,
layers.Dense(1)
])
y_model.compile(optimizer='rmsprop', loss='mse', metrics = ['mae'])
y_hist = y_model.fit(x, y, epochs=100, verbose=0, validation_split = 0.2)
hist = pd.DataFrame(y_hist.history)
hist['epoch'] = y_hist.epoch
hist.head()
hist.tail()
xin = [51,53,59,64]
ypred = y_model.predict(xin)
ypred
plt.scatter(x, y)
plt.scatter(xin, ypred, color = 'r')
plt.grid(linestyle = '--')
Use StandardScaler instead of Normalization
Normalizer acts row-wise and StandardScaler column-wise.
Normalizer does not remove the mean and scale by deviation but scales
the whole row to unit norm.
Found here: Difference between StandardScaler and Normalizer
This is how you can process the data:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from sklearn.preprocessing import StandardScaler
x = np.linspace(50, 70, 100).reshape(-1, 1)
y = 10 * x + 5
x_standard_scaler = StandardScaler().fit(x)
y_standard_scaler = StandardScaler().fit(y)
x_scaled = x_standard_scaler.transform(x)
y_scaled = y_standard_scaler.transform(y)
Remember that you need two separate scalers for x and y so don't use the same object for that. Also if you want to use that scaler to process new data for testing, save the scaler in some variable. A good practice is to not refit the scaler again on test data.
model = Sequential([
Dense(1, input_dim=1, activation='linear'),
])
model.compile(optimizer='rmsprop', loss='mse')
history = model.fit(x_scaled, y_scaled, epochs=1000, verbose=0, validation_split = 0.2).history
pd.DataFrame(history).plot()
plt.show()
As you can see the model is converging. Its worth to plot the loss history which helps to tell if your model is learning or not.
x_test = np.linspace(20, 100, 10).reshape(-1, 1)
y_test = 10 * x_test + 5
x_test_scaled = x_standard_scaler.transform(x_test)
y_test_scaled = y_standard_scaler.transform(y_test)
If you have a test data that you want to use for validation or just predict it, remember to use standard scaler again, but without fitting. It should be fitted on train data only in most cases.
y_test_pred_scaled = model.predict(x_test_scaled)
y_test_pred = y_standard_scaler.inverse_transform(y_test_pred_scaled)
plt.scatter(x_test, y_test, s=30, label='true')
plt.scatter(x_test, y_test_pred, s=15, label='pred')
plt.legend()
plt.show()
If you want to get your prediction rescaled back to its original range use inverse_transform. Notice that prediction on x_test after rescaling is very close to y_test.
I have trained a neural network with an LSTM model using a time series dataset.
I am using this dataset which records data from 1970-2016 of the daily rainfall of 35 locations : https://www.kaggle.com/redikod/historical-rainfall-data-in-bangladesh
It looks like this :
StationIndex Station Year Month Day Rainfall dayofyear
1970-01-01 1 Dhaka 1970 1 1 0 1
1970-01-02 1 Dhaka 1970 1 2 0 2
1970-01-03 1 Dhaka 1970 1 3 0 3
1970-01-04 1 Dhaka 1970 1 4 0 4
1970-01-05 1 Dhaka 1970 1 5 0 5
I have completed the training using the train and test data. Then I checked the predicted value against true value.
Here is the complete code, sorry if it is messy. I have put comments for each section:
import numpy as np
from pandas.plotting import register_matplotlib_converters
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rc
from pylab import rcParams
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from keras.layers import (
Input,
Dense,
LSTM,
AveragePooling1D,
TimeDistributed,
Flatten,
Bidirectional,
Dropout
)
from keras.models import Model
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
tf.keras.backend.clear_session()
register_matplotlib_converters()
sns.set(style='whitegrid', palette='muted', font_scale=1.5)
rcParams['figure.figsize'] = 22, 10
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)
#reading from CSV
df = pd.read_csv("\customized_daily_rainfall_data_Copy.csv")
#droping bad data
df = df[df.Rainfall != -999]
#droping dates (leapyear, wrong day numbers of month)
df.drop(df[(df['Day']>28) & (df['Month']==2) & (df['Year']%4!=0)].index,inplace=True)
df.drop(df[(df['Day']>29) & (df['Month']==2) & (df['Year']%4==0)].index,inplace=True)
df.drop(df[(df['Day']>30) & ((df['Month']==4)|(df['Month']==6)|(df['Month']==9)|(df['Month']==11))].index,inplace=True)
#date parcing (Index)
date = [str(y)+'-'+str(m)+'-'+str(d) for y, m, d in zip(df.Year, df.Month, df.Day)]
df.index = pd.to_datetime(date)
df['Date'] = df.index
df['Dayofyear']=df['Date'].dt.dayofyear
df.drop('Date',axis=1,inplace=True)
df.drop(['Station'],axis=1,inplace=True)
df.head()
#limiting the dataframe to just rows where StationIndex is 11
datarange = df.loc[df['StationIndex'] == 11]
#splitting train and test set
train_size = int(len(datarange) * 0.9)
test_size = len(datarange) - train_size
train, test = df.iloc[0:train_size], df.iloc[train_size:len(datarange)]
#Scaling the feature and label columns of the dataset
from sklearn.preprocessing import RobustScaler
f_columns = ['Year', 'Month','Day','Dayofyear']
f_transformer = RobustScaler()
l_transformer = RobustScaler()
f_transformer = f_transformer.fit(train[f_columns].to_numpy())
l_transformer = l_transformer.fit(train[['Rainfall']])
train.loc[:, f_columns] = f_transformer.transform(train[f_columns].to_numpy())
train['Rainfall'] = l_transformer.transform(train[['Rainfall']])
test.loc[:, f_columns] = f_transformer.transform(test[f_columns].to_numpy())
test['Rainfall'] = l_transformer.transform(test[['Rainfall']])
#making smaller train and test sections withing the dataset
def create_dataset(X, y, time_steps=1):
Xs, ys = [], []
for i in range(len(X) - time_steps):
v = X.iloc[i:(i + time_steps)].to_numpy()
Xs.append(v)
ys.append(y.iloc[i + time_steps])
return np.array(Xs), np.array(ys)
time_steps = 7
# reshape to [samples, time_steps, n_features]
X_train, y_train = create_dataset(train, train.Rainfall, time_steps)
X_test, y_test = create_dataset(test, test.Rainfall, time_steps)
#testing
X_test[0][0]
#model code
model = keras.Sequential()
#3 biderectional LSTM layers
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128, input_shape=(X_train.shape[1], X_train.shape[2]), return_sequences = True)))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128, return_sequences = True)))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128 )))
model.add(keras.layers.Dropout(rate=0.1))
model.add(keras.layers.Dense(units=1))
model.compile(loss="mean_squared_error", optimizer="RMSprop")
#training the model
history = model.fit(
X_train, y_train,
epochs=500,
batch_size=1052,
validation_split=0.2,
shuffle=False,
)
#saving the model
from tensorflow.keras.models import load_model
model.save("\Timeseries-timestep7-batchsize1052.h5")
#Using text dataset to do a prediction
y_pred = model.predict(X_test)
#inverst transformation
y_train_inv = l_transformer.inverse_transform(y_train.reshape(1, -1))
y_test_inv = l_transformer.inverse_transform(y_test.reshape(1, -1))
y_pred_inv = l_transformer.inverse_transform(y_pred)
#score
from sklearn import metrics
score = np.sqrt(metrics.mean_squared_error(y_pred,y_test))
print(score)
What I want to do next, is for the predictions to go beyond the data in the dataset.
For example, using the model that I have trained to predict data of the future. Or possibly a random/custom range.
Suppose I want to predict the day by day rainfall data of 2017. Or get the predicted data of 25-02-2017. Or maybe the data of X number of days after the end of the dataset.
Is there a good intuitive way to do that?
Thank you in advanced to whoever can answer this question. It's been bugging me for a few days now.
The code shown below should serve your purpose.
class WindowGenerator():
def __init__(self, input_width, label_width, shift,
train_df=train_df, val_df=val_df, test_df=test_df,
label_columns=None):
# Store the raw data.
self.train_df = train_df
self.val_df = val_df
self.test_df = test_df
# Work out the label column indices.
self.label_columns = label_columns
if label_columns is not None:
self.label_columns_indices = {name: i for i, name in
enumerate(label_columns)}
self.column_indices = {name: i for i, name in
enumerate(train_df.columns)}
# Work out the window parameters.
self.input_width = input_width
self.label_width = label_width
self.shift = shift
self.total_window_size = input_width + shift
self.input_slice = slice(0, input_width)
self.input_indices = np.arange(self.total_window_size)[self.input_slice]
self.label_start = self.total_window_size - self.label_width
self.labels_slice = slice(self.label_start, None)
self.label_indices = np.arange(self.total_window_size)[self.labels_slice]
def __repr__(self):
return '\n'.join([
f'Total window size: {self.total_window_size}',
f'Input indices: {self.input_indices}',
f'Label indices: {self.label_indices}',
f'Label column name(s): {self.label_columns}'])
For example, if your Data is from the years, 1960 (Jan 1st) to 2016 (Dec 31st) and that you want to predict the weather of the entire February Month for the year 2017, by considering a Data Window of past 2 Years, the values of the arguments of the above class are shown below:
input_width: 2 Years => 365 * 2 = 730
label_width: Entire Feb Month => 28
shift: We are not predicting from Jan 1st 2017 but are shifting by entire Month of Jan => 30
train_df, test_df, val_df => Self Explanatory
label_columns : Name of the Target Column
For more information, please refer this Tensorflow Tutorial on Time Series Analysis.