Develop Reference

ValueError: The target structure is of type `<class 'NoneType'>` None However the input structure is a sequence (<class 'list'>) of length 0

I am trying to deploy classification model for fall and no fall using accelerometer data in raspberry pi 4. However something seems wrong with either tensorflow or numpy when reshaping them.
Exact error that I am getting is
ValueError: The target structure is of type <class 'NoneType'>
None
However the input structure is a sequence (<class 'list'>) of length 0.
[]
nest cannot guarantee that it is safe to map one to the other.
Deployment code can be found here.
import numpy as np
import pandas as pd
from tensorflow.keras.models import load_model
import board
import scipy.stats as stats
import adafruit_mpu6050
from math import atan2,degrees,pow
import warnings
warnings.filterwarnings("ignore")
import pandas as pd
import numpy as np
print("Inititating")
model_trained = load_model('model_cnn.h5')
print('\n',model_trained.summary())
i2c = board.I2C()
mpu = adafruit_mpu6050.MPU6050(i2c)
mpu_accelerometer_range = adafruit_mpu6050.Range.RANGE_4_G
data1 = []
i = 0
while(i<2000):
i+=1
ax,ay,az = mpu.acceleration
data1.append([ax,ay,az])
df = pd.DataFrame(data1)
#print('length of df',len(df))
columns = ['x','y','z']
df.columns = columns
#print(df)
Fs = 50
frame_size = Fs*4
hop_size = Fs*2
frames = []
N_FEATURES = 3
for i in range(0,len(df)-frame_size,hop_size):
x = df['x'].values[i:i+frame_size]
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
frames.append([x,y,z])
#print('leeeee',len(frames))
# converting frames to numpy array
frames = np.asarray(frames).reshape(-1,frame_size,N_FEATURES)
#print('frames',len(frames))
k = int((len(df)-frame_size)/hop_size)+1
print('type is',type(k))
X_test = frames.reshape(k,200,3,1)
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
I am using hopping window approach on cnn data having 3 columns x-axis acceleration, y-axis acceleration, z-axis acceleration.
When I run below code outside of while loop, model gives output but that can not be used for real time.
X_test = frames.reshape(k,200,3,1)
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
Training code
# Importing
import numpy as np
import pandas as pd
#import seaborn as sns
import tensorflow as tf
#import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Flatten, Dense, Dropout, BatchNormalization,MaxPooling2D
from tensorflow.keras.layers import Conv2D,MaxPool2D
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import load_model
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
import os
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = "2"
df = pd.read_csv('fall_nofall_10_with_labels.csv')
df.columns = ['a','activity','time','realtime','x','y','z','gyro-x','gyro-y','gyro-z']
data = df.drop(['a','realtime','gyro-x','gyro-y','gyro-z'],axis=1)
print(data['activity'].value_counts())
# # sampling rate
Fs = 50
activities = data['activity'].value_counts().index
balanced_data = data.drop(['time'], axis = 1).copy()
balanced_data['activity'].value_counts()
from sklearn.preprocessing import LabelEncoder
label = LabelEncoder()
balanced_data['label'] = label.fit_transform(df['activity'])
### Frame Prepration
import scipy.stats as stats
Fs = 50
frame_size = Fs*4 #(4 seconds)
# 200x200x3 will be feeded in
hop_size = Fs*2 #(How much overlap) make advancement with 100 data samples
def get_frames(df,frame_size,hop_size):
N_FEATURES = 3 # input feature is x,y and z
frames = []
labels = []
for i in range(0,len(df) - frame_size,hop_size):
x = df['x'].values[i:i+frame_size] # 0 to 4 second then 1 to 5 seconds
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
# activity which comes most number of time we'll be considering that
label = stats.mode(df['label'][i:i+frame_size])[0][0]
# labels = label[0][0]
frames.append([x,y,z])
labels.append(label)
# convert into numpy array
frames = np.asarray(frames).reshape(-1,frame_size,N_FEATURES)
labels = np.asarray(labels)
return frames,labels
X,y = get_frames(balanced_data,frame_size,hop_size)
print(X.shape,y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0, stratify = y)
X_train.shape, X_test.shape
X_train[0].shape, X_test[0].shape
X_train = X_train.reshape(39,200,3,1)
X_test = X_test.reshape(10,200,3,1)
X_train[0].shape,X_test[0].shape
# ## Creating CNN model
model = Sequential()
model.add(Conv2D(16,(2,2),activation='relu',input_shape=X_train[0].shape))
# model.add(MaxPooling2D(pool_size=(1,1)))
model.add(Dropout(0.1))
model.add(Conv2D(64,(2,2),activation='relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(64,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2,activation='sigmoid'))
model.summary()
model.compile(optimizer=Adam(learning_rate = 0.001), loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
history = model.fit(X_train, y_train, epochs = 10, validation_data= (X_test, y_test), verbose=1)
model.save("model_cnn.h5")
# load the model
model_trained = load_model('model_cnn.h5')
# summary of the model
print('\n',model_trained.summary())
from sklearn.metrics import confusion_matrix,classification_report
y_pred = model_trained.predict_classes(X_test)
print(y_pred)
print(classification_report(y_pred,y_test))
print(confusion_matrix(y_pred,y_test))
complete traceback:
Traceback (most recent call last):
File "cnn_nofall_live.py", line 68, in <module>
y_pred = model_trained.predict_classes(X_test)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/sequential.py", line 338, in predict_classes
proba = self.predict(x, batch_size=batch_size, verbose=verbose)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training.py", line 1013, in predict
use_multiprocessing=use_multiprocessing)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 498, in predict
workers=workers, use_multiprocessing=use_multiprocessing, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 475, in _model_iteration
total_epochs=1)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_v2.py", line 187, in run_one_epoch
aggregator.finalize()
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/keras/engine/training_utils.py", line 353, in finalize
self.results = nest.pack_sequence_as(self._structure, self.results)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 504, in pack_sequence_as
return _pack_sequence_as(structure, flat_sequence, expand_composites)
File "/usr/local/lib/python3.7/dist-packages/tensorflow_core/python/util/nest.py", line 453, in _pack_sequence_as
len(flat_sequence), truncate(flat_sequence, 100)))
ValueError: The target structure is of type `<class 'NoneType'>`
None
However the input structure is a sequence (<class 'list'>) of length 0.
[]
nest cannot guarantee that it is safe to map one to the other.

I think you need to post the full traceback. It is difficult to debug/help without seeing that
But OOTH one possible issue is that you're using "i" for the while loop and the for loop. Maybe change the for loop index to "j" or something.
Try running this code snippet to see what I mean
i=0
while(i<10):
i+=1
for i in range(100):
print(f"for loop i is {i}")
print(f"while loop i is {i}")

btw, you trying for this?
data1 = []
i = 0
while(i<2000):
i+=1
ax,ay,az = mpu.acceleration
data1.append([ax,ay,az])
df = pd.DataFrame(data1, columns=['x','y','z'])
frame_size = 200
hop_size = 100
frames = []
N_FEATURES = 3
for i in range(0,len(df)-frame_size,hop_size):
x = df['x'].values[i:i+frame_size]
y = df['y'].values[i:i+frame_size]
z = df['z'].values[i:i+frame_size]
frames.append([x,y,z])
frames = np.array(frames)
k = int((len(df)-frame_size)/hop_size)
X_test = frames.reshape(k,200,3,1)

Keras Sequential model. Generating predictions on non-daily time intervals

I am trying to generate price predictions for cryptocurrencies using keras. I have it working on 1 day intervals using 1 day data from the Kraken API. My problem comes when I try to generate predictions on other time frames. 1hr 4hr and 1week. The prediction timestamps come out as daily regardless of the input data. For hourly the output dataframe is completely scrambled. These are screenshots of the output.This is correctly working
This is the scrambled 1 hour dataframe
This is the 4hr dataframe with wrong timestamps
My code is as follows. Sorry it's not a minimum reproducible example.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense, Dropout, GRU, SimpleRNN, Conv1D
from tensorflow.keras.optimizers import Adam
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler, Normalizer
import os
import tensorflow as tf
import krakenex
from pykrakenapi import KrakenAPI
#Pulling a DataFrame from the API
api = krakenex.API()
k = KrakenAPI(api)
ohlc, last = k.get_ohlc_data("ETHUSD", interval=60)
df = ohlc
last_close = ohlc
last_close = last_close[['close']]
last_close = last_close.iloc[::-1]
last_price = last_close.close[-1]
print(last_price)
#Clipping Dataframe to only containg index anc close prices
df = df[['close']]
#Reversing dataframe order
df = df.iloc[::-1]
df.tail()
#LSTM works better if data is scaled but removing for a moment
scaler = MinMaxScaler()
df = pd.DataFrame(scaler.fit_transform(df), columns = df.columns, index=df.index)
#print(df.columns)
df.tail()
#Visualizing price action
df.plot(figsize=(14,8))
plt.title('ETH prices')
plt.ylabel('normalized prices')
plt.show()
def split_sequence(seq, n_steps_in, n_steps_out):
"""
Splits the unzivariate time sequence
"""
# Creating a list for both variables
X, y = [], []
for i in range(len(seq)):
# Finding the end of the current sequence
end = i + n_steps_in
out_end = end + n_steps_out
# Breaking out of the loop if we have exceeded the dataset's length
if out_end > len(seq):
break
# Splitting the sequences into: x = past prices and indicators, y = prices ahead
seq_x, seq_y = seq[i:end], seq[end:out_end]
X.append(seq_x)
y.append(seq_y)
return np.array(X), np.array(y)
#Some charting for accuracy tests
def visualize_results(results):
history = results.history
plt.figure(figsize=(14,8))
plt.plot(history['val_loss'])
plt.plot(history['loss'])
plt.legend(['val_loss', 'loss'])
plt.title('Loss')
plt.xlabel('epochs')
plt.ylabel('loss')
plt.show()
plt.figure(figsize=(14,8))
plt.plot(history['val_accuracy'])
plt.plot(history['accuracy'])
plt.legend(['val_accuracy', 'accuracy'])
plt.title('Accuracy')
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.show()
#number of periods to look back at
n_per_in = 680
#number of periods to predict
n_per_out = 20
n_features = 1
#making a sequence for predictions
X, y = split_sequence(list(df.close), n_per_in, n_per_out)
#Giving the model the proper dimensions
X = X.reshape(X.shape[0], X.shape[1], n_features)
# Instantiating the model
d = 0.2
activ = 'softsign'
model = Sequential()
#model.add(Conv1D(100, kernel_size=2, input_shape=(n_per_in, n_features), activation=activ))
model.add(LSTM(60, input_shape=(n_per_in, n_features), activation=activ))
model.add(Dense(units=n_per_out))
# Model summary
model.summary()
model.compile(loss = "mse", optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), metrics=['accuracy'])
res = model.fit(X,y, epochs=10, batch_size=4, validation_split=0.2)
plt.figure(figsize=(14, 8))
yhat = model.predict(X[-1].reshape(1,n_per_in,n_features)).tolist()[0]
yhat = np.array(yhat).reshape(-1,1).tolist()
actual = y[-1].reshape(-1,1)
print("predicted ", yhat)
plt.plot(yhat, label='predicted')
print('actuals ', actual.tolist())
plt.plot(actual.tolist(), label='actual')
plt.title("Predicted vs Actual")
plt.ylabel('price')
plt.legend()
plt.show()
yhat = model.predict(np.array(df.head(n_per_in)).reshape(1, n_per_in, n_features)).tolist()[0]
yhat = np.array(yhat).reshape(-1,1).tolist()
preds = pd.DataFrame(yhat, index=pd.date_range(start=df.index[-1], periods=len(yhat)), columns=df.columns)
print(preds)
periods = 30
actual = pd.DataFrame(df[['close']].tail(periods), index = df.close.tail(periods).index, columns= df.columns).append(preds.head(1))
plt.figure(figsize=(14,8))
plt.plot(actual, label='actuals')
plt.plot(preds, label='predictions')
plt.ylabel("price")
plt.xlabel('times')
plt.title(f'Forecasting the next {len(yhat)} days')
plt.legend()
plt.show()
How can I get the correct time frames in my output? I've gone to the docs and scoured google for hours and can't come up with a solution other than switching to prophet rather than Keras, and I don't want to do that.

How to display Predictive model graph on Django framework?

I've made a predictive model using LSTM which predicts future prices for raw materials like cotton,fibre,yarn etc. At the end of code I used matplotlib library to plot graph which displays the original prices, predicted prices and future predicted prices.
This is the graph which shows future prices according to dates
How do I display this graph on Django framework? Because I need to deploy this model on a web application using Django but the tutorials I've seen so far show predictive models which take user input and don't really show anything related to plots or graphs.
Following is the code:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime as dt
from datetime import datetime
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, TensorBoard
import os
import glob
import pandas
import numpy
from sklearn import preprocessing
import numpy as np
# Importing Training Set
dataset_train = pd.read_csv('201222-yarn-market-price-china--034.csv1.csv')
dataset_train.info()
# Select features (columns) to be involved intro training and predictions
cols = list(dataset_train)[1:5]
# Extract dates (will be used in visualization)
datelist_train = list(dataset_train.iloc[0])
datelist_train = [dt.datetime.strptime(date, '%m/%d/%Y').date() for date in datelist_train]
print('Training set shape == {}'.format(dataset_train.shape))
print('All timestamps == {}'.format(len(datelist_train)))
print('Featured selected: {}'.format(cols))
dataset_train = dataset_train[cols].astype(str)
for i in cols:
for j in range(0, len(dataset_train)):
dataset_train[i][j] = dataset_train[i][j].replace(',', '')
dataset_train = dataset_train.astype(float)
# Using multiple features (predictors)
training_set = dataset_train.values
print('Shape of training set == {}.'.format(training_set.shape))
training_set
# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
training_set_scaled = sc.fit_transform(training_set)
sc_predict = StandardScaler()
sc_predict.fit_transform(training_set[:, 0:1])
# Creating a data structure with 90 timestamps and 1 output
X_train = []
y_train = []
n_future = 60 # Number of days we want top predict into the future
n_past = 90 # Number of past days we want to use to predict the future
for i in range(n_past, len(training_set_scaled) - n_future +1):
X_train.append(training_set_scaled[i - n_past:i, 0:dataset_train.shape[1] - 1])
y_train.append(training_set_scaled[i + n_future - 1:i + n_future, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
print('X_train shape == {}.'.format(X_train.shape))
print('y_train shape == {}.'.format(y_train.shape))
# Import Libraries and packages from Keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from tensorflow.keras.optimizers import Adam
# Initializing the Neural Network based on LSTM
model = Sequential()
# Adding 1st LSTM layer
model.add(LSTM(units=64, return_sequences=True, input_shape=(n_past, dataset_train.shape[1]-1)))
# Adding 2nd LSTM layer
model.add(LSTM(units=10, return_sequences=False))
# Adding Dropout
model.add(Dropout(0.25))
# Output layer
model.add(Dense(units=1, activation='linear'))
# Compiling the Neural Network
model.compile(optimizer = Adam(learning_rate=0.01), loss='mean_squared_error')
es = EarlyStopping(monitor='val_loss', min_delta=1e-10, patience=10, verbose=1)
rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, verbose=1)
mcp = ModelCheckpoint(filepath='weights.h5', monitor='val_loss', verbose=1,
save_best_only=True, save_weights_only=True)
tb = TensorBoard('logs')
history = model.fit(X_train, y_train, shuffle=True, epochs=30, callbacks=[es, rlr, mcp, tb],
validation_split=0.2, verbose=1, batch_size=256)
# Generate list of sequence of days for predictions
datelist_future = pd.date_range(datelist_train[-1], periods=n_future, freq='1d').tolist()
'''
Remeber, we have datelist_train from begining.
'''
# Convert Pandas Timestamp to Datetime object (for transformation) --> FUTURE
datelist_future_ = []
for this_timestamp in datelist_future:
datelist_future_.append(this_timestamp.date())
# Perform predictions
predictions_future = model.predict(X_train[-n_future:])
predictions_train = model.predict(X_train[n_past:])
# Inverse the predictions to original measurements
# ---> Special function: convert <datetime.date> to <Timestamp>
def datetime_to_timestamp(x):
'''
x : a given datetime value (datetime.date)
'''
return datetime.strptime(x.strftime('%m%d%Y'), '%m%d%Y')
y_pred_future = sc_predict.inverse_transform(predictions_future)
y_pred_train = sc_predict.inverse_transform(predictions_train)
a=dataset_train.iloc[:, 3]
print(a)
PREDICTIONS_FUTURE = pd.DataFrame(y_pred_future, columns=['Cotton
Yarn1']).set_index(pd.Series(datelist_future))
PREDICTION_TRAIN = pd.DataFrame(y_pred_train, columns=['Cotton
Yarn1']).set_index(pd.Series(datelist_train[2 * n_past + n_future -1:]))
# Convert <datetime.date> to <Timestamp> for PREDCITION_TRAIN
PREDICTION_TRAIN.index = PREDICTION_TRAIN.index.to_series().apply(datetime_to_timestamp)
print(PREDICTION_TRAIN.head(3))
#plt.rcParams["figure.figsize"] = (20,3)
#rcParams['figure.figsize'] = 14, 5
# Plot parameters
START_DATE_FOR_PLOTTING = '12/24/2019'
dataset_train = pd.DataFrame(dataset_train, columns=cols)
dataset_train.index = datelist_train
dataset_train.index = pd.to_datetime(dataset_train.index)
plt.plot(PREDICTIONS_FUTURE.index, PREDICTIONS_FUTURE['Cotton Yarn1'], color='r',
label='Predicted Stock Price')
plt.plot(PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:].index,
PREDICTION_TRAIN.loc[START_DATE_FOR_PLOTTING:]['Cotton Yarn1'], color='orange',
label='Training predictions')
plt.plot(dataset_train.loc[START_DATE_FOR_PLOTTING:].index,
dataset_train.loc[START_DATE_FOR_PLOTTING:]['Cotton Yarn1'], color='b', label='Actual Stock
Price')
plt.axvline(x = min(PREDICTIONS_FUTURE.index), color='green', linewidth=2, linestyle='--')
plt.grid(which='major', color='#cccccc', alpha=0.5)
plt.legend(shadow=True)
plt.title('Predcitions and Acutal Stock Prices', family='Arial', fontsize=12)
plt.xlabel('Timeline', family='Arial', fontsize=10)
plt.ylabel('Stock Price Value', family='Arial', fontsize=10)
plt.xticks(rotation=45, fontsize=8)
plt.show()

Bayesian-NN with Tensorflow probability: predictions not working

I'm trying to implement a Bayesian neural network for genomic predictions. My X is a matrix that is scaled and gets normalized so that the values are between 0 and 1. The y is a vector of values that are again normalized so that the values are between 0 and 1.
The network seems to learn as seen here:
But, when I try to make predictions these look strange and seem to behave randomly. While the true values of y are distributed between 0 and 1. The predicted values are between ~ 0.4 - 0.6 and my R2 is negative. The MSE is around 0.02, what seems not to bad, but might be caused by the fact that the range of the predictions is quite narrow.
I'm a bit running out of ideas what could be wrong. Any suggestions are appreciated :).
I've also tried to predict the training data. That is also not working and I'm getting a negative R2.
X has the dimensions (5000,500) and y (5000,)
Increasing the number of hidden layers (up to 3) and units (upt to 128) doesn't change anything.
# Import necessary packages:
import sys
from os.path import join
import warnings
warnings.filterwarnings('ignore')
from IPython import display
import tensorflow as tf
from tensorflow import keras
import tensorflow_probability as tfp
import kerastuner as kt
from keras import backend as K
from keras import activations, initializers
from keras.layers import Layer
import tensorflow_docs as tfdocs
import tensorflow_docs.modeling
import tensorflow_docs.plots
import numpy as np
import numpy.ma as ma
import pandas as pd
import seaborn as sns
import matplotlib.pylab as plt
import time
import tempfile
import math
import statsmodels.api as sm
from sklearn.model_selection import train_test_split, KFold
from sklearn.metrics import r2_score,mean_absolute_error
from sklearn.linear_model import LinearRegression, BayesianRidge
from sklearn.utils import shuffle
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, scale
from pandas_plink import read_plink
from pandas_plink import read_plink1_bin
from pandas_plink import get_data_folder
tfd = tfp.distributions
# Set random seed and start timer
np.random.seed(12345)
start = time.time()
### functions
def get_optimizer():
return tf.keras.optimizers.SGD()
def get_callbacks():
return [
#tfdocs.modeling.EpochDots(),
tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=1, patience=500, restore_best_weights=True),
]
def normalize_data(df):
return (df - df.min())/(df.max() - df.min())
def compile_model(model, optimizer=None):
if optimizer is None:
optimizer = get_optimizer()
model.compile(optimizer=optimizer,
loss=keras.losses.MeanSquaredError())
return model
def MSE(test,pred):
sqr_err = np.subtract(test,pred)**2
return sqr_err.mean()
# ## load & preprocess data
# ### load genotype data
G = np.genfromtxt("genotype.txt")
G[np.isnan(G)] = 0.
G = normalize_data(G)
print(G.mean())
print(G.var())
# ### load phenotype data
traits = np.genfromtxt("phenotype.txt")
traits = normalize_data(traits)
print(traits.mean())
print(traits.var())
# ### split training and validation set
train_X, test_X, train_y, test_y = train_test_split(G, traits, test_size = 0.2, random_state = 42)
X = np.concatenate((train_X, test_X), axis=0)
y = np.concatenate((train_y, test_y), axis=0)
# ### parameter definition
N = G.shape[0]
p = G.shape[1]
NUM_FOLDS = 5
kfold = KFold(n_splits=NUM_FOLDS, shuffle=True)
INPUT_SHAPE = X.shape[1]
OUTPUT_SHAPE = y.shape[0]
BATCH_SIZE = 32
STEPS_PER_EPOCH = math.ceil((X.shape[0]*(1-1/NUM_FOLDS)*0.8)/BATCH_SIZE)
MAX_EPOCHS = 5000
df = pd.DataFrame(columns = ['method','MSE','R2'])
histories = {}
# Specify the surrogate posterior over `keras.layers.Dense` `kernel` and `bias`.
def posterior_mean_field(kernel_size, bias_size=0, dtype=None):
n = kernel_size + bias_size
c = np.log(np.expm1(1.))
return tf.keras.Sequential([
tfp.layers.VariableLayer(2 * n, dtype=dtype),
tfp.layers.DistributionLambda(lambda t: tfd.Independent(
tfd.Normal(loc=t[..., :n],
scale=1e-5 + tf.nn.softplus(c + t[..., n:])),
reinterpreted_batch_ndims=1)),
])
# Specify the prior over `keras.layers.Dense` `kernel` and `bias`.
def prior_trainable(kernel_size, bias_size=0, dtype=None):
n = kernel_size + bias_size
return tf.keras.Sequential([
tfp.layers.VariableLayer(n, dtype=dtype),
tfp.layers.DistributionLambda(lambda t: tfd.Independent(
tfd.Normal(loc=t, scale=1),
reinterpreted_batch_ndims=1)),
])
def neg_log_likelihood(y_true, y_pred, sigma=1.0):
dist = tfp.distributions.Normal(loc=y_pred, scale=sigma)
return K.sum(-dist.log_prob(y_true))
#neg_log_likelihood = lambda y, p_y: -p_y.log_prob(y)
kl_loss_weight = 1.0 / STEPS_PER_EPOCH
histories = {}
fold_no = 1
for train, test in kfold.split(X, y):
model = tf.keras.Sequential([
keras.layers.InputLayer(input_shape=(INPUT_SHAPE,)),
tfp.layers.DenseVariational(units=32,
make_posterior_fn=posterior_mean_field,
make_prior_fn=prior_trainable,
kl_weight=kl_loss_weight,
activation='sigmoid'),
tfp.layers.DenseVariational(units=1,
make_posterior_fn=posterior_mean_field,
make_prior_fn=prior_trainable,
kl_weight=kl_loss_weight,
activation='sigmoid'),
tfp.layers.DistributionLambda(lambda t: tfd.Normal(loc=t, scale=1)),
])
model.compile(loss=neg_log_likelihood, optimizer=tf.keras.optimizers.Adam(lr=0.0001), metrics=['mse'])
history = model.fit(X[train], y[train],
validation_split = 0.2,
steps_per_epoch = STEPS_PER_EPOCH,
epochs=MAX_EPOCHS,
callbacks=get_callbacks(),
verbose=0)
histories['BNN2_'+str(fold_no)] = history
y_pred_list = []
for i in range(500):
y_pred = model.predict(X[test])
y_pred_list.append(y_pred)
y_preds = np.concatenate(y_pred_list, axis=1)
y_mean = np.mean(y_preds, axis=1)
m_err = MSE(y[test],y_mean)
r2_acc = r2_score(y[test],y_mean)
df = df.append({'MSE':m_err, 'R2':r2_acc, 'method':'BNN2'}, ignore_index=True)
fold_no = fold_no + 1
df.to_csv("results.csv")

How to predict future data or data of an unknown range after training an LSTM model with a time series dataset?

I have trained a neural network with an LSTM model using a time series dataset.
I am using this dataset which records data from 1970-2016 of the daily rainfall of 35 locations : https://www.kaggle.com/redikod/historical-rainfall-data-in-bangladesh
It looks like this :
StationIndex Station Year Month Day Rainfall dayofyear
1970-01-01 1 Dhaka 1970 1 1 0 1
1970-01-02 1 Dhaka 1970 1 2 0 2
1970-01-03 1 Dhaka 1970 1 3 0 3
1970-01-04 1 Dhaka 1970 1 4 0 4
1970-01-05 1 Dhaka 1970 1 5 0 5
I have completed the training using the train and test data. Then I checked the predicted value against true value.
Here is the complete code, sorry if it is messy. I have put comments for each section:
import numpy as np
from pandas.plotting import register_matplotlib_converters
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rc
from pylab import rcParams
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from keras.layers import (
Input,
Dense,
LSTM,
AveragePooling1D,
TimeDistributed,
Flatten,
Bidirectional,
Dropout
)
from keras.models import Model
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
tf.keras.backend.clear_session()
register_matplotlib_converters()
sns.set(style='whitegrid', palette='muted', font_scale=1.5)
rcParams['figure.figsize'] = 22, 10
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)
#reading from CSV
df = pd.read_csv("\customized_daily_rainfall_data_Copy.csv")
#droping bad data
df = df[df.Rainfall != -999]
#droping dates (leapyear, wrong day numbers of month)
df.drop(df[(df['Day']>28) & (df['Month']==2) & (df['Year']%4!=0)].index,inplace=True)
df.drop(df[(df['Day']>29) & (df['Month']==2) & (df['Year']%4==0)].index,inplace=True)
df.drop(df[(df['Day']>30) & ((df['Month']==4)|(df['Month']==6)|(df['Month']==9)|(df['Month']==11))].index,inplace=True)
#date parcing (Index)
date = [str(y)+'-'+str(m)+'-'+str(d) for y, m, d in zip(df.Year, df.Month, df.Day)]
df.index = pd.to_datetime(date)
df['Date'] = df.index
df['Dayofyear']=df['Date'].dt.dayofyear
df.drop('Date',axis=1,inplace=True)
df.drop(['Station'],axis=1,inplace=True)
df.head()
#limiting the dataframe to just rows where StationIndex is 11
datarange = df.loc[df['StationIndex'] == 11]
#splitting train and test set
train_size = int(len(datarange) * 0.9)
test_size = len(datarange) - train_size
train, test = df.iloc[0:train_size], df.iloc[train_size:len(datarange)]
#Scaling the feature and label columns of the dataset
from sklearn.preprocessing import RobustScaler
f_columns = ['Year', 'Month','Day','Dayofyear']
f_transformer = RobustScaler()
l_transformer = RobustScaler()
f_transformer = f_transformer.fit(train[f_columns].to_numpy())
l_transformer = l_transformer.fit(train[['Rainfall']])
train.loc[:, f_columns] = f_transformer.transform(train[f_columns].to_numpy())
train['Rainfall'] = l_transformer.transform(train[['Rainfall']])
test.loc[:, f_columns] = f_transformer.transform(test[f_columns].to_numpy())
test['Rainfall'] = l_transformer.transform(test[['Rainfall']])
#making smaller train and test sections withing the dataset
def create_dataset(X, y, time_steps=1):
Xs, ys = [], []
for i in range(len(X) - time_steps):
v = X.iloc[i:(i + time_steps)].to_numpy()
Xs.append(v)
ys.append(y.iloc[i + time_steps])
return np.array(Xs), np.array(ys)
time_steps = 7
# reshape to [samples, time_steps, n_features]
X_train, y_train = create_dataset(train, train.Rainfall, time_steps)
X_test, y_test = create_dataset(test, test.Rainfall, time_steps)
#testing
X_test[0][0]
#model code
model = keras.Sequential()
#3 biderectional LSTM layers
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128, input_shape=(X_train.shape[1], X_train.shape[2]), return_sequences = True)))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128, return_sequences = True)))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=128 )))
model.add(keras.layers.Dropout(rate=0.1))
model.add(keras.layers.Dense(units=1))
model.compile(loss="mean_squared_error", optimizer="RMSprop")
#training the model
history = model.fit(
X_train, y_train,
epochs=500,
batch_size=1052,
validation_split=0.2,
shuffle=False,
)
#saving the model
from tensorflow.keras.models import load_model
model.save("\Timeseries-timestep7-batchsize1052.h5")
#Using text dataset to do a prediction
y_pred = model.predict(X_test)
#inverst transformation
y_train_inv = l_transformer.inverse_transform(y_train.reshape(1, -1))
y_test_inv = l_transformer.inverse_transform(y_test.reshape(1, -1))
y_pred_inv = l_transformer.inverse_transform(y_pred)
#score
from sklearn import metrics
score = np.sqrt(metrics.mean_squared_error(y_pred,y_test))
print(score)
What I want to do next, is for the predictions to go beyond the data in the dataset.
For example, using the model that I have trained to predict data of the future. Or possibly a random/custom range.
Suppose I want to predict the day by day rainfall data of 2017. Or get the predicted data of 25-02-2017. Or maybe the data of X number of days after the end of the dataset.
Is there a good intuitive way to do that?
Thank you in advanced to whoever can answer this question. It's been bugging me for a few days now.

The code shown below should serve your purpose.
class WindowGenerator():
def __init__(self, input_width, label_width, shift,
train_df=train_df, val_df=val_df, test_df=test_df,
label_columns=None):
# Store the raw data.
self.train_df = train_df
self.val_df = val_df
self.test_df = test_df
# Work out the label column indices.
self.label_columns = label_columns
if label_columns is not None:
self.label_columns_indices = {name: i for i, name in
enumerate(label_columns)}
self.column_indices = {name: i for i, name in
enumerate(train_df.columns)}
# Work out the window parameters.
self.input_width = input_width
self.label_width = label_width
self.shift = shift
self.total_window_size = input_width + shift
self.input_slice = slice(0, input_width)
self.input_indices = np.arange(self.total_window_size)[self.input_slice]
self.label_start = self.total_window_size - self.label_width
self.labels_slice = slice(self.label_start, None)
self.label_indices = np.arange(self.total_window_size)[self.labels_slice]
def __repr__(self):
return '\n'.join([
f'Total window size: {self.total_window_size}',
f'Input indices: {self.input_indices}',
f'Label indices: {self.label_indices}',
f'Label column name(s): {self.label_columns}'])
For example, if your Data is from the years, 1960 (Jan 1st) to 2016 (Dec 31st) and that you want to predict the weather of the entire February Month for the year 2017, by considering a Data Window of past 2 Years, the values of the arguments of the above class are shown below:
input_width: 2 Years => 365 * 2 = 730
label_width: Entire Feb Month => 28
shift: We are not predicting from Jan 1st 2017 but are shifting by entire Month of Jan => 30
train_df, test_df, val_df => Self Explanatory
label_columns : Name of the Target Column
For more information, please refer this Tensorflow Tutorial on Time Series Analysis.