Develop Reference

How to forecast a univariate time series 20/30 days ahead using tensorflow LSTM?

I have used below code for training and validation. It gives decent result but I don't know the code to forecast n periods ahead (like 30/50 days ahead) using the trained model.
GitHub Link for the code with data output is here.
Import the libraries:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import math
from keras.models import Sequential
from keras.layers import Dense,Dropout,Conv1D,Bidirectional
from keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')
import tensorflow
np.random.seed(1)
tensorflow.random.set_seed(1)
Load the univariate time series data and normalize the values:
dataframe=pd.read_sas('train.sas7bdat')
dataframe['Datetime']=pd.to_datetime(dataframe['Datetime'],format='%d%b%Y:%H:%M:%S')
dataframe.set_index('Datetime',inplace=True)
data=dataframe
#filter input data according to datetime i.e: 01th May 2020
dataset=data[data.index>='2021-05-01 00:00:00']
# Replcae null value with previous 15 minute value
dataset.ffill(axis ='rows',inplace=True)
dataset.shape
def normalize_cols(df,cols):
"""Scale the values of each feature
according to the columns max value"""
data = df.loc[:,cols]
for col in cols:
scaler = lambda x: x / data[col].max()
data[col] = data[col].apply(scaler)
print(data[cols].head())
return data[cols].values
features = df.columns.values # columns to train model on
X = normalize_cols(df,features)
Turn each signal into a labeled dataset:
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
window_size = 15 #15 # num. days per training sample
batch_size = 64 # num. of samples per epoch
buffer_size = 1000 # num of samples in memory for random selection
split_time = 400 # where to split the data for training/validation
forecast_length=10*24*4
def window_dataset(series, window_size, batch_size, shuffle_buffer):
"""Funtion to turn time series data into set of sequences
where the last value is the intended output of our model"""
ser = tf.expand_dims(series, axis=-1)
data = tf.data.Dataset.from_tensor_slices(series)
data = data.window(window_size + 1, shift=1, drop_remainder=True)
data = data.flat_map(lambda w: w.batch(window_size + 1))
data = data.shuffle(shuffle_buffer)
data = data.map(lambda w: (w[:-1], w[1:]))
return data.batch(batch_size).prefetch(1)
x_train = X[:split_time]
x_test = X[split_time:]
print(f"Training data shape: {x_train.shape}")
print(f"Validation data shape: {x_test.shape}")
train_set = window_dataset(x_train,window_size,batch_size,buffer_size)
keras.backend.clear_session()
Choose and connect the model components:
# 1D convolution layers
conv1 = layers.Conv1D(
filters=60,kernel_size=15,strides=1,
padding="causal",activation="relu",
input_shape=[None,len(features)])
conv2 = layers.Conv1D(
filters=60,kernel_size=5,strides=1,
padding="causal",activation="tanh")
# Bidirectional LSTM layers
lstm1 = layers.Bidirectional(layers.LSTM(50,return_sequences=True))
lstm2 = layers.Bidirectional(layers.LSTM(20,return_sequences=True))
# Model construction
inputs = layers.Input(shape=(None,len(features)))
x = conv1(inputs)
x = lstm1(x)
x = lstm2(x)
x = conv2(x)
x = layers.Dense(60,activation='relu')(x)
x = layers.Dropout(.1)(x)
x = layers.Dense(1,activation='tanh')(x)
outputs = layers.Lambda(lambda x: 25*abs(x))(x)
#outputs = layers.Lambda(lambda x: 1*abs(x))(x)
# SGD optimizer and Huber loss
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
loss = keras.losses.Huber()
model = keras.Model(inputs=inputs,outputs=outputs)
model.compile(optimizer,loss,
metrics=["mae"])
model.summary()
"""
### Train model
"""
epochs = 100
history = model.fit(train_set, epochs=epochs, verbose=1)
print(f"Model trained for {epochs} epochs")
Inspect training results:
def model_forecast(model, X, window_size):
"""Takes in numpy array, creates a windowed tensor
and predicts the following value on each window"""
data = tf.data.Dataset.from_tensor_slices(X)
data = data.window(window_size, shift=1, drop_remainder=True)
data = data.flat_map(lambda w: w.batch(window_size))
data = data.batch(32).prefetch(1)
forecast = model.predict(data)
return forecast
train_window = [i for i in range(split_time-window_size)]
forecast = model_forecast(model,x_train,window_size)
import seaborn as sns
plt.figure(figsize=(8,5),dpi=120)
sns.lineplot(train_window,forecast[:-1,1,0].reshape(-1),label='Forecast') #forecast[:-1,1,0]
sns.lineplot(train_window,X[:split_time-window_size].reshape(-1),label='actual_load')
Make predictions on test data:
val_window = [i for i in range(split_time,len(df)-window_size)]
forecast = model_forecast(model,x_test,window_size)
plt.figure(figsize=(8,5),dpi=120)
sns.lineplot(val_window,forecast[:-1,1,0].reshape(-1),label='Forecast')
sns.lineplot(val_window,X[split_time:-window_size].reshape(-1),label='actual_load')

How to input multiple time-series data(wave) for RNN

I made the simple RNN model to learn and fit the one wave file.
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dropout,Dense
from tensorflow.keras.layers import SimpleRNN
import librosa
import librosa.display
import numpy as np
a, sr = librosa.load("A.wav",sr=22050)
rawdata = librosa.stft(a, n_fft=512,hop_length= 512 // 4, window='hann') #make fourier transfered A.wav data.
rawdata = rawdata.transpose() # [Frame,Freq] => [Freq,Frame]
input_len = 10 # the frame number for learning to make next one
input=[]
target=[]
for i in range(0, len(rawdata) - input_len):
input.append( rawdata[i:i+input_len] ) # frames
target.append( rawdata[i+input_len] ) # one step forward frame for answer.
X = np.array(input)
Y = np.array(target)
#Separate 8:2 for training and test
x, val_x, y, val_y = train_test_split(X, Y, test_size=int(X.shape[0] * 0.2), shuffle=False)
n_hidden = 512
epoch = 100
model = Sequential()
model.add(SimpleRNN(n_hidden, input_shape=(input_len, n_in), return_sequences=False))
model.add(Dense(n_hidden, activation="linear"))
model.add(Dense(n_in, activation="linear"))
opt = Adam(lr=0.001)
model.compile(loss='mse', optimizer=opt)
model.summary()
history = model.fit(x, y, epochs=epoch, batch_size=10,validation_data=(val_x, val_y))
OK it works fine.
It learns the one wave file A.wav
However how can I learn multiple wave files??
B.wav C.wav
For example,
If I use model.fit() multiple times for each wav, does this model remember the past learning??

yes, model does remember previous train during fit, you can use fit multiple times as well. but its better to use model.train_on_batch this is simple version of fit to be used on small batch of data.
you can also modify your code to add other wav file feature to data.
# second way
input_len = 10 # the frame number for learning to make next one
input=[]
target=[]
for f in ['A.wav','B.wav','C.wav']:
a, sr = librosa.load(f,sr=22050)
rawdata = librosa.stft(a, n_fft=512,hop_length= 512 // 4, window='hann') #make fourier transfered A.wav data.
rawdata = rawdata.transpose() # [Frame,Freq] => [Freq,Frame]
for i in range(0, len(rawdata) - input_len):
input.append( rawdata[i:i+input_len] ) # frames
target.append( rawdata[i+input_len] ) # one step forward frame for answer.

The accuracy problem of hand sign gestures recognition with using CNN in Python

Im working on my senior project in my university and I have only 2 days to fix this problem.I created a hand gesture recognition with using CNN in Python.I used 78000 images with 50x50px values.But I got stuck in the last part of my model.I can not improve my accuracy.When I start to train the data with 100 epochs,the first 15 epochs show 0,039 accuracy and it is horrible,because of that I'm not waiting the end of the train.Maybe it happens because of the values of conv2d or pooling because I don't know how to put the correct values into conv2d,pooling etc.
I'm new and I could not fix the problem.If you help me,I will be grateful for you
The code I wrote is given below;
from keras.models import Sequential
from keras.layers import Convolution2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.layers import Dropout
from keras.preprocessing.image import ImageDataGenerator
import tensorflow as tf
import pickle
import cv2
import os
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from PIL import Image
from numpy import asarray
DATADIR = "asl_alphabet_train"
CATEGORIES = ["A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"]
X_train = []
y_train = []
X_test=[]
y_test=[]
IMG_SIZE=50
def create_training_data():
for category in CATEGORIES:
path = os.path.join(DATADIR,category) # create path to dogs and cats
class_num = CATEGORIES.index(category) # get the classification (0 or a 1).
for img in tqdm(os.listdir(path)): # iterate over each image per dogs and cats
try:
img_array = cv2.imread(os.path.join(path,img)) # convert to array
#new_array = cv2.resize(img_array, (28, 50 )) # resize to normalize data size
X_train.append(img_array) # add this to our trainingdata
# add this to our X_train
y_train.append(class_num) # add this to our X_train
except Exception as e: # in the interest in keeping the output clean...
pass
create_training_data()
X_train = asarray(X_train)
y_train = asarray(y_train)
"""
nsamples, nx, ny = X_train.shape
X_train = X_train.reshape((nsamples,nx*ny))
"""
X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2,random_state=0)
N = y_train.size
M = y_train.max()+1
resultArray = np.zeros((N,M),int)
idx = (np.arange(N)*M) + y_train
resultArray.ravel()[idx] = 1
y_train=resultArray
classifier=Sequential()
#convolution step
classifier.add(Convolution2D(filters=96, input_shape=(50,50,3), kernel_size=(11,11), padding='valid',activation="relu"))
#pooling step
classifier.add(MaxPooling2D(pool_size=(2,2)))
#convolution step
classifier.add(Convolution2D(filters=256,kernel_size=(11,11),padding="valid",activation="relu"))
#pooling step
classifier.add(MaxPooling2D(pool_size=(2,2)))
classifier.add(Convolution2D(filters=384,kernel_size=(3,3),padding="valid",activation="relu"))
classifier.add(MaxPooling2D(pool_size=(2,2)))
#flatten step
classifier.add(Flatten())
#Dense(Fully connected step)
classifier.add(Dense(output_dim=128,activation="relu"))
#Dropout to decrease the possibility of overfitting
classifier.add(Dropout(0.5))
#Dense to determine the output
classifier.add(Dense(output_dim=26,activation="softmax"))
#compile step
classifier.compile(optimizer="adam",loss="categorical_crossentropy",metrics=["accuracy"])
enter code here
classifier.fit(X_train,y_train,epochs=100,batch_size=32)
filename="CNN_TEST.sav"
pickle.dump(classifier, open(filename, 'wb'))
y_pred=classifier.predict(X_test)
print(y_pred)

Would recommend the following :
1) Reduce the kernel size in the first two convolutional layers of your model.
2) I believe the MaxPooling layer is not necessary after every convolutional layer. Do verify this.
3) A DropOut of 0.5 could drop out a large number of essential neurons, you might want to lower that.
4) Vary the number of epochs and see how your model performs each time.
Plot "train accuracy vs val accuracy" and "train loss vs val loss" at each attempt and see if your model overfits or underfits.

Input Pipeline for LSTM with Timeseries Data Using a Large Dataset with Multiple .csv in Tensorflow

Currently I can train a LSTM network using one csv file based on this tutorial: https://machinelearningmastery.com/how-to-develop-lstm-models-for-time-series-forecasting/
This code generate sliding windows where the last n_steps of the features are saved to predict the actual target (similar to this: Keras LSTM - feed sequence data with Tensorflow dataset API from the generator):
#%% Import
import pandas as pd
import tensorflow as tf
from tensorflow.python.keras.models import Sequential, model_from_json
from tensorflow.python.keras.layers import LSTM
from tensorflow.python.keras.layers import Dense
# for path
import pathlib
import os
#%% Define functions
# Function to split multivariate input data into samples according to the number of timesteps (n_steps) used for the prediction ("sliding window")
def split_sequences(sequences, n_steps):
X, y = list(), list()
for i in range(len(sequences)):
# find end of this pattern
end_ix = i + n_steps
# check if beyond maximum index of input data
if end_ix > len(sequences):
break
# gather input and output parts of the data in corresponding format (depending on n_steps)
seq_x, seq_y = sequences[i:end_ix, :-1], sequences[end_ix-1, -1]
X.append(seq_x)
y.append(seq_y)
#Append: Adds its argument as a single element to the end of a list. The length of the list increases by one.
return array(X), array(y)
# Set source files
csv_train_path = os.path.join(dir_of_file, 'SimulationData', 'SimulationTrainData', 'SimulationTrainData001.csv')
# Load data
df_train = pd.read_csv(csv_train_path, header=0, parse_dates=[0], index_col=0)
#%% Select features and target
features_targets_considered = ['Fz1', 'Fz2', 'Fz3', 'Fz4', 'Fz5', 'Fz_res']
n_features = len(features_targets_considered)-1 # substract the target
features_targets_train = df_train[features_targets_considered]
# "Convert" to array
train_values = features_targets_train.values
# Set number of previous timesteps, which are considered to predict
n_steps = 100
# Convert into input (400x5) and output (1) values
X, y = split_sequences(train_values, n_steps)
X_test, y_test = split_sequences(test_values, n_steps)
#%% Define model
model = Sequential()
model.add(LSTM(200, activation='relu', return_sequences=True, input_shape=(n_steps, n_features)))
model.add(LSTM(200, activation='relu', return_sequences=True))
model.add(LSTM(200, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse', metrics=['mae'])
#%% Fit model
history = model.fit(X, y, epochs=200, verbose=1)
I now want to expand this example to efficiently train the network with different csv files. In the data folder I have the files 'SimulationTrainData001.csv', 'SimulationTrainData002.csv', ..., 'SimulationTrainData300.csv' (about 14 GB).
To achieve this, I tried to adopt the code of this input pipeline example: https://www.tensorflow.org/guide/data#consuming_sets_of_files, which works to a certain extend. I can show the training files in the folder with this change:
# Set source folders
csv_train_path = os.path.join(dir_of_file, 'SimulationData', 'SimulationTrainData')
csv_train_path = pathlib.Path(csv_train_path)
#%% Show five example files from training folder
list_ds = tf.data.Dataset.list_files(str(csv_train_path/'*'))
for f in list_ds.take(5):
print(f.numpy())
One problem is, that in the example the files are pictures of flowers and not time series values and I do not know at which point I can use the split_sequences(sequences, n_steps) function to create the sliding windows to provide the necessary data format to train the LSTM network.
Also, as far as I know, it would be better for the training process, if the generated windows of the different files would be shuffled. I could use the split_sequences(sequences, n_steps) function on every csv file (to generate X_test , y_test) and join the result in one big variable or file and shuffle the windows, but I do not think this is an efficient way and it also had to be redone if n_steps will be changed.
If somebody could suggest a (established) method or example to preprocess my data, I would be very thankful.

You can use the TimeSeriesGenerator after consuming those sets of files.
Here is the reference link.
As per the documentation:
'''
This class takes in a sequence of data-points gathered at equal intervals, along with time-series parameters such as stride, length of history, etc., to produce batches for training/validation.
'''
Provided examples for both univariate & multiple variate scenario
Univariate Example:
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, LSTM
import numpy as np
import tensorflow as tf
# define dataset
series = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
# reshape to [10, 1]
n_features = 1
series = series.reshape((len(series), n_features))
# define generator
n_input = 2
generator = TimeseriesGenerator(series, series, length=n_input, batch_size=8)
# create model
model = Sequential()
model.add(LSTM(100, activation='relu', input_shape=(n_input, n_features)))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit_generator(generator, steps_per_epoch=1, epochs=500, verbose=1)
#sample prediction
inputs = np.array([9, 10]).reshape((1, n_input, n_features))
result = model.predict(inputs, verbose=0)
print(result)
Multi-variate Example
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, LSTM
import numpy as np
import tensorflow as tf
# define dataset
in_seq1 = np.array([10, 20, 30, 40, 50, 60, 70, 80, 90, 100])
in_seq2 = np.array([15, 25, 35, 45, 55, 65, 75, 85, 95, 105])
# reshape series
in_seq1 = in_seq1.reshape((len(in_seq1), 1))
in_seq2 = in_seq2.reshape((len(in_seq2), 1))
# horizontally stack columns
dataset = np.hstack((in_seq1, in_seq2))
# define generator
n_features = dataset.shape[1]
n_input = 2
generator = TimeseriesGenerator(dataset, dataset, length=n_input, batch_size=8)
# define model
model = Sequential()
model.add(LSTM(100, activation='relu', input_shape=(n_input, n_features)))
model.add(Dense(2))
model.compile(optimizer='adam', loss='mse')
# fit model
model.fit_generator(generator, steps_per_epoch=1, epochs=500, verbose=1)
# make a one step prediction out of sample
inputs = np.array([[90, 95], [100, 105]]).reshape((1, n_input, n_features))
result = model.predict(inputs, verbose=1)
print(result)
Note: All of these were simulated using Google Colaboratory

Predict the next location based on the previous days data using lstm

Based on the 10 days of the data I need to predict the 11th-day location of the user. Example of my input is in this link
The dataset contains:
200,000 users with 1550 random location points and 10days of history for each user
from datetime import datetime
from random import seed
import pandas as pd
seed(150)
df = pd.read_csv('input.csv', delimiter=';')
df = df.iloc[:-1,:]
df = df.drop(columns='tslot')
def normalize_cols(df, cols):
for col in df.columns:
print(col, df[col].max(), df[col].min())
df[col] = (df[col] - df[col].min())/(df[col].max() - df[col].min())
return df
df = normalize_cols(df, ['slot', 'location', 'user'])
df.to_csv('cleaned_data.csv', index=False)
from keras.models import Sequential, load_model
from keras.layers import Dense, LSTM, Dropout
from matplotlib import pyplot
import numpy as np
import pandas as pd
df = pd.read_csv('cleaned_data.csv')
train_x = df[['slot', 'user']].copy()
train_y = df[['tower']].copy().values
train_x = train_x.values.reshape((train_x.shape[0], 1, train_x.shape[1]))
model = Sequential()
model.add(LSTM(50, input_shape=(train_x.shape[1], train_x.shape[2])))
model.add(Dropout(0.5))
model.add(Dense(40))
model.add(Dropout(0.3))
model.add(Dense(20))
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(loss='mae', optimizer='adam')
history = model.fit(train_x, train_y, epochs=100, batch_size=25, verbose=2, shuffle=True)
model.save('5g_tower.h5')
del model
model = load_model('5g_tower.h5')
## Prediction
def get_tower_number(model, slot, user):
## normalize input slot, tower, user
slot = ((slot - 0)*1.0) / ((95.0 - 0.0)*1.0)
user = ((user - 1)*1.0) / ((2.0 - 1.0)*1.0)
## get a slot for next 15 mins
res = model.predict(np.array([[[slot, user]]]))
## de-normalize that tower
return int(res[0][0] * ((1354.0 - 9.0)*1.0) + 9.0)
print(get_tower_number(model, 4, 1))
But failed to predict the location of the user for the 11th day. The output I got is wrong compared to the required output.
The example of the output is given in this link. How to implement individual model rather than the global model in this system?
The example output should be the 11th day:
User location slot
1 145 0
1 94 95
2 562 0
2 583 95
The actual dataset contains a total of 350,000 users

Just looking at the model implementation, you're missing activation function for every Dense block, so keras is applying linear activation.
You can modify that using:
model.add(Activation('relu'))