Develop Reference

How to properly shape data to use with LSTM model?

I am working with an LSTM project for learning purposes where I am using time-series data that has 3 columns [current, sma, target] where sma is the simple moving average; I extracted these values from the dataframe like so
data = df[['current', 'sma', 'target']].values
# normlize data
scaler = MinMaxScaler(feature_range=(0,1))
dataset = scaler.fit_transform(data)
# then split inputs from targets
X = dataset[:, :2]
y = dataset[:, 2]
# split into xtrain ytrain xtest ytest
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
Everything works fine so far, and I understand, but the uncharted territory for me would be to convert the x_*, y_* arrays into a 3-d arrays to feed the model; I am using a simple model just to make this work, I am not looking for impressive results, this is purely educational.
The model that I will use:
model = tf.keras.Sequential()
model.add(tf.keras.layers.LSTM(128, input_shape=(timesteps, features), return_sequences=True))
model.add(tf.keras.layers.LSTM(64, return_sequences=False))
model.add(tf.keras.layers.Dense(features))
model.compile(loss='mean_squared_error', optimizer='adam')
How to reshape the data to feed it to the model?

Incompatible shapes in output layer - Tensorflow

I am trying to build a bi-LSTM model in tensorflow, environment google colab. In the training process, the model have an issue: the last layer says that there is shape incompatibility. I wonder if there is any way to reshape the x_train and y_train, to fix this problem
Traceback
ValueError: Shapes (16, 11) and (16, 10) are incompatible
If I change the value of the neurons units to my output layer, from 11 to 10, it does not give any error and the model can be trained. However, I want the output to be 10 and not 11.
# current output layer (run perfectly)
tf.keras.layers.Dense (11, activation = 'softmax')
# expected output layer (shape incompatibility)
tf.keras.layers.Dense (10, activation = 'softmax')
BiLSTM Model
def build_model(vocab_size, embedding_dim=64, input_length=30):
print('\nbuilding the model...\n')
model = tf.keras.Sequential([
tf.keras.layers.Embedding(input_dim=(vocab_size + 1), output_dim=embedding_dim, input_length=input_length),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(rnn_units,return_sequences=True, dropout=0.2)),
tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(rnn_units,return_sequences=True, dropout=0.2)),
tf.keras.layers.GlobalMaxPool1D(),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(64, activation='tanh'),
# softmax output layer
tf.keras.layers.Dense(10, activation='softmax')
])
# optimizer & loss
opt = 'RMSprop' #tf.optimizers.Adam(learning_rate=1e-4)
loss = 'categorical_crossentropy'
# Metrics
metrics = ['accuracy', 'AUC','Precision', 'Recall']
# compile model
model.compile(optimizer=opt,
loss=loss,
metrics=metrics)
model.summary()
return model
The BATCH_SIZE is set to 16. And the shapes of y_train and x_train are:
x_train.shape
(800, 30)
y_train.shape
(800,)
Training
def train(model, x_train, y_train, x_validation, y_validation,
epochs, batch_size=32, patience=5,
verbose=2, monitor_es='accuracy', mode_es='auto', restore=True,
monitor_mc='val_accuracy', mode_mc='max'):
print('\ntraining...\n')
# callback
early_stopping = tf.keras.callbacks.EarlyStopping(monitor=monitor_es,
verbose=1, mode=mode_es, restore_best_weights=restore,
min_delta=1e-3, patience=patience)
model_checkpoint = tf.keras.callbacks.ModelCheckpoint('tfjsmode.h5', monitor=monitor_mc, mode=mode_mc,
verbose=1, save_best_only=True)
# Define Tensorboard as a Keras callback
tensorboard = TensorBoard(
log_dir='./logs',
histogram_freq=1,
write_images=True
)
keras_callbacks = [tensorboard, early_stopping, model_checkpoint]
# train model
history = model.fit(x_train, y_train,
batch_size=batch_size, epochs=epochs, verbose=verbose,
validation_data=(x_validation, y_validation),
callbacks=keras_callbacks)
return history
Preprocessing
def preprocess(x, padding_shape=30):
return np.array([ord(i.lower()) - ord('a')+1 if not i.isdigit() and i != ' ' else 0 for i in list(x)] + ([0] * (padding_shape - len(x))), dtype=int)
def prepare_dataset(labeldict : dict, test_size=.3, validation_size=.1):
print('preparing the dataset...\n')
from sklearn import preprocessing
# load dataset
# split dataset (as string into panda.core.series.Serie object)
x, y = load_clean_dataset()
x = np.array(list(map(preprocess, x)))
y = np.array(list(map(lambda x: labeldict[x.replace(' ', '_')], y)))
print(('y: {}').format(y))
# create/split train, validation and test and shuffle the data
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=test_size, shuffle=True)
print(x.max(), x.min())
x_train_val, x_validation, y_train_val, y_validation = train_test_split(x_train, y_train, test_size=test_size, shuffle=True)
# pandas.core.series.Series to numpy array
x_train, y_train = np.array(x_train), np.array(y_train)
x_validation, y_validation = np.array(x_validation), np.array(y_validation)
x_test, y_test = np.array(x_test), np.array(y_test)
x_train_val, y_train_val = np.array(x_train_val), np.array(y_train_val)
print(('\nx_train: \n{}\n\ny_train: \n{}').format(x_train_val, y_train_val))
y_train = tf.keras.utils.to_categorical(y, num_classes=10)
return (x_train, y_train), (x_validation, y_validation), (x_test, y_test), (x_train_val, y_train_val)

It seems you currently have labels as integers (i.e. not one-hot encoded vectors). For example your y seems to be like,
[0, 1, 8, 9, ....] # a vector of 800 elements
There's two ways to train a model on such data.
Alternative 1 (easiest I guess)
Use sparse_categorical_crossentropy as the loss function of the model
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=metrics)
Alternative 2
Convert your labels to one-hot encoded using,
y_onehot = tf.keras.utils.to_categorical(y, num_classes=10)
and then keep the loss of the model as categorical_crossentropy

making prediction on the entire testset

I want to make predictions on the entire test set, here the test set is only 20% of datasetA, I understand that this is because its only for training purposes, when I save the weights and then make predictions on another datasetB, will it also split the test-set datasetB.
How can I make predictions on the entire test-set datasetB using the weights of datasetA that it was trained on.
Thanks.
x = dataset.iloc[:, :-1].values
# Dependent Variable:
y = dataset.iloc[:, -1].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
# Initialising the ANN
classifier = Sequential()
# Adding the input layer and the first hidden layer
classifier.add(Dense(units = 27, kernel_initializer = 'uniform', activation = 'relu', input_dim = 6))
# Adding the second hidden layer
classifier.add(Dense(units = 27, kernel_initializer = 'uniform', activation = 'relu'))
# Adding the output layer
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
# Compiling the ANN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
# Fitting the ANN to the Training set
classifier.fit(X_train, y_train, batch_size = 10, epochs = 20)
#making predictions on test data
classifier.predict(X_test)

If I am understanding correctly, you want to use your trained model on a completely new dataset?
Keras provides several ways to do this, but I think the most common one would be to export your trained model into a .hd5 file using the command
model.save("filepath/model.hd5")
Now you can load in and use your model to wherever you want using the commands
model = model.load("filepath/model.hd5")
score = model.evaluate(X, Y)
where X is the feature columns of Dataset B and Y is the response to get your scoring. If dataset B is in the same instance, you can always just use
model.predict(X)
Where X is now the feature columns of dataset B

From what I understand you are asking 2 questions here:
First, the splitting of "dataset B" into a train and test set is done manually by you in the line
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0).
If, when you use your "dataset B", you want to test your classifier on ALL the data points of "dataset B", you do not have to do this train test split, and can simply pass the X values of "dataset B" to your classifier.
As for how to do this, as per your second question, it is the same as what you have already done with "dataset A"'s test set:
classifier.predict(X) will make predictions using the fit it already learned on "dataset A", assuming you do not recompile or call .fit() again.

How to simultaneously apply augmentation to X_train and y_train

I am facing this problem of creating a dataset from a very few images.
Both input (X_train) and output (y_train) contains (28x28) size images such as MNIST. For example in my code:
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape((X_train.shape[0], 28, 28, 1))
X_train = X_train.astype('float32')
y_train=X_train
datagen = ImageDataGenerator(zca_whitening=True)
How can I fit this datagen to both X_train and y_train simultaneously and save them in a dataset array. Don't want to pass it to training.
Thank you for the help

Beware that augmentation per se is not applied on the target variable y_train but only on the input variables X_train. The generator is only going to reproduce the same ground truth labels y for the newly generated X.
Hence fitting the generator is only using X_train:
datagen.fit(X_train)
If you do not want to pass the augmented data to training, you can loop over the generator after fitting to get the generated samples:
for X_batch, y_batch in datagen.flow(X_train, y_train, batch_size=32):
# Do whatever you want with the generated X_batch and y_batch.
I understand that is what you are willing to do.
See examples on keras doc.

Keras fit() on ImageDataGenerator() with featurewise_center gives poor validation accuracy

I have a question about using fit() on ImageDataGenerator.
I run MNIST testing successfully with Dense layers , in batches.
Following code works perfectly( Validation Accuracy 98.5%).
load
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# separate data into train and validation
from sklearn.model_selection import train_test_split
# Split the data
valid_per = 0.15
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=valid_per, shuffle= True)
N1 = X_train.shape[0] # training size
N2 = X_test.shape[0] # test size
N3 = X_valid.shape[0] # valid size
h = X_train.shape[1]
w = X_train.shape[2]
num_pixels = h*w
# reshape N1 samples to num_pixels
#x_train = X_train.reshape(N1, num_pixels).astype('float32') # shape is now (51000,784)
#x_test = X_test.reshape(N2, num_pixels).astype('float32') # shape is now (9000,784)
y_train = np_utils.to_categorical(y_train) #(51000,10): 10000 lables for 10 classes
y_valid = np_utils.to_categorical(y_valid) #(9000,10): 9000 labels for 10 classes
y_test = np_utils.to_categorical(y_test) # (10000,10): 10000 lables for 10 classes
num_classes = y_test.shape[1]
def baseline_model():
# create model
model = Sequential()
# flatten input to (N1,w*h) as fit_generator expects (N1,w*h), but dont' have x,y as inputs(so cant reshape)
model.add(Flatten(input_shape=(h,w,1)))
model.add(Dense(num_pixels, input_dim=num_pixels, kernel_initializer='normal', activation='relu'))
# Define output layer with softmax function
model.add(Dense(num_classes, kernel_initializer='normal', activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
model = baseline_model()
model.summary()
batch_size = 200
epochs = 20
steps_per_epoch_tr = int(N1/ batch_size) # 51000/200
steps_per_epoch_val = int(N3/batch_size)
# reshape to be [samples][width][height][ channel] for ImageData Gnerator->datagen.flow
x_t = X_train.reshape(N1, w, h, 1).astype('float32')
x_v = X_valid.reshape(N3, w, h, 1).astype('float32')
# define data preparation
#datagen = ImageDataGenerator(rescale=1./255,featurewise_center= True,featurewise_std_normalization=True,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
datagen = ImageDataGenerator(rescale=1./255,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
#datagen.fit(x_t)
#datagen.fit(x_v)
train_gen = datagen.flow(x_t, y_train, batch_size=batch_size)
valid_gen = datagen.flow(x_v,y_valid, batch_size=batch_size)
model.fit_generator(train_gen,steps_per_epoch = steps_per_epoch_tr,validation_data = valid_gen,
validation_steps = steps_per_epoch_val,epochs=epochs)
now, if i comment out line 53, and un-comment line 52, 54 and 55, I get validation accuracy of 1%.
so, this gives poor accuracy:
datagen = ImageDataGenerator(rescale=1./255,featurewise_center= True,featurewise_std_normalization=True,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
##datagen = ImageDataGenerator(rescale=1./255,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
datagen.fit(x_t)
datagen.fit(x_v)
If I un-comment line 52, but keep lines 54,55 commented out, accuracy is again 98.5%,
datagen = ImageDataGenerator(rescale=1./255,featurewise_center= True,featurewise_std_normalization=True,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
##datagen = ImageDataGenerator(rescale=1./255,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
#datagen.fit(x_t)
#datagen.fit(x_v)
but as per Keras documentation, we need lines 54 and 55 if we use featurewise_center.
So, I am confused what is going wrong.

You've used both rescaling and feature normalization which is the cause of the problem. Don't use rescaling when doing feature_normalization. This causes all the input values to the network to be negative. Remove, 'rescale=1./255' from ImageDataGenerator.
datagen = ImageDataGenerator(featurewise_center= True,featurewise_std_normalization=True,width_shift_range=0.1,height_shift_range=0.1) # scales x_t
datagen.fit(x_t)
Also, use separate ImageDataGenerators for train and validation since data augmentation is usually done only for training data. And, the mean/std is calculated on the training data and applied on the validation/test data.
Like this:
x_v = (x_v - datagen.mean)/(datagen.std + 1e-6)
datagen_valid = ImageDataGenerator(...)
valid_gen = datagen_valid.flow(x_v, y_valid, batch_size=batch_size)