Develop Reference

Can't load LSTM encoder decoder model Keras IndexError: list assignment index out of range

I'm using keras and functional API to create an encoder decoder model containing 2 LSTM layers each for binary classification.
The shape of input to encoder x is (samples, time steps, in_features) = (126144, 1, 113)
The shape of labels y is (samples, time steps, out_features) =(126144, 1, 2)
x and y are both numpy arrays.
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import Dense
from numpy import array
from numpy import array_equal
from tensorflow.keras.layers import Lambda
from tensorflow.keras import backend as K
n_timesteps_in = 1
n_features = 113
out_features = 2
numberOfLSTMunits = 256
def create_hard_coded_decoder_input_model(batch_size):
# The first part is encoder
encoder_inputs = Input(shape=(n_timesteps_in, n_features), name='encoder_inputs')
encoder_lstm = LSTM(numberOfLSTMunits, return_state=True,return_sequences=True,
name='encoder_lstm')
encoder_outputs, state_h1, state_c1 = encoder_lstm(encoder_inputs)
# Second LSTM Added
encoder_lstm2 = LSTM(numberOfLSTMunits, return_state=True, name='encoder_lstm2')
_, state_h2, state_c2 = encoder_lstm2(encoder_outputs)
states = [state_h1, state_c1, state_h2, state_c2]
decoder_inputs = Input(shape=(1, out_features), name='decoder_inputs')
decoder_lstm = LSTM(numberOfLSTMunits, return_sequences=True, return_state=True,
name='decoder_lstm')
# Second LSTM
decoder_lstm2 = LSTM(numberOfLSTMunits, return_sequences=True, return_state=True,
name='decoder_lstm2')
decoder_dense = Dense(out_features, activation='softmax', name='decoder_dense')
# New input decoder
all_outputs = []
decoder_input_data = np.zeros((batch_size, 1, out_features))
decoder_input_data[:, 0, 0] = -1
inputs = decoder_input_data
states1 = [state_h1, state_c1]
states2 = [state_h2, state_c2]
for _ in range(n_timesteps_in):
# Run the decoder on one time step
outputs, dh1, dc1 = decoder_lstm(inputs,initial_state= states1)
final, dh2, dc2 = decoder_lstm2(outputs, initial_state=states2)
outputs = decoder_dense(final)
# Store the current prediction (we will concatenate all predictions later)
all_outputs.append(outputs)
# Reinject the outputs as inputs for the next loop iteration
# as well as update the states
inputs = outputs
states1 = [state_h1, state_c1]
states2 = [state_h2, state_c2]
decoder_outputs = Lambda(lambda x: K.concatenate(x, axis=1))(all_outputs)
model = Model(encoder_inputs, decoder_outputs, name='model_encoder_decoder')
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
return model
I'm using 192 as the batch size.
After training, I save the model with this code:
model.save('lstm.h5')
When I load the model:
savedModel=load_model('lstm.h5')
I get this error:
/usr/local/lib/python3.7/dist-packages/keras/layers/recurrent.py in get_input_spec(shape)
547 batch_index, time_step_index = (1, 0) if self.time_major else (0, 1)
548 if not self.stateful:
--> 549 input_spec_shape[batch_index] = None
550 input_spec_shape[time_step_index] = None
551 return InputSpec(shape=tuple(input_spec_shape))
IndexError: list assignment index out of range
I've trying to solve the problem for days, but nothing seemed to work. I really appreciate any help with it.

Class weights for multi-output classification

I have a problem where i created a model like this :
from keras.models import Sequential, Model
from keras.layers import Dense, Dropout, Flatten
from keras.layers import LSTM, Conv1D, Input, MaxPooling1D, GlobalMaxPooling1D
from keras.layers.embeddings import Embedding
posts_input = Input(shape=(None,), dtype='int32', name='posts')
embedded_posts = Embedding(max_nb_words, embedding_vector_length, input_length=max_post_len)(posts_input)
x = Conv1D(128, 5, activation='relu')(embedded_posts)
x = Dropout(0.25)(x)
x = MaxPooling1D(5)(x)
x = Conv1D(256, 5, activation='relu')(x)
x = Conv1D(256, 5, activation='relu')(x)
x = Dropout(0.25)(x)
x = MaxPooling1D(5)(x)
x = Conv1D(256, 5, activation='relu')(x)
x = Conv1D(256, 5, activation='relu')(x)
x = Dropout(0.25)(x)
x = GlobalMaxPooling1D()(x)
x = Dense(128, activation='relu')(x)
Axe1_prediction = Dense(1, activation='sigmoid', name='axe1')(x)
Axe2_prediction = Dense(1, activation='sigmoid', name='axe2')(x)
Axe3_prediction = Dense(1, activation='sigmoid', name='axe3')(x)
Axe4_prediction = Dense(1, activation='sigmoid', name='axe4')(x)
model = Model(posts_input, [Axe1_prediction, Axe2_prediction, Axe3_prediction, Axe4_prediction])
as you can see, this model has 4 outputs.
Then i compile this model like this :
model.compile(optimizer='rmsprop',
loss=['binary_crossentropy',
'binary_crossentropy',
'binary_crossentropy',
'binary_crossentropy'],
metrics=['accuracy'])
For fitting this model, i think i need to set the class weights, so i create these :
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import class_weight
le = LabelEncoder()
y1 = le.fit_transform(df2["Axe1"])
y2 = le.fit_transform(df2["Axe2"])
y3 = le.fit_transform(df2["Axe3"])
y4 = le.fit_transform(df2["Axe4"])
cw1 = class_weight.compute_class_weight('balanced', np.unique(y1), y1)
cw2 = class_weight.compute_class_weight('balanced', np.unique(y2), y2)
cw3 = class_weight.compute_class_weight('balanced', np.unique(y3), y3)
cw4 = class_weight.compute_class_weight('balanced', np.unique(y4), y4)
But finally i don't know how to set this parameters in the fitting :
history = model.fit(X_train,
[y1_train, y2_train, y3_train, y4_train],
epochs=10,
validation_data=(X_val, [y1_val, y2_val, y3_val, y4_val]));
Could you help me by showing how i can add the "class_weights =" parameter ?

You have to use tensorflow 2.1 or earlier. The class weights fuctionality has beed removed after TF2.1 for multioutput models

If you want still to use tensorflow > 2.1, you need to define a custom loss instead, something similar as below :
from functools import partial
import tensorflow as tf
import keras.backend as K
def weighted_binary_crossentropy(target, output, weights_table):
# get the given weight
weights_vect = weights_table.lookup(target)
return K.binary_crossentropy(target, output) * weights_vect
# transform dictionnary of weights into lookup table that can be used
def to_lookup_table(dictionnary):
return tf.lookup.StaticHashTable(
tf.lookup.KeyValueTensorInitializer(
list(dictionnary.keys()), #[0,1]
list(dictionnary.values()), # corresponding weights
key_dtype=tf.int64,
value_dtype=tf.float32,
),
default_value=-1)
cw1 = ...
cw2 = ...
cw3 = ...
cw4 = ...
# define function where weights_table already defined
binary_crossentropy_1 = partial(weighted_binary_crossentropy, weights_table=to_lookup_table(cw1))
...
binary_crossentropy_4 = partial(weighted_binary_crossentropy, weights_table=to_lookup_table(cw4))
model.compile(optimizer='rmsprop',
loss=[binary_crossentropy_1, ..., binary_crossentropy_4],
metrics=['accuracy'])
Depending on how you define the weights dictionnary and the output of your model, you might have to change the type of the target, its shape, or removing the argmax. You might also to change the type of the keys and values in the to_lookup_table function.
For those who need a categorical crossentropy at the end, just replace K.binary_crossentropy by K.categorical_crossentropy.

Keras Functional API producing different Results from Keras Sequential API with LSTM Layer

I am comparing models built with both the Sequential and Functional API with Keras, and am finding that certain types of models produce very different results.
I am using the IMDB dataset for my model.
If I create a model with only Embedding and Dense layers, the results are very similar.
Here's an example:
# imports
from tensorflow.keras.layers import Dense, LSTM, Flatten, Embedding
from tensorflow.keras import Input, Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers.experimental.preprocessing import TextVectorization
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
CODE FOR SEQUENTIAL API:
df = pd.read_csv('url_for_imdb_csv_file_here')
df['sentiment'].replace(['positive', 'negative'], [1, 0], inplace=True)
X_train, X_test, y_train, y_test = train_test_split(df['review'], df['sentiment'], test_size=0.2, random_state=1985)
tokenizer = Tokenizer(num_words=10000)
tokenizer.fit_on_texts(X_train)
train_seq = tokenizer.texts_to_sequences(X_train)
test_seq = tokenizer.texts_to_sequences(X_test)
train_seq = pad_sequences(train_seq, maxlen=250)
test_seq = pad_sequences(test_seq, maxlen=250)
mod = Sequential([
Embedding(10000, 25, input_length=250),
Flatten(),
Dense(25, activation='relu'),
Dense(25, activation='relu'),
Dense(1, activation = 'sigmoid')
])
mod.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
mod.fit(train_seq, y_train.values, validation_split=0.2, epochs=2)
This produces an accuracy of ~ 88% after two epochs.
If I recreate the same model using the Functional API, I'll get the same results. Code below.
CODE USING SEQUENTIAL API
vectorizer = TextVectorization(max_tokens=10000, output_sequence_length=250)
vectorizer.adapt(X_train.values)
text_input = Input(shape=(1,), dtype='string')
x = vectorizer(text_input)
x = Embedding(10000, 25, input_length=250)(x)
x = Flatten()(x)
x = Dense(25, activation='relu')(x)
x = Dense(25, activation='relu')(x)
output = Dense(1, activation='sigmoid')(x)
mod1 = Model(text_input, output)
mod1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
mod1.fit(X_train.values, y_train.values, validation_split=0.2, epochs=5)
This gives me about the same results.
So far so good.
However, when I try and add LSTM layers, the results diverge significantly.
Here are the two examples I'm using. I'm swapping out one Dense layer with an LSTM layer and keeping everything else the same.
NEW SEQUENTIAL API CODE:
mod = Sequential([
Embedding(10000, 25, input_length=250),
LSTM(25),
Dense(25, activation='relu'),
Dense(1, activation = 'sigmoid')
])
mod.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
mod.fit(train_seq, y_train.values, validation_split=0.2, epochs=2)
This gives about the same results as the previous models.
However, when I recreate this version with the functional API, the accuracy peaks at around 64% on round 2, and then heads back down to ~50% after that.
NEW FUNCTIONAL API:
vectorizer = TextVectorization(max_tokens=10000, output_sequence_length=250)
vectorizer.adapt(X_train.values)
text_input = Input(shape=(1,), dtype='string')
x = vectorizer(text_input)
x = Embedding(10000, 25, input_length=250)(x)
x = LSTM(25)(x)
x = Dense(25, activation='relu')(x)
output = Dense(1, activation='sigmoid')(x)
mod1 = Model(text_input, output)
This does not produce the same results, even though it looks the same to me as the Sequential example.
Is there something I'm missing that does not make the two the same, or is this an idiosyncrasy of how the Functional API works?
Thank you.

Keras Models Detecting Hand Drawing Incorrect

I Use Keras, Tensorflow to Build QuickDraw Model From Dataset by Google
Quick Draw Data set Link. It's dataset content 345 classes.
I create Model Successfully But when I use that Model exported in my project , it's detect incorrect my drawing . When I draw an Apple that say Axe....
I don't know to fix that
I'm iOS developer. Biginner in CoreML
classification.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os as os
from keras.models import load_model
from keras.utils import np_utils
import model as md
import preparation as prep
import visualization as vis
from keras.utils import to_categorical
from keras import metrics
from keras.layers import LSTM
# ----------- dataset settings -----------
# number of instances per class used for train and test in total:
# should be smaller or equal than generated subset
INSTANCES_PER_CLASS = 5000
NUM_CLASS_LIMIT = 345 # limit of classes
# path of the dataset seperated in train and test
DATA_PATH = os.path.join(os.getcwd(), "../Make_model/dataset/train_test_20k/")
# path for all created files
MODEL_PATH = os.path.join(os.getcwd(), "models/" + str(NUM_CLASS_LIMIT) + "/" + str(INSTANCES_PER_CLASS) + "/")
# ----------- model settings -----------
MODEL_NAME = 'model.h5' # name for the freezed model
# input size
IMG_WIDTH = 28
IMG_HEIGHT = 28
IMG_SIZE = IMG_WIDTH * IMG_HEIGHT
IMG_DIM = 1
# training settings
EPOCHS = 10
BATCH_SIZE = 256
if __name__ == "__main__":
# create new directories if required
if not os.path.isdir(MODEL_PATH):
os.makedirs(MODEL_PATH)
# get the dataset
num_classes, x_train, x_test, y_train, y_test, classes_dict = prep.collect_data(NUM_CLASS_LIMIT)
print("trainingsset instances {}".format(x_train.shape))
print("trainingsset labels {}".format(y_train.shape))
# plot first test images
#vis.plot_first_n_images(x_test, y_test, classes_dict, 100)
# class representation as "binary" vector
y_train = np_utils.to_categorical(y_train, num_classes=num_classes)
y_test = np_utils.to_categorical(y_test, num_classes=num_classes)
# create or load keras model
if not os.path.isfile(MODEL_PATH + MODEL_NAME):
print("create model...")
model = md.build_model(input_shape=x_train.shape[1:], num_classes=num_classes)
# model.add(keras.layers.Dense(250, activation='tanh', input_dim=100))
else:
print("load existing model...")
model = load_model(MODEL_PATH + MODEL_NAME)
# score trained model using validation set
scores = model.evaluate(x_test, y_test, verbose=1)
print('test loss:', scores[0])
print('test accuracy:', scores[1])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
# print model information if desired
print(model.summary())
# model training from scratch or retrain by existing model
hist = model.fit(x_train, y_train, batch_size=BATCH_SIZE,
epochs=EPOCHS,
validation_data=[x_test, y_test],
shuffle=True)
#from keras.utils import plot_model
#plot_model(model, to_file=MODEL_PATH + 'model.png')
# evaluation process
print("evaluate model...")
# summarize history during training phase
# plot training and validation set accuracy
vis.plot_training_history_accuracy(hist)
# test set evaluation
scores = model.evaluate(x_test, y_test, verbose=1)
print(scores)
print('test loss:', scores[0])
print('test accuracy:', scores[1])
# create and plot confusion matrix
#y_pred = model.predict(x_test)
#vis.plot_confusion_matrix(y_pred, y_test, classes=list(classes_dict.values()))
# freeze the model (architecture and weights)
model.save(os.path.join(MODEL_PATH, MODEL_NAME))
print('saved trained model at {}'.format(os.path.join(MODEL_PATH, MODEL_NAME)))
Model.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from keras.models import Model
from keras.layers import Input, Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
def build_model(input_shape, num_classes):
"""
Builds the model architecture based on MNIST CNN
https://www.tensorflow.org/tutorials/estimators/cnn
Args:
input_spape: Input shape of the model
num_classes: Number of classes
Returns:
keras.models.Model: The created model
"""
inputs = Input(shape=input_shape)
x = Conv2D(32, (5,5), activation='relu')(inputs)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Conv2D(128, (3, 3), activation='relu')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(0.2)(x)
x = Flatten()(x)
x = Dense(512, activation='relu')(x)
x = Dense(256, activation='relu')(x)
predictions = Dense(num_classes, activation='softmax')(x)
return Model(inputs=inputs, outputs=predictions)

Multiple outputs in Keras

I have a problem which deals with predicting two outputs when given a vector of predictors.
Assume that a predictor vector looks like x1, y1, att1, att2, ..., attn, which says x1, y1 are coordinates and att's are the other attributes attached to the occurrence of x1, y1 coordinates. Based on this predictor set I want to predict x2, y2. This is a time series problem, which I am trying to solve using multiple regresssion.
My question is how do I setup keras, which can give me 2 outputs in the final layer.

from keras.models import Model
from keras.layers import *
#inp is a "tensor", that can be passed when calling other layers to produce an output
inp = Input((10,)) #supposing you have ten numeric values as input
#here, SomeLayer() is defining a layer,
#and calling it with (inp) produces the output tensor x
x = SomeLayer(blablabla)(inp)
x = SomeOtherLayer(blablabla)(x) #here, I just replace x, because this intermediate output is not interesting to keep
#here, I want to keep the two different outputs for defining the model
#notice that both left and right are called with the same input x, creating a fork
out1 = LeftSideLastLayer(balbalba)(x)
out2 = RightSideLastLayer(banblabala)(x)
#here, you define which path you will follow in the graph you've drawn with layers
#notice the two outputs passed in a list, telling the model I want it to have two outputs.
model = Model(inp, [out1,out2])
model.compile(optimizer = ...., loss = ....) #loss can be one for both sides or a list with different loss functions for out1 and out2
model.fit(inputData,[outputYLeft, outputYRight], epochs=..., batch_size=...)

You can make a model with multiple output with
the Functional API
by subclassing tf.keras.Model.
Here's an example of dual outputs (regression and classification) on the Iris Dataset, using the Functional API:
from sklearn.datasets import load_iris
from tensorflow.keras.layers import Dense
from tensorflow.keras import Input, Model
import tensorflow as tf
data, target = load_iris(return_X_y=True)
X = data[:, (0, 1, 2)]
Y = data[:, 3]
Z = target
inputs = Input(shape=(3,), name='input')
x = Dense(16, activation='relu', name='16')(inputs)
x = Dense(32, activation='relu', name='32')(x)
output1 = Dense(1, name='cont_out')(x)
output2 = Dense(3, activation='softmax', name='cat_out')(x)
model = Model(inputs=inputs, outputs=[output1, output2])
model.compile(loss={'cont_out': 'mean_absolute_error',
'cat_out': 'sparse_categorical_crossentropy'},
optimizer='adam',
metrics={'cat_out': tf.metrics.SparseCategoricalAccuracy(name='acc')})
history = model.fit(X, {'cont_out': Y, 'cat_out': Z}, epochs=10, batch_size=8)
Here's a simplified version:
from sklearn.datasets import load_iris
from tensorflow.keras.layers import Dense
from tensorflow.keras import Input, Model
data, target = load_iris(return_X_y=True)
X = data[:, (0, 1, 2)]
Y = data[:, 3]
Z = target
inputs = Input(shape=(3,))
x = Dense(16, activation='relu')(inputs)
x = Dense(32, activation='relu')(x)
output1 = Dense(1)(x)
output2 = Dense(3, activation='softmax')(x)
model = Model(inputs=inputs, outputs=[output1, output2])
model.compile(loss=['mae', 'sparse_categorical_crossentropy'], optimizer='adam')
history = model.fit(X, [Y, Z], epochs=10, batch_size=8)
Here's the same example, subclassing tf.keras.Model and with a custom training loop:
import tensorflow as tf
from tensorflow.keras.layers import Dense
from tensorflow.keras import Model
from sklearn.datasets import load_iris
tf.keras.backend.set_floatx('float64')
iris, target = load_iris(return_X_y=True)
X = iris[:, :3]
y = iris[:, 3]
z = target
ds = tf.data.Dataset.from_tensor_slices((X, y, z)).shuffle(150).batch(8)
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.d0 = Dense(16, activation='relu')
self.d1 = Dense(32, activation='relu')
self.d2 = Dense(1)
self.d3 = Dense(3, activation='softmax')
def call(self, x, training=None, **kwargs):
x = self.d0(x)
x = self.d1(x)
a = self.d2(x)
b = self.d3(x)
return a, b
model = MyModel()
loss_obj_reg = tf.keras.losses.MeanAbsoluteError()
loss_obj_cat = tf.keras.losses.SparseCategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
loss_reg = tf.keras.metrics.Mean(name='regression loss')
loss_cat = tf.keras.metrics.Mean(name='categorical loss')
error_reg = tf.keras.metrics.MeanAbsoluteError()
error_cat = tf.keras.metrics.SparseCategoricalAccuracy()
#tf.function
def train_step(inputs, y_reg, y_cat):
with tf.GradientTape() as tape:
pred_reg, pred_cat = model(inputs)
reg_loss = loss_obj_reg(y_reg, pred_reg)
cat_loss = loss_obj_cat(y_cat, pred_cat)
gradients = tape.gradient([reg_loss, cat_loss], model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
loss_reg(reg_loss)
loss_cat(cat_loss)
error_reg(y_reg, pred_reg)
error_cat(y_cat, pred_cat)
for epoch in range(50):
for xx, yy, zz in ds:
train_step(xx, yy, zz)
template = 'Epoch {:>2}, SCCE: {:>5.2f},' \
' MAE: {:>4.2f}, SAcc: {:>5.1%}'
print(template.format(epoch+1,
loss_cat.result(),
error_reg.result(),
error_cat.result()))
loss_reg.reset_states()
loss_cat.reset_states()
error_reg.reset_states()
error_cat.reset_states()