Develop Reference

WARNING:tensorflow:Model was constructed with shape (None, ....)

I have a classification problem (0 or 1) with 78 features.
Let's say I have that data in a dataframe with 78 columns and 202 rows, each cell value holding an integer in the range [0, infinity).
Trying to achieve prediction with TensorFlow, when I fit my model I get the following warning:
WARNING:tensorflow:Model was constructed with shape (None, 101, 78) for input Tensor("input_2:0", shape=(None, 101, 78), dtype=float32), but it was called on an input with incompatible shape (101, 78).
I would have thought my shape definition was correct, so why am I getting this warning? Perhaps I'm misunderstanding how to use the framework.
X_train, X_test, y_train, y_test = train_test_split(df_x, series_y, random_state=1, test_size=0.5)
numpy_x_train = X_train.to_numpy()
numpy_y_train = y_train.to_numpy()
numpy_x_test = X_test.to_numpy()
shape_x = len(X_train)
shape_y = len(X_train.columns)
inputs = keras.Input(shape=(shape_x, shape_y))
x = Rescaling(scale=1.0 / 255)(inputs)
num_classes = 1
outputs = layers.Dense(num_classes, activation="softmax")(x)
model = keras.Model(inputs=inputs, outputs=outputs)
processed_data = model(numpy_x_train)
optimiser = keras.optimizers.RMSprop(learning_rate=1e-3)
loss = keras.losses.CategoricalCrossentropy()
model.compile(optimizer=optimiser, loss=loss)
history = model.fit(numpy_x_train, numpy_y_train, batch_size=32, epochs=10)

here a full example based on your problem. you have 2D data so in the input layer you have to specify only the feature dimension and not also the sample dimension. you also are carrying out a binary classification task so the best choice is to use a final dense layer with 1 dimension, a sigmoid activation function, and binary crossentropy as a loss. the predicted class will be 1 if the prob are > 0.5 otherwise it is 0
from tensorflow import keras
import numpy as np
# create dummy data
train_size, test_size = 101, 101
n_features = 78
num_classes = 2 # 0 or 1
numpy_x_train = np.random.uniform(0,256, (train_size,n_features))
numpy_y_train = np.random.randint(0,num_classes,train_size)
numpy_x_test = np.random.uniform(0,256, (test_size,n_features))
numpy_y_test = np.random.randint(0,num_classes,test_size)
# rescaling data
numpy_x_train = numpy_x_train / 255
numpy_x_test = numpy_x_test / 255
# define model
inputs = keras.layers.Input(shape=(numpy_x_train.shape[1],))
outputs = keras.layers.Dense(1, activation="sigmoid")(inputs)
optimiser = keras.optimizers.RMSprop(learning_rate=1e-3)
loss = keras.losses.BinaryCrossentropy()
model = keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=optimiser, loss=loss)
history = model.fit(numpy_x_train, numpy_y_train, batch_size=32, epochs=10)
# get test predictions
test_prob = model.predict(numpy_x_test).ravel()
test_class = (test_prob>0.5)+0 # if prob > 0.5 is 1 else is 0

model.fit() Keras Classification Multiple Inputs-Single Output gives error: AttributeError: 'NoneType' object has no attribute 'fit'

I am constructing a Keras Classification model with Multiple Inputs (3 actually) to predict one single output. Specifically, my 3 inputs are:
Actors
Plot Summary
Relevant Movie Features
Output:
Genre tags
All the above inputs and the single output are related to 10,000 IMDB Movies.
Even though the creation of the model is successful, when I try to fit the model on my three different X_train's I get an Attribute error. I have one X_train and X_test for actors, a different one X_train and X_test for plot summary and a different one X_train and X_test for movie features. My y_train and y_test are the same for all the inputs.
Python Code (create the multiple input keras)
def kera_multy_classification_model():
sentenceLength_actors = 15
vocab_size_frequent_words_actors = 20001
sentenceLength_plot = 23
vocab_size_frequent_words_plot = 17501
sentenceLength_features = 69
vocab_size_frequent_words_features = 20001
model = keras.Sequential(name='Multy-Input Keras Classification model')
actors = keras.Input(shape=(sentenceLength_actors,), name='actors_input')
plot = keras.Input(shape=(sentenceLength_plot,), name='plot_input')
features = keras.Input(shape=(sentenceLength_features,), name='features_input')
emb1 = layers.Embedding(input_dim = vocab_size_frequent_words_actors + 1,
# based on keras documentation input_dim: int > 0. Size of the vocabulary, i.e. maximum integer index + 1.
output_dim = Keras_Configurations_model1.EMB_DIMENSIONS,
# int >= 0. Dimension of the dense embedding
embeddings_initializer = 'uniform',
# Initializer for the embeddings matrix.
mask_zero = False,
input_length = sentenceLength_actors,
name="actors_embedding_layer")(actors)
encoded_layer1 = layers.LSTM(100)(emb1)
emb2 = layers.Embedding(input_dim = vocab_size_frequent_words_plot + 1,
output_dim = Keras_Configurations_model2.EMB_DIMENSIONS,
embeddings_initializer = 'uniform',
mask_zero = False,
input_length = sentenceLength_plot,
name="plot_embedding_layer")(plot)
encoded_layer2 = layers.LSTM(100)(emb2)
emb3 = layers.Embedding(input_dim = vocab_size_frequent_words_features + 1,
output_dim = Keras_Configurations_model3.EMB_DIMENSIONS,
embeddings_initializer = 'uniform',
mask_zero = False,
input_length = sentenceLength_features,
name="features_embedding_layer")(features)
encoded_layer3 = layers.LSTM(100)(emb3)
merged = layers.concatenate([encoded_layer1, encoded_layer2, encoded_layer3])
layer_1 = layers.Dense(Keras_Configurations_model1.BATCH_SIZE, activation='relu')(merged)
output_layer = layers.Dense(Keras_Configurations_model1.TARGET_LABELS, activation='softmax')(layer_1)
model = keras.Model(inputs=[actors, plot, features], outputs=output_layer)
print(model.output_shape)
print(model.summary())
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
Python Code (fit the multiple input keras on my inputs)
def fit_keras_multy_input(model, x_train_seq_actors, x_train_seq_plot, x_train_seq_features, x_test_seq_actors, x_test_seq_plot, x_test_seq_features, y_train, y_test):
s = time()
fit_model = model.fit([x_train_seq_actors, x_train_seq_plot, x_train_seq_features], y_train,
epochs=Keras_Configurations_model1.NB_EPOCHS,
verbose = Keras_Configurations_model1.VERBOSE,
batch_size=Keras_Configurations_model1.BATCH_SIZE,
validation_data=([x_test_seq_actors, x_test_seq_plot, x_test_seq_features], y_test),
callbacks=callbacks)
duration = time() - s
print("\nTraining time finished. Duration {} secs".format(duration))
Model's Structure
Error produced
Note: Please not that each of the X_train and X_test are sequences of numbers. (Text that have been tokenized)
Having done a little research, the problem starts in the model.compile() function. Although, I am not sure what should be changed in my model's compile function so as to fix this.
Thank you in advance for any advice or help on this matter. Feel free to ask on the comments any additional information that I may have missed, to make this question more complete.

Your function kera_multy_classification_model() doesn't return anything, so after model = kera_multy_classification_model(), you get model == None as your function returns nothing. And None's type is NoneType and it really doesn't have a method called fit().
Just add return model in the end of kera_multy_classification_model().

How to set up LSTM network for predict multi-sequence?

I am learning how to set up the RNN-LSTM network for prediction. I have created the dataset with one input variable.
x y
1 2.5
2 6
3 8.6
4 11.2
5 13.8
6 16.4
...
By the following python code, I have created the window data, like [x(t-2), x(t-1), x(t)] to predict [y(t)]:
df= pd.read_excel('dataset.xlsx')
# split a univariate dataset into train/test sets
def split_dataset(data):
train, test = data[:-328], data[-328:-6]
return train, test
train, test = split_dataset(df.values)
# scale train and test data
def scale(train, test):
# fit scaler
scaler = MinMaxScaler(feature_range=(0,1))
scaler = scaler.fit(train)
# transform train
#train = train.reshape(train.shape[0], train.shape[1])
train_scaled = scaler.transform(train)
# transform test
#test = test.reshape(test.shape[0], test.shape[1])
test_scaled = scaler.transform(test)
return scaler, train_scaled, test_scaled
scaler, train_scaled, test_scaled = scale(train, test)
def to_supervised(train, n_input, n_out=7):
# flatten data
data = train
X, y = list(), list()
in_start = 0
# step over the entire history one time step at a time
for _ in range(len(data)):
# define the end of the input sequence
in_end = in_start + n_input
out_end = in_end + n_out
# ensure we have enough data for this instance
if out_end <= len(data):
x_input = data[in_start:in_end, 0]
x_input = x_input.reshape((len(x_input), 1))
X.append(x_input)
y.append(data[in_end:out_end, 0])
# move along one time step
in_start += 1
return np.array(X), np.array(y)
train_x, train_y = to_supervised(train_scaled, n_input = 3, n_out = 1)
test_x, test_y = to_supervised(test_scaled, n_input = 3, n_out = 1)
verbose, epochs, batch_size = 0, 20, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
model = Sequential()
model.add(LSTM(200, return_sequences= False, input_shape = (train_x.shape[1],train_x.shape[2])))
model.add(Dense(1))
model.compile(loss = 'mse', optimizer = 'adam')
history = model.fit(train_x, train_y, epochs=epochs, verbose=verbose, validation_data = (test_x, test_y))
However, I have other questions about this:
Q1: What is the meaning of units in LSTM? [model.add(LSTM(units, ...))]
(I have tried different units for the model, it would be more accurate as units increased.)
Q2: How many layers should I set?
Q3: How can I predict multi-steps ? e.g base on (x(t),x(t-1)) to predict y(t), y(t+1) I have tried to set the n_out = 2 in the to_supervised function, but when I applied the same method, it returned the error
train_x, train_y = to_supervised(train_scaled, n_input = 3, n_out = 2)
test_x, test_y = to_supervised(test_scaled, n_input = 3, n_out = 2)
verbose, epochs, batch_size = 0, 20, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
model = Sequential()
model.add(LSTM(200, return_sequences= False, input_shape = (train_x.shape[1],train_x.shape[2])))
model.add(Dense(1))
model.compile(loss = 'mse', optimizer = 'adam')
history = model.fit(train_x, train_y, epochs=epochs, verbose=verbose, validation_data = (test_x, test_y))
ValueError: Error when checking target: expected dense_27 to have shape (1,) but got array with shape (2,)
Q3(cont): What should I add or change in the model setting?
Q3(cont): What is the return_sequences ? When should I set True?

Q1. Units in LSTM is the number of neurons in your LSTM layer.
Q2. That depends on your model / data. Try changing them around to see the effect.
Q3. That depends which apporach you take.
Q4. Ideally you'll want to predict a single time step every time.
It is possible to predict several at a time, but in my experience you will get better results like as i have described below
e.g
use y(t-1), y(t) to predict y_hat(t+1)
THEN
use y(t), y_hat(t+1) to predict y_hat(t+2)
Are you sure you're actually using X to predict Y in this case?
how does train x/y and test x/y look like?

Re Q1: It is the number of LSTM cells (=LSTM units), which consist of several neurons themselves but have (in the standard case as given) only one output each. Thus, the number of units corresponds directly to the dimensionality of your output.

How to set up the LSTM network?

I am learning how to set up the RNN-LSTM network for prediction. I have created the dataset with one-variables.
x y
1 2.5
2 6
3 8.6
4 11.2
5 13.8
6 16.4
...
And the relationship of the y(t) = 2.5x(t) + x(t-1) -0.9*x(t-2). And I am trying to set up the RNN-LSTM to learn the pattern, but it occurred the error of my program. My program is like below:
df= pd.read_excel('dataset.xlsx')
def split_dataset(data):
# split into standard weeks
train, test = data[:-328], data[-328:-6]
# restructure into windows of weekly data
train = np.array(np.split(train, len(train)/1))
test = np.array(np.split(test, len(test)/1))
return train, test
verbose, epochs, batch_size = 0, 20, 16
train, test = split_dataset(df.values)
train_x, train_y = train[:,:,0], train[:,:,1]
model = Sequential()
model.add(LSTM(200, return_sequences=True, input_shape = train_x.shape))
model.compile(loss='mse', optimizer='adam')
It occurred the ValueError:
ValueError: Error when checking input: expected lstm_35_input to have 3 dimensions, but got array with shape (8766, 1)
Any experienced DS or pythoner can teach me how to set up the network?
Thanks

For LSTM based RNN, the input should be of 3 dimensions (batch, time, data_point). I assume index of your x variable is its time. In this case, you have to transform your input into batches of some window, say a window of 3, then your input is:
batch # input target
0 x[0:3] y[3]
1 x[1:4] y[4]
2 x[2:5] y[5]
Note: your y's start from t=3 since you are using last 3 time steps to predict the next 4th value. If your y's are already calculated from the last three time steps as you have said, then y's should start from 0 index, i.e. at batch 0 you have y[0] as the target
UPDATE as per below comment
If you want to have multiple sequences, then you can model it as a sequence to sequence problem and will be an N to M mapping, you need five x values to predict three y's:
batch # input target
0 x[0:5] y[3:6]
1 x[1:6] y[4:7]
2 x[2:7] y[5:8]

In current, I have created the data window and it look like work for I mentioned case.
Below is my code:
df= pd.read_excel('dataset.xlsx')
# split a univariate dataset into train/test sets
def split_dataset(data):
train, test = data[:-328], data[-328:-6]
return train, test
train, test = split_dataset(df.values)
# scale train and test data to [-1, 1]
def scale(train, test):
# fit scaler
scaler = MinMaxScaler(feature_range=(0,1))
scaler = scaler.fit(train)
# transform train
#train = train.reshape(train.shape[0], train.shape[1])
train_scaled = scaler.transform(train)
# transform test
#test = test.reshape(test.shape[0], test.shape[1])
test_scaled = scaler.transform(test)
return scaler, train_scaled, test_scaled
scaler, train_scaled, test_scaled = scale(train, test)
def to_supervised(train, n_input, n_out=7):
# flatten data
data = train
X, y = list(), list()
in_start = 0
# step over the entire history one time step at a time
for _ in range(len(data)):
# define the end of the input sequence
in_end = in_start + n_input
out_end = in_end + n_out
# ensure we have enough data for this instance
if out_end <= len(data):
x_input = data[in_start:in_end, 0]
x_input = x_input.reshape((len(x_input), 1))
X.append(x_input)
y.append(data[in_end:out_end, 0])
# move along one time step
in_start += 1
return np.array(X), np.array(y)
train_x, train_y = to_supervised(train_scaled, n_input = 3, n_out = 1)
test_x, test_y = to_supervised(test_scaled, n_input = 3, n_out = 1)
verbose, epochs, batch_size = 0, 20, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
model = Sequential()
model.add(LSTM(200, return_sequences= False, input_shape = (train_x.shape[1],train_x.shape[2])))
model.add(Dense(1))
model.compile(loss = 'mse', optimizer = 'adam')
history = model.fit(train_x, train_y, epochs=epochs, verbose=verbose, validation_data = (test_x, test_y))
However, I have other questions about this:
Q1: What is the meaning of units in LSTM? [model.add(LSTM(units, ...))]
(I have tried different units for the model, it would be more accurate as units increased.)
Q2: How many layers should I set?
Q3: How can I predict multi-steps ? e.g base on (x(t),x(t-1)) to predict y(t), y(t+1)
I have tried to set the n_out = 2 in the to_supervised function, but when I applied the same method, it returned the error
train_x, train_y = to_supervised(train_scaled, n_input = 3, n_out = 2)
test_x, test_y = to_supervised(test_scaled, n_input = 3, n_out = 2)
verbose, epochs, batch_size = 0, 20, 16
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
model = Sequential()
model.add(LSTM(200, return_sequences= False, input_shape = (train_x.shape[1],train_x.shape[2])))
model.add(Dense(1))
model.compile(loss = 'mse', optimizer = 'adam')
history = model.fit(train_x, train_y, epochs=epochs, verbose=verbose, validation_data = (test_x, test_y))
ValueError: Error when checking target: expected dense_27 to have shape (1,) but got array with shape (2,)
Q3(cont): What should I add or change in the model setting?
Q3(cont): What is the return_sequences ? When should I set True?

Neural network has <0.001 validation and testing loss but 0% accuracy when doing a prediction

I've been training an MLP to predict the time remaining on an assembly sequence. The Training loss, Validation loss and MSE are all less 0.001, however, when I try to do a prediction with one of the datasets I trained the network with the it can't correctly identify any of the outputs from the set of inputs. What am I doing wrong that is producing this error?
I am also struggling to understand how, when the model is deployed, how do I perform the scaling of the result for one prediction? scaler.inverse_transform won't work because the data for that scaler used during training has been lost as the prediction would be done in a separate script to the training using the model the training produced. Is this information saved in the model builder?
I have tried to change the batch size during training, rounding the time column of the dataset to the nearest second (previously was 0.1 seconds), trained over 50, 100 and 200 epochs and I always end up with no correct predictions. I am also training an LSTM to see which is more accurate but that is also having the same issue. The dataset is split 70-30 training-testing and then training is then split 75-25 into training and validation.
Data scaling and model training code:
def scale_data(training_data, training_data_labels, testing_data, testing_data_labels):
# Create X and Y scalers between 0 and 1
x_scaler = MinMaxScaler(feature_range=(0, 1))
y_scaler = MinMaxScaler(feature_range=(0, 1))
# Scale training data
x_scaled_training = x_scaler.fit_transform(training_data)
y_scaled_training = y_scaler.fit_transform(training_data_labels)
# Scale testing data
x_scaled_testing = x_scaler.transform(testing_data)
y_scaled_testing = y_scaler.transform(testing_data_labels)
return x_scaled_training, y_scaled_training, x_scaled_testing, y_scaled_testing
def train_model(training_data, training_labels, testing_data, testing_labels, number_of_epochs, number_of_columns):
model_hidden_neuron_number_list = []
model_repeat_list = []
model_error_rate_list = []
for hidden_layer_1_units in range(int(np.floor(number_of_columns / 2)), int(np.ceil(number_of_columns * 2))):
print("Training starting, number of hidden units = %d" % hidden_layer_1_units)
for repeat in range(1, 6):
print("Repeat %d" % repeat)
model = k.Sequential()
model.add(Dense(hidden_layer_1_units, input_dim=number_of_columns,
activation='relu', name='hidden_layer_1'))
model.add(Dense(1, activation='linear', name='output_layer'))
model.compile(loss='mean_squared_error', optimizer='adam')
# Train Model
model.fit(
training_data,
training_labels,
epochs=number_of_epochs,
shuffle=True,
verbose=2,
callbacks=[logger],
batch_size=1024,
validation_split=0.25
)
# Test Model
test_error_rate = model.evaluate(testing_data, testing_labels, verbose=0)
print("Error on testing data is %.3f" % test_error_rate)
model_hidden_neuron_number_list.append(hidden_layer_1_units)
model_repeat_list.append(repeat)
model_error_rate_list.append(test_error_rate)
# Save Model
model_builder = tf.saved_model.builder.SavedModelBuilder("MLP/models/{hidden_layer_1_units}/{repeat}".format(hidden_layer_1_units=hidden_layer_1_units, repeat=repeat))
inputs = {
'input': tf.saved_model.build_tensor_info(model.input)
}
outputs = { 'time_remaining':tf.saved_model.utils.build_tensor_info(model.output)
}
signature_def = tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs,
outputs=outputs, method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
)
model_builder.add_meta_graph_and_variables(
K.get_session(),
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map={tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_def
}
)
model_builder.save()
And then to do a prediction:
file_name = top_level_file_path + "./MLP/models/19/1/"
testing_dataset = pd.read_csv(file_path + os.listdir(file_path)[0])
number_of_rows = len(testing_dataset.index)
number_of_columns = len(testing_dataset.columns)
newcol = [number_of_rows]
max_time = testing_dataset['Time'].max()
for j in range(0, number_of_rows - 1):
newcol.append(max_time - testing_dataset.iloc[j].iloc[number_of_columns - 1])
x_scaler = MinMaxScaler(feature_range=(0, 1))
y_scaler = MinMaxScaler(feature_range=(0, 1))
# Scale training data
data_scaled = x_scaler.fit_transform(testing_dataset)
labels = pd.read_csv("Labels.csv")
labels_scaled = y_scaler.fit_transform(labels)
signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
input_key = 'input'
output_key = 'time_remaining'
with tf.Session(graph=tf.Graph()) as sess:
saved_model = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], file_name)
signature = saved_model.signature_def
x_tensor_name = signature[signature_key].inputs[input_key].name
y_tensor_name = signature[signature_key].outputs[output_key].name
x = sess.graph.get_tensor_by_name(x_tensor_name)
y = sess.graph.get_tensor_by_name(y_tensor_name)
#np.expand_dims(data_scaled[600], axis=0)
predictions = sess.run(y, {x: data_scaled})
predictions = y_scaler.inverse_transform(predictions)
#print(np.round(predictions, 2))
correct_result = 0
for i in range(0, number_of_rows):
correct_result = 0
print(np.round(predictions[i]), " ", np.round(newcol[i]))
if np.round(predictions[i]) == np.round(newcol[i]):
correct_result += 1
print((correct_result/number_of_rows)*100)
The output of the first row should 96.0 but it produces 110.0, the last should be 0.1 but is -40.0 when no negatives appear in the dataset.

You can't compute accuracy when you do regression. Compute the mean squared error on the test set as well.
Second, when it comes to the scalers, you always do scaler.fit_transform on the training date so the scaler will compute the parameters (in this case min and max if you use min-max scaler) on the training data. Then, when performing inference on the test set, you should only do scaler.transform prior to feeding the data to the model.