Develop Reference

Error generated during hyperparameter tuning

My trainX is (3350, 1, 8) and my TrainY is (3350, 2). I am getting some errors, but I don't understand what's the problem. I am getting this error when tuning the hyperparameters using the LSTM layer.
Output exceeds the size limit. Open the full output data in a text editor
--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) /tmp/ipykernel_4525/3330873115.py in <module>
----> 1 bayesian_opt_tuner.search(trainX, trainY,epochs=100,
2 #validation_data=(X_test, y_test)
3 validation_split=0.2,verbose=1)
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/base_tuner.py in search(self, *fit_args, **fit_kwargs)
181
182 self.on_trial_begin(trial)
--> 183 results = self.run_trial(trial, *fit_args, **fit_kwargs)
184 # `results` is None indicates user updated oracle in `run_trial()`.
185 if result is None:
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/tuner.py in run_trial(self, trial, *args, **kwargs)
293 callbacks.append(model_checkpoint)
294 copied_kwargs["callbacks"] = callbacks
--> 295 obj_value = self._build_and_fit_model(trial, *args, **copied_kwargs)
296
297 histories.append(obj_value)
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/tuner.py in
_build_and_fit_model(self, trial, *args, **kwargs)
220 hp = trial.hyperparameters
221 model = self._try_build(hp)
--> 222 results = self.hypermodel.fit(hp, model, *args, **kwargs) ...
File "/home/vareeshadki/anaconda3/lib/python3.9/site-packages/keras/backend.py", line 5238, in sparse_categorical_crossentropy
res = cf.nn.sparse_softmax_cross_entropy_with_logits( Node: 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' logits and labels must have the same first dimension, got logits shape [32,2] and labels shape [64] [[{{node sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_train_function_1763748]
My code:
import os
from kerastuner.tuners import BayesianOptimization
def build_model(hp):
model = tf.keras.Sequential()
hp_units = hp.Int('units', min_value=32, max_value=512, step=10)
model.add(LSTM(hp_units,activation='relu'))
model.add(Dense(2))
hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=['accuracy'])
return model
bayesian_opt_tuner = BayesianOptimization(
build_model,
objective='mse',
max_trials=3,
executions_per_trial=2,
directory=os.path.normpath('C:/keras_tuning'),
project_name='kerastuner_bayesian_poc',
overwrite=True)
bayesian_opt_tuner.search(trainX, trainY,epochs=100,
#validation_data=(X_test, y_test)
validation_split=0.2,verbose=1)

Using CalibratedClassifierCV returns error: TypeError: predict_proba() missing 1 required positional argument: 'x'

I am trying to train a model using Sequential model of Keras, and then calibrate it using scikit-learn's CalibratedClassifierCV. For this I use the KerasClassifier wrapper. Here is the code I use:
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,
patience=5, min_lr=0.000001)
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)
def create_model():
model_1 = Sequential()
n_cols = X_train_1.shape[1]
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal",input_shape=(n_cols,)))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(1, activation='sigmoid'))
opt = keras.optimizers.Nadam(lr=0.0001)
loss = tf.keras.losses.BinaryCrossentropy(reduction='sum')
model_1.compile(optimizer=opt, loss=loss)
return model_1
X_train_1, X_test_1, y_train_1, y_test_1, w_train_1, w_test_1 = train_test_split(scaled_X_1, y_1, w_1, test_size=0.35, random_state=42)
model_1 = KerasClassifier(build_fn=create_model, epochs=5, batch_size=5000, verbose=1)
history_1 = model_1.fit(X_train_1, y_train_1, callbacks=[reduce_lr, es], epochs=5, validation_split=0.35, batch_size=5000, sample_weight=w_train_1, verbose=1)
plt.plot(history_1.history['loss'])
plt.plot(history_1.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
calibrator_1 = CalibratedClassifierCV(model_1, cv='prefit')
calibrator_1.fit(X_test_1, y_test_1, sample_weight = w_test_1)
As you see, I clearly call the instance of CalibratedClassifierCV to calibrator_1 and then use fit(). Despite this I get this error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-21-1cab7be4f6d6> in <module>
46 calibrator_1 = CalibratedClassifierCV(model_1, cv='prefit')
47
---> 48 calibrator_1.fit(X_test_1, y_test_1, sample_weight = w_test_1)
~/.local/lib/python3.6/site-packages/sklearn/calibration.py in fit(self, X, y, sample_weight)
263 pred_method = _get_prediction_method(base_estimator)
264 n_classes = len(self.classes_)
--> 265 predictions = _compute_predictions(pred_method, X, n_classes)
266
267 calibrated_classifier = _fit_calibrator(
~/.local/lib/python3.6/site-packages/sklearn/calibration.py in _compute_predictions(pred_method, X, n_classes)
499 (X.shape[0], 1).
500 """
--> 501 predictions = pred_method(X=X)
502 if hasattr(pred_method, '__name__'):
503 method_name = pred_method.__name__
TypeError: predict_proba() missing 1 required positional argument: 'x'
Does anyone spot any errors here?

This seems like an issue with the newer version of scikit-learn (0.24). I installed the older 0.23 version and it runs well with the same exact code.

Get 'function' object has no attribute 'loss' when doing GridsearchCV

I want to try GridsearchCV on my model, my import is :
from keras import models
from keras import layers
from keras import regularizers
from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier
my code is:
def build_model(X_train = X_train,neurons=4,optimizer='Adam'):
model = models.Sequential()
model.add(layers.Dense(X_train.shape[1], kernel_regularizer=regularizers.l2(0.001),
activation='relu', input_shape=(X_train.shape[1],)))
model.add(layers.BatchNormalization())
model.add(layers.Dense(neurons, kernel_regularizer=regularizers.l2(0.001), activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['accuracy'])
return build_model
model = KerasClassifier(build_fn=build_model, verbose=1)
# define the grid search parameters
batch_size = [16, 32, 64]
epochs = [50, 100]
param_grid = dict(batch_size=batch_size, epochs=epochs)
# search the grid
grid = GridSearchCV(estimator=model,
param_grid=param_grid,
cv=10,
verbose=2)
grid_result = grid.fit(X_train, y_train)
but I get a bug as below:
AttributeError Traceback (most recent call last)
<ipython-input-93-2eb813d3aab7> in <module>
12 verbose=2) # include n_jobs=-1 if you are using CPU
13
---> 14 grid_result = grid.fit(X_train, y_train)
15
16 print(model)
/anaconda3/envs/lance/lib/python3.7/site-packages/keras/wrappers/scikit_learn.py in fit(self, x, y, sample_weight, **kwargs)
208 if sample_weight is not None:
209 kwargs['sample_weight'] = sample_weight
--> 210 return super(KerasClassifier, self).fit(x, y, **kwargs)
211
212 def predict(self, x, **kwargs):
/anaconda3/envs/lance/lib/python3.7/site-packages/keras/wrappers/scikit_learn.py in fit(self, x, y, **kwargs)
141 self.model = self.build_fn(**self.filter_sk_params(self.build_fn))
142
--> 143 loss_name = self.model.loss
144 if hasattr(loss_name, '__name__'):
145 loss_name = loss_name.__name__
AttributeError: 'function' object has no attribute 'loss'
I can't understand what the bug is, and I'm sure the data processing is correct because it goes well without grid search, did I do something wrong?

At the end of the build_model function, you write return build_model. This returns a reference to the function itself, not to the model object you've been building so far. I'm pretty sure you want return model instead.

ValueError: Unknown loss function:focal_loss_fixed when loading model with my custom loss function

I designed my own loss function. However when trying to revert to the best model encountered during training with
model = load_model("lc_model.h5")
I got the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-105-9d09ef163b0a> in <module>
23
24 # revert to the best model encountered during training
---> 25 model = load_model("lc_model.h5")
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in load_model(filepath, custom_objects, compile)
417 f = h5dict(filepath, 'r')
418 try:
--> 419 model = _deserialize_model(f, custom_objects, compile)
420 finally:
421 if opened_new_file:
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in _deserialize_model(f, custom_objects, compile)
310 metrics=metrics,
311 loss_weights=loss_weights,
--> 312 sample_weight_mode=sample_weight_mode)
313
314 # Set optimizer weights.
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
137 loss_functions = [losses.get(l) for l in loss]
138 else:
--> 139 loss_function = losses.get(loss)
140 loss_functions = [loss_function for _ in range(len(self.outputs))]
141 self.loss_functions = loss_functions
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in get(identifier)
131 if isinstance(identifier, six.string_types):
132 identifier = str(identifier)
--> 133 return deserialize(identifier)
134 if isinstance(identifier, dict):
135 return deserialize(identifier)
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in deserialize(name, custom_objects)
112 module_objects=globals(),
113 custom_objects=custom_objects,
--> 114 printable_module_name='loss function')
115
116
C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
163 if fn is None:
164 raise ValueError('Unknown ' + printable_module_name +
--> 165 ':' + function_name)
166 return fn
167 else:
ValueError: Unknown loss function:focal_loss_fixed
Here is the neural network :
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
model = create_model(x_train.shape[1], y_train.shape[1])
epochs = 35
batch_sz = 64
print("Beginning model training with batch size {} and {} epochs".format(batch_sz, epochs))
checkpoint = ModelCheckpoint("lc_model.h5", monitor='val_acc', verbose=0, save_best_only=True, mode='auto', period=1)
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.constraints import maxnorm
def create_model(input_dim, output_dim):
print(output_dim)
# create model
model = Sequential()
# input layer
model.add(Dense(100, input_dim=input_dim, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# hidden layer
model.add(Dense(60, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# output layer
model.add(Dense(output_dim, activation='softmax'))
# Compile model
# model.compile(loss='categorical_crossentropy', loss_weights=None, optimizer='adam', metrics=['accuracy'])
model.compile(loss=focal_loss(alpha=1), loss_weights=None, optimizer='adam', metrics=['accuracy'])
return model
# train the model
history = model.fit(x_train.as_matrix(),
y_train.as_matrix(),
validation_split=0.2,
epochs=epochs,
batch_size=batch_sz, # Can I tweak the batch here to get evenly distributed data ?
verbose=2,
class_weight = weights, # class_weight tells the model to "pay more attention" to samples from an under-represented fraud class.
callbacks=[checkpoint])
# revert to the best model encountered during training
model = load_model("lc_model.h5")
And here is my loss function:
import tensorflow as tf
def focal_loss(gamma=2., alpha=4.):
gamma = float(gamma)
alpha = float(alpha)
def focal_loss_fixed(y_true, y_pred):
"""Focal loss for multi-classification
FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
Notice: y_pred is probability after softmax
gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
Focal Loss for Dense Object Detection
https://arxiv.org/abs/1708.02002
Arguments:
y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
y_pred {tensor} -- model's output, shape of [batch_size, num_cls]
Keyword Arguments:
gamma {float} -- (default: {2.0})
alpha {float} -- (default: {4.0})
Returns:
[tensor] -- loss.
"""
epsilon = 1.e-9
y_true = tf.convert_to_tensor(y_true, tf.float32)
y_pred = tf.convert_to_tensor(y_pred, tf.float32)
model_out = tf.add(y_pred, epsilon)
ce = tf.multiply(y_true, -tf.log(model_out))
weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
fl = tf.multiply(alpha, tf.multiply(weight, ce))
reduced_fl = tf.reduce_max(fl, axis=1)
return tf.reduce_mean(reduced_fl)
return focal_loss_fixed
# model.compile(loss=focal_loss(alpha=1), optimizer='nadam', metrics=['accuracy'])
# model.fit(X_train, y_train, epochs=3, batch_size=1000)

You have to load the custom_objects of focal_loss_fixed as shown below:
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
However, if you wish to just perform inference with your model and not further optimization or training your model, you can simply wish to ignore the loss function like this:
model = load_model("lc_model.h5", compile=False)

The answer by #Prasad is great, but I would like to add a little explanation and a little correction:
while mentioning your custom loss function in the custom_objects dictionary you don't have to call your loss function, as it can give some parameter missing errors.
# Instead of this
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
# ty this without calling your loss function
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss})
Also, the thing I would like to add here is that you have to use the name of your custom loss function as a key and your function object as its value in custom_objects. I know this is very basic but the thing to mention but I hope this will be helpful to someone.

Keras: "An operation has `None` for gradient" in a simple network

I am getting started with Keras and trying to implement a simple network to test it on CIFAR10 (32x32 images, 10 classes), but the model fails to fit the dataset.
Model code:
import keras, keras.layers as L
input_shape = [32, 32, 3]
n_inputs = 32 * 32 * 3
model = keras.models.Sequential()
model.add(L.InputLayer(input_shape))
model.add(L.Flatten())
model.add(L.Dense(n_inputs, activation='relu'))
model.add(L.Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss=keras.metrics.sparse_categorical_accuracy,
metrics=['accuracy'])
model.fit(x=X_train, y=y_train,
epochs=10)
Error:
ValueError Traceback (most recent call last)
<ipython-input-16-be14b685c358> in <module>
4 model.fit(x=X_train, y=y_train,
5 # validation_data=[X_val, y_val],
----> 6 epochs=10)
...\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
-> 1010 self._make_train_function()
...\keras\engine\training.py in _make_train_function(self)
--> 509 loss=self.total_loss)
...\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
---> 91 return func(*args, **kwargs)
...\keras\optimizers.py in get_updates(self, loss, params)
473 #interfaces.legacy_get_updates_support
474 def get_updates(self, loss, params):
--> 475 grads = self.get_gradients(loss, params)
476 self.updates = [K.update_add(self.iterations, 1)]
477
...\keras\optimizers.py in get_gradients(self, loss, params)
---> 91 raise ValueError('An operation has `None` for gradient. '
ValueError: An operation has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.
I have seen similar snippets working correctly in tutorials and cannot find the mistake. What is wrong with this model?