Develop Reference

How to compute Receiving Operating Characteristic (ROC) and AUC using Keras Tuner?

I want to add additional ROC and AUC custom metrics to my Keras model. I tried to adapt the solution provided here but the code gave me an error.
The PerformanceVisualizationCallback plots the ROC-AUC curve.
from sklearn.metrics import roc_auc_score
from keras.callbacks import Callback
Callback:
class PerformanceVisualizationCallback(Callback):
def __init__(self, model, validation_data, dat_dir):
super().__init__()
self.model = model
self.X = X_train
self.y = y_train
self.validation_data = validation_data
os.makedirs(dat_dir, exist_ok=True)
self.dat_dir = dat_dir
def on_epoch_end(self, epoch, logs={}):
y_pred_train = self.model.predict_proba(self.x)
roc_train = auc_roc_score(self.y, y_pred_train)
y_pred_val = self.model.predict_proba(self.validation_data[0])
roc_val = auc_roc_score(self.y_val, y_pred_val)
#y_pred = tf.constant(self.model.predict(self.validation_data[0])).numpy()
#y_true = self.validation_data[1]
#y_pred_class = tf.math.argmax(y_pred, axis=1).numpy()
clf = SVC(random_state=0)
clf.fit(tf.constant( self.validation_data).numpy(), tf.cast( tf.linspace( 0, 19, 20, name='linspace', axis=0 ), dtype=tf.int64 ).numpy())
predictions = clf.predict(tf.constant( self.validation_data).numpy())
cm = sklearn.metrics.confusion_matrix(
[tf.math.argmax(self.validation_data[1], axis=1).numpy()[0], tf.math.argmax(self.validation_data[2], axis=1).numpy()[0],
tf.math.argmax(self.validation_data[3], axis=1).numpy()[0], tf.math.argmax(self.validation_data[4], axis=1).numpy()[0], tf.math.argmax(self.validation_data[5], axis=1).numpy()[0]],
[1, 2, 3, 4, 5], labels=clf.classes_)
disp = sklearn.metrics.ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=clf.classes_)
disp.plot()
plt.show()
fig.savefig(os.path.join(self.dat_dir, f'confusion_matrix_epoch_{epoch}'))
clf = sklearn.svm.SVC(random_state=0)
clf.fit(tf.constant( self.validation_data).numpy(), tf.linspace( 0, 19, 20, name='linspace', axis=0 ).numpy())
fpr, tpr, thresholds = sklearn.metrics.roc_curve([0, 0, 1, 1], [0, list_auc_roc[0], list_auc_roc[1], list_auc_roc[2]])
auc_roc = sklearn.metrics.auc(fpr, tpr)
display = sklearn.metrics.RocCurveDisplay(fpr=fpr, tpr=tpr, roc_auc=auc_roc, estimator_name='example estimator')
display.plot()
plt.show()
fig.savefig(os.path.join(self.dat_dir, f'roc_curve_epoch_{epoch}'))
fig, ax = plt.subplots(figsize=(8,4))
plt.scatter(y_val, y_pred_val, alpha=0.6, color='#FF0000', lw=1, ec='black')
plt.show()
fig.savefig(os.path.join(self.dat_dir, f'dl_scatterplot'))
Build the model:
def model_builder(hp):
model = Sequential()
for i in range(hp.Int("num_layers", 1, 50)):
model.add(
layers.Dense(
# Tune number of units separately.
units=hp.Int(f"units_{i}", min_value=1, max_value=200, step=5),
#activation=hp.Choice("activation", ["relu", "tanh"])
activation=hp.Choice("activation", ["relu", "tanh", "sigmoid", "softmax", "softplus", "softsign", "selu", "elu", "exponential"])
))
model.add(Dense(4, kernel_initializer='normal', activation='linear')) # output layer
if hp.Boolean("dropout"):
model.add(layers.Dropout(rate=0.1))
model.add(layers.Dense(10, activation="softmax"))
# Tune the learning rate for the optimizer
# Choose an optimal value from 0.01, 0.001, or 0.0001
hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
# Performance visualization callback
performance_viz_cbk = PerformanceVisualizationCallback(
model=model,
validation_data=X_val,
dat_dir='c:\performance_charts')
model.compile(optimizer=Adam(learning_rate=hp_learning_rate),
loss=SparseCategoricalCrossentropy(from_logits=True),
metrics=['auc_roc'])
return model
Search for optimal hyperparameters:
tuner = kt.Hyperband(model_builder,
objective=kt.Objective('val_auc', direction='max'),
max_epochs=200,
factor=3,
directory='my_dir',
overwrite=True,
project_name='intro_to_kt')
stop_early = EarlyStopping(monitor='auc_roc', patience=5)
tuner.search(X_train, y_train, epochs=10, validation_split=0.2, callbacks=[stop_early])
Traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_11015/3831715835.py in <module>
----> 1 tuner.search(X_train, y_train, epochs=10, validation_split=0.2, callbacks=[stop_early])
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/base_tuner.py in search(self, *fit_args, **fit_kwargs)
181
182 self.on_trial_begin(trial)
--> 183 results = self.run_trial(trial, *fit_args, **fit_kwargs)
184 # `results` is None indicates user updated oracle in `run_trial()`.
185 if results is None:
/opt/conda/lib/python3.7/site-packages/keras_tuner/tuners/hyperband.py in run_trial(self, trial, *fit_args, **fit_kwargs)
382 fit_kwargs["epochs"] = hp.values["tuner/epochs"]
383 fit_kwargs["initial_epoch"] = hp.values["tuner/initial_epoch"]
--> 384 return super(Hyperband, self).run_trial(trial, *fit_args, **fit_kwargs)
385
386 def _build_model(self, hp):
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in run_trial(self, trial, *args, **kwargs)
293 callbacks.append(model_checkpoint)
294 copied_kwargs["callbacks"] = callbacks
--> 295 obj_value = self._build_and_fit_model(trial, *args, **copied_kwargs)
296
297 histories.append(obj_value)
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in _build_and_fit_model(self, trial, *args, **kwargs)
220 hp = trial.hyperparameters
221 model = self._try_build(hp)
--> 222 results = self.hypermodel.fit(hp, model, *args, **kwargs)
223 tuner_utils.validate_trial_results(
224 results, self.oracle.objective, "HyperModel.fit()"
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/hypermodel.py in fit(self, hp, model, *args, **kwargs)
138 If return a float, it should be the `objective` value.
139 """
--> 140 return model.fit(*args, **kwargs)
141
142
/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py in error_handler(*args, **kwargs)
68 # To get the full stack trace, call:
69 # `tf.debugging.disable_traceback_filtering()`
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in tf__train_function(iterator)
13 try:
14 do_return = True
---> 15 retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
16 except:
17 do_return = False
ValueError: in user code:
File "/opt/conda/lib/python3.7/site-packages/keras/engine/training.py", line 1160, in train_function *
return step_function(self, iterator)
File "/opt/conda/lib/python3.7/site-packages/keras/engine/training.py", line 1146, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/opt/conda/lib/python3.7/site-packages/keras/engine/training.py", line 1135, in run_step **
outputs = model.train_step(data)
File "/opt/conda/lib/python3.7/site-packages/keras/engine/training.py", line 998, in train_step
return self.compute_metrics(x, y, y_pred, sample_weight)
File "/opt/conda/lib/python3.7/site-packages/keras/engine/training.py", line 1092, in compute_metrics
self.compiled_metrics.update_state(y, y_pred, sample_weight)
File "/opt/conda/lib/python3.7/site-packages/keras/engine/compile_utils.py", line 577, in update_state
self.build(y_pred, y_true)
File "/opt/conda/lib/python3.7/site-packages/keras/engine/compile_utils.py", line 484, in build
y_pred, self._get_metric_objects, self._metrics, y_true, y_pred
File "/opt/conda/lib/python3.7/site-packages/keras/engine/compile_utils.py", line 631, in _get_metric_objects
return [self._get_metric_object(m, y_t, y_p) for m in metrics]
File "/opt/conda/lib/python3.7/site-packages/keras/engine/compile_utils.py", line 631, in <listcomp>
return [self._get_metric_object(m, y_t, y_p) for m in metrics]
File "/opt/conda/lib/python3.7/site-packages/keras/engine/compile_utils.py", line 650, in _get_metric_object
metric_obj = metrics_mod.get(metric)
File "/opt/conda/lib/python3.7/site-packages/keras/metrics/__init__.py", line 181, in get
return deserialize(str(identifier))
File "/opt/conda/lib/python3.7/site-packages/keras/metrics/__init__.py", line 140, in deserialize
printable_module_name="metric function",
File "/opt/conda/lib/python3.7/site-packages/keras/utils/generic_utils.py", line 770, in deserialize_keras_object
f"Unknown {printable_module_name}: {object_name}. Please "
ValueError: Unknown metric function: auc_roc. Please ensure this object is passed to the `custom_objects` argument. See https://www.tensorflow.org/guide/keras/save_and_serialize#registering_the_custom_object for details.

InvalidArgumentError: Graph execution error: Detected at node

My idea is to train a collaborative filter model for arts. I'm trying to train my model like this:
def utils_plot_keras_training(training):
metrics = [k for k in training.history.keys() if ("loss" not in k) and ("val" not in k)]
fig, ax = plt.subplots(nrows=1, ncols=2, sharey=True, figsize=(15,3))
ax[0].set(title="Training")
ax11 = ax[0].twinx()
ax[0].plot(training.history['loss'], color='black')
ax[0].set_xlabel('Epochs')
ax[0].set_ylabel('Loss', color='black')
for metric in metrics:
ax11.plot(training.history[metric], label=metric)
ax11.set_ylabel("Score", color='steelblue')
ax11.legend()
ax[1].set(title="Validation")
ax22 = ax[1].twinx()
ax[1].plot(training.history['val_loss'], color='black')
ax[1].set_xlabel('Epochs')
ax[1].set_ylabel('Loss', color='black')
for metric in metrics:
ax22.plot(training.history['val_'+metric], label=metric)
ax22.set_ylabel("Score", color="steelblue")
plt.show()
training = model.fit(x=[train["user_id"], train["art_id"]], y=train["y"],
epochs=100, batch_size=128, shuffle=True, verbose=0, validation_split=0.3)
model = training.model
utils_plot_keras_training(training)
And getting next error:
--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call
last) Input In [30], in <cell line: 2>()
1 # train
----> 2 training = model.fit(x=[train["user_id"], train["art_id"]], y=train["y"],
3 epochs=100, shuffle=True, verbose=0, validation_split=0.3)
4 model = training.model
5 utils_plot_keras_training(training)
File
~\DataspellProjects\Arts\venv\lib\site-packages\keras\utils\traceback_utils.py:67,
in filter_traceback..error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.traceback)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
File
~\DataspellProjects\Arts\venv\lib\site-packages\tensorflow\python\eager\execute.py:54,
in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
52 try:
53 ctx.ensure_initialized()
---> 54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
57 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node 'CollaborativeFiltering/xusers_emb/embedding_lookup'
defined at (most recent call last):
File "C:\Python310\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Python310\lib\runpy.py", line 86, in _run_code
exec(code, run_globals) ........
Node: 'CollaborativeFiltering/xusers_emb/embedding_lookup'
indices[28,0] = 1000 is not in [0, 1000) [[{{node
CollaborativeFiltering/xusers_emb/embedding_lookup}}]]
[Op:__inference_test_function_2209]
Any thoughts on how to resolve it? Full code and datasets are here: Github.

how to pass train_generator and test_generator to autoencoder.fit

I have a data generator as follows:
def datagenerator(x1,x2,batchsize):
n1 = x1.shape[0]
n2 = x2.shape[0]
while True:
num1 = np.random.randint(0, n1, batchsize)
num2 = np.random.randint(0, n2, batchsize)
x_data = (x1[num1] + x2[num2]) / 2.0
y_data = np.concatenate((x1[num1], x2[num2]), axis=2)
yield x_data, y_data
which gets two images and returns their average. Then I pass two datasets to this datagenerator ('mnist' and 'fashion_mnist') as follows:
train_generator = datagenerator(mnist_x_train,fashion_mnist_x_train,1)
test_generator = datagenerator(mnist_x_test,fashion_mnist_x_test,1)
but when I want to fit them using autoencoder:
autoencoder.fit(
train_generator,
epochs=100,
batch_size=128,
shuffle=True,
validation_data=test_generator,
)
it throws the error below:
quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
53 ctx.ensure_initialized()
54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
57 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node 'binary_crossentropy/mul' defined at (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py", line 16, in
<module>
app.launch_new_instance()
what is wrong with my code?

InvalidArgumentError: Can not squeeze dim[2], expected a dimension of 1, got 10

I am doing Covid19 facemask detection project and when I train my image dataset I find a error which I can't understand. So, please help me to solve this problem. the error is given below.
Epoch 1/20
Traceback (most recent call last):
File "Mask_detection.py", line 108, in <module>
epochs=Epoch)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 108, in _method_wrapper
return method(self, *args, **kwargs)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1098, in fit
tmp_logs = train_function(iterator)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\def_function.py", line 780, in __call__
result = self._call(*args, **kwds)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\def_function.py", line 840, in _call
return self._stateless_fn(*args, **kwds)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 2829, in __call__
return graph_function._filtered_call(args, kwargs) # pylint: disable=protected-access
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 1848, in _filtered_call
cancellation_manager=cancellation_manager)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 1924, in _call_flat
ctx, args, cancellation_manager=cancellation_manager))
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\function.py", line 550, in call
ctx=ctx)
File "C:\Users\ABDEALIVORA\AppData\Local\Programs\Python\Python37\lib\site-packages\tensorflow\python\eager\execute.py", line 60, in quick_execute
inputs, attrs, num_outputs)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Can not squeeze dim[2], expected a dimension of 1, got 10
[[node categorical_crossentropy/remove_squeezable_dimensions/Squeeze (defined at Mask_detection.py:108) ]] [Op:__inference_train_function_889]
Function call stack:
train_function
2020-09-28 12:37:31.761507: W tensorflow/core/kernels/data/generator_dataset_op.cc:103] Error occurred when finalizing GeneratorDataset iterator: Failed precondition: Python interpreter s
tate is not initialized. The process may be terminated.
[[{{node PyFunc}}]]
I provide my python code that helps you to understand the problem. My system is without GPU so this error is related with GPU.
DIRECTORY = 'images'
Categories = ["With_mask","Without_mask"]
batch_size= 10
num_class = 10
Epoch= 20
data = []
label =[]
for category in Categories:
path = os.path.join(DIRECTORY,category)
for img in os.listdir(path):
img_path = os.path.join(path,img)
image = load_img(img_path,target_size =(64,64))
image = img_to_array(image)
image = preprocess_input(image)
data.append(image)
label.append(category)
lb = LabelBinarizer()
label = lb.fit_transform(label)
label = to_categorical(label)
data = numpy.asarray(data,dtype = 'float32')
label = numpy.array(label)
print("////")
x_train,x_test,y_train,y_test = train_test_split(data,label,stratify=label,test_size=0.2,random_state=3)
y_train = utils.to_categorical(y_train, num_class)
y_test = utils.to_categorical(y_test, num_class)
mask_model = Sequential()
mask_model.add(Conv2D(32,kernel_size=(3,3),activation= 'linear',padding ="same",input_shape=(64,64,3)))
mask_model.add(LeakyReLU(alpha = 0.3))
mask_model.add(Conv2D(32,kernel_size=(3,3),activation= 'linear',padding ="same",input_shape=(64,64,3)))
mask_model.add(LeakyReLU(alpha=0.3))
mask_model.add(MaxPooling2D(pool_size =(2,2)))
mask_model.add(Conv2D(32,kernel_size=(3,3),activation= 'linear',padding ="same",input_shape=(64,64,3)))
mask_model.add(LeakyReLU(alpha=0.3))
mask_model.add(MaxPooling2D(pool_size =(2,2)))
mask_model.add(Flatten())
mask_model.add(Dense(128,activation = "linear"))
mask_model.add(LeakyReLU(alpha=0.3))
mask_model.add(Dense(10,activation= "softmax"))
mask_model.compile(optimizer ='adam',loss = 'categorical_crossentropy',metrics =['accuracy'] )
mask_model.summary()
datagen = ImageDataGenerator(
featurewise_center=True,
featurewise_std_normalization=True,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True)
datagen.fit(x_train)
mask_model.fit(datagen.flow(x_train,y_train,batch_size=10),
steps_per_epoch=len(x_train),
validation_data=(x_test, y_test),
validation_steps=len(x_test) // batch_size,
workers=0,
epochs=Epoch)
print("//")
for e in range(Epoch):
print('Epoch', e)
batches = 0
for x_batch, y_batch in datagen.flow(x_train, y_train, batch_size=32):
mask_model.fit(x_batch, y_batch)
batches += 1
if batches >= len(x_train) / 32:
break
mask_model.save("Mask_model/mask_model.h5")

Tensorflow Keras - UnknownError: UnidentifiedImageError

I've downloaded dataset for Malaria detection from this website. Afterwards, I've updated images to my google drive and tried to train neural net with in-built fit() function as follows:
train_gen = train_aug.flow_from_directory(
training_data_dir,
class_mode="categorical",
target_size=(64, 64),
color_mode="rgb",
shuffle=True,
batch_size=BATCH_SIZE)
val_gen = val_aug.flow_from_directory(
validation_data_dir,
class_mode="categorical",
target_size=(64, 64),
color_mode="rgb",
shuffle=False,
batch_size=BATCH_SIZE)
history = model.fit(x=train_gen, steps_per_epoch=steps_per_epoch, epochs=EPOCH_NUM,
validation_data=val_gen, validation_steps=val_steps, callbacks=CALLBACKS)
In the middle of training I am getting following error message:
Epoch 1/100
302/603 [==============>...............] - ETA: 44:54 - loss: 8.3442 - binary_accuracy: 0.4935
---------------------------------------------------------------------------
UnknownError Traceback (most recent call last)
<ipython-input-45-2fe1e94cba86> in <module>()
1 history = model.fit(x=train_gen, steps_per_epoch=steps_per_epoch, epochs=EPOCH_NUM,
----> 2 validation_data=val_gen, validation_steps=val_steps, callbacks=CALLBACKS)
8 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
UnknownError: UnidentifiedImageError: cannot identify image file <_io.BytesIO object at 0x7f42ff5c2518>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/script_ops.py", line 243, in __call__
ret = func(*args)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py", line 309, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/dataset_ops.py", line 785, in generator_py_func
values = next(generator_state.get_iterator(iterator_id))
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/data_adapter.py", line 801, in wrapped_generator
for data in generator_fn():
File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/data_adapter.py", line 932, in generator_fn
yield x[i]
File "/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/iterator.py", line 65, in __getitem__
return self._get_batches_of_transformed_samples(index_array)
File "/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/iterator.py", line 230, in _get_batches_of_transformed_samples
interpolation=self.interpolation)
File "/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/utils.py", line 114, in load_img
img = pil_image.open(io.BytesIO(f.read()))
File "/usr/local/lib/python3.6/dist-packages/PIL/Image.py", line 2862, in open
"cannot identify image file %r" % (filename if filename else fp)
PIL.UnidentifiedImageError: cannot identify image file <_io.BytesIO object at 0x7f42ff5c2518>
[[{{node PyFunc}}]]
[[IteratorGetNext]] [Op:__inference_train_function_35711]
Function call stack:
train_function
What is this error exactly, and how can I properly handle it? Do I need to use custom training loops with GradientTape object and then use try/catch blocks or is there another way?
Thing that confuses me is that it seems like some image cannot be decoded or something like that. But, how come ImageDataGenerator did not report any error before training?

Deleting all images and re-uploading them did the trick for me. Closing this question.

Reducing the number of workers in model.fit helped me solve this issue.