My trainX is (3350, 1, 8) and my TrainY is (3350, 2). I am getting some errors, but I don't understand what's the problem. I am getting this error when tuning the hyperparameters using the LSTM layer.
Output exceeds the size limit. Open the full output data in a text editor
--------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last) /tmp/ipykernel_4525/3330873115.py in <module>
----> 1 bayesian_opt_tuner.search(trainX, trainY,epochs=100,
2 #validation_data=(X_test, y_test)
3 validation_split=0.2,verbose=1)
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/base_tuner.py in search(self, *fit_args, **fit_kwargs)
181
182 self.on_trial_begin(trial)
--> 183 results = self.run_trial(trial, *fit_args, **fit_kwargs)
184 # `results` is None indicates user updated oracle in `run_trial()`.
185 if result is None:
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/tuner.py in run_trial(self, trial, *args, **kwargs)
293 callbacks.append(model_checkpoint)
294 copied_kwargs["callbacks"] = callbacks
--> 295 obj_value = self._build_and_fit_model(trial, *args, **copied_kwargs)
296
297 histories.append(obj_value)
~/anaconda3/lib/python3.9/site-packages/keras_tuner/engine/tuner.py in
_build_and_fit_model(self, trial, *args, **kwargs)
220 hp = trial.hyperparameters
221 model = self._try_build(hp)
--> 222 results = self.hypermodel.fit(hp, model, *args, **kwargs) ...
File "/home/vareeshadki/anaconda3/lib/python3.9/site-packages/keras/backend.py", line 5238, in sparse_categorical_crossentropy
res = cf.nn.sparse_softmax_cross_entropy_with_logits( Node: 'sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits' logits and labels must have the same first dimension, got logits shape [32,2] and labels shape [64] [[{{node sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_train_function_1763748]
My code:
import os
from kerastuner.tuners import BayesianOptimization
def build_model(hp):
model = tf.keras.Sequential()
hp_units = hp.Int('units', min_value=32, max_value=512, step=10)
model.add(LSTM(hp_units,activation='relu'))
model.add(Dense(2))
hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=hp_learning_rate),
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=['accuracy'])
return model
bayesian_opt_tuner = BayesianOptimization(
build_model,
objective='mse',
max_trials=3,
executions_per_trial=2,
directory=os.path.normpath('C:/keras_tuning'),
project_name='kerastuner_bayesian_poc',
overwrite=True)
bayesian_opt_tuner.search(trainX, trainY,epochs=100,
#validation_data=(X_test, y_test)
validation_split=0.2,verbose=1)
Related
I have searched through stackoverflow and read through documentation but somehow still my trained model is not being loaded!
I have checked following links and many more so please do not mark as duplicate straight away. thank you
How to save final model using keras?
https://www.tensorflow.org/guide/keras/save_and_serialize
https://www.tensorflow.org/tutorials/distribute/save_and_load
https://www.tensorflow.org/guide/keras/functional
How to load a model from an HDF5 file in Keras?
here is my model:
def get_model(train=True):
set_seed(33)
pre_process = Lambda(preprocess_input)
vgg = VGG16(weights = 'imagenet', include_top = True, input_shape = SHAPE)
vgg = Model(vgg.input, vgg.layers[-3].output)
vgg.trainable = False
inp = Input(SHAPE)
vgg_16_process = pre_process(GaussianNoise(0.1)(inp))
vgg_out = vgg(vgg_16_process)
noise = Lambda(tf.zeros_like)(vgg_out)
noise = GaussianNoise(0.1)(noise)
if train:
x = Lambda(lambda z: tf.concat(z, axis=0))([vgg_out,noise])
x = Activation('relu')(x)
else:
x = vgg_out
x = Dense(512, activation='relu')(x)
x = Dense(128, activation='relu')(x)
out = Dense(2, activation='softmax')(x)
model = Model(inp, out)
model.compile(Adam(learning_rate=1e-4), loss='binary_crossentropy')
return model
And After that I have some test and train generator as follows,
The dataset is classical dog-vs-cat dataset and I'm trying to achieve one-class classification task.
train_datagen = ImageDataGenerator()
test_datagen = ImageDataGenerator()
train_generator = train_datagen.flow_from_directory(
base_path + 'training_set/training_set/',
target_size = (SHAPE[0], SHAPE[1]),
batch_size = batch_size,
class_mode = 'categorical',
shuffle = True,
seed = 33,
classes = ['cats']
)
test_generator = test_datagen.flow_from_directory(
base_path + 'test_set/test_set/',
target_size = (SHAPE[0], SHAPE[1]),
batch_size = batch_size,
class_mode = 'categorical',
shuffle = True,
seed = 33,
classes = ['dogs','cats']
)
Then finally, I do compile the model
model = get_model()
model.fit(wrap_generator(train_generator), steps_per_epoch=train_generator.samples/train_generator.batch_size, epochs=1)
Then I do save model as follow
model.save('my_custom_model')
Either I do save in that format or 'my_custom_model.h5' it saves perfectly without any error.
If I use just save, then a folder is created with 'assets, variables, .pb files' that is expected as mentioned in posts of stackoverflow.
** Then here comes the problem **
from keras.models import load_model
loaded_model = load_model('my_custom_model.h5')
#or
loaded_model = tf.keras.models.load_model('my_custom_model.h5')
#or
loaded_model = load_model('my_custom_model') # as from folder
All throw same error
TypeError: 'module' object is not callable
I know I'm doing some mistake or model is bit different, please guide me to the right direction where I can look for solutions.
Python: 3.7.2
Tensorflow: 2.6.0
Keras: 2.6.0
Full Stack Trace
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
~\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\ops\array_ops.py in wrapper(*args, **kwargs)
205 try:
--> 206 return target(*args, **kwargs)
207 except (TypeError, ValueError):
TypeError: 'str' object is not callable
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-10-cd6655ece7c5> in <module>
7 # loaded_model = tf.keras.models.load_model("my_custom_model2")
8
----> 9 loaded_model = tf.keras.models.load_model('model_notebook.h5')
10 # print(loaded_model)
11 # print("------------")
~\AppData\Roaming\Python\Python37\site-packages\keras\saving\save.py in load_model(filepath, custom_objects, compile, options)
199 (isinstance(filepath, h5py.File) or h5py.is_hdf5(filepath))):
200 return hdf5_format.load_model_from_hdf5(filepath, custom_objects,
--> 201 compile)
202
203 filepath = path_to_string(filepath)
~\AppData\Roaming\Python\Python37\site-packages\keras\saving\hdf5_format.py in load_model_from_hdf5(filepath, custom_objects, compile)
179 model_config = json_utils.decode(model_config)
180 model = model_config_lib.model_from_config(model_config,
--> 181 custom_objects=custom_objects)
182
183 # set weights
~\AppData\Roaming\Python\Python37\site-packages\keras\saving\model_config.py in model_from_config(config, custom_objects)
50 '`Sequential.from_config(config)`?')
51 from keras.layers import deserialize # pylint: disable=g-import-not-at-top
---> 52 return deserialize(config, custom_objects=custom_objects)
53
54
~\AppData\Roaming\Python\Python37\site-packages\keras\layers\serialization.py in deserialize(config, custom_objects)
210 module_objects=LOCAL.ALL_OBJECTS,
211 custom_objects=custom_objects,
--> 212 printable_module_name='layer')
~\AppData\Roaming\Python\Python37\site-packages\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
676 custom_objects=dict(
677 list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 678 list(custom_objects.items())))
679 else:
680 with CustomObjectScope(custom_objects):
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\functional.py in from_config(cls, config, custom_objects)
661 with generic_utils.SharedObjectLoadingScope():
662 input_tensors, output_tensors, created_layers = reconstruct_from_config(
--> 663 config, custom_objects)
664 model = cls(inputs=input_tensors, outputs=output_tensors,
665 name=config.get('name'))
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\functional.py in reconstruct_from_config(config, custom_objects, created_layers)
1281 if layer in unprocessed_nodes:
1282 for node_data in unprocessed_nodes.pop(layer):
-> 1283 process_node(layer, node_data)
1284
1285 input_tensors = []
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\functional.py in process_node(layer, node_data)
1229 input_tensors = (
1230 base_layer_utils.unnest_if_single_tensor(input_tensors))
-> 1231 output_tensors = layer(input_tensors, **kwargs)
1232
1233 # Update node index map.
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\base_layer.py in __call__(self, *args, **kwargs)
975 if _in_functional_construction_mode(self, inputs, args, kwargs, input_list):
976 return self._functional_construction_call(inputs, args, kwargs,
--> 977 input_list)
978
979 # Maintains info about the `Layer.call` stack.
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\base_layer.py in _functional_construction_call(self, inputs, args, kwargs, input_list)
1113 # Check input assumptions set after layer building, e.g. input shape.
1114 outputs = self._keras_tensor_symbolic_call(
-> 1115 inputs, input_masks, args, kwargs)
1116
1117 if outputs is None:
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\base_layer.py in _keras_tensor_symbolic_call(self, inputs, input_masks, args, kwargs)
846 return tf.nest.map_structure(keras_tensor.KerasTensor, output_signature)
847 else:
--> 848 return self._infer_output_signature(inputs, args, kwargs, input_masks)
849
850 def _infer_output_signature(self, inputs, args, kwargs, input_masks):
~\AppData\Roaming\Python\Python37\site-packages\keras\engine\base_layer.py in _infer_output_signature(self, inputs, args, kwargs, input_masks)
886 self._maybe_build(inputs)
887 inputs = self._maybe_cast_inputs(inputs)
--> 888 outputs = call_fn(inputs, *args, **kwargs)
889
890 self._handle_activity_regularization(inputs, outputs)
~\AppData\Roaming\Python\Python37\site-packages\keras\layers\core.py in call(self, inputs, mask, training)
901 with tf.GradientTape(watch_accessed_variables=True) as tape,\
902 tf.variable_creator_scope(_variable_creator):
--> 903 result = self.function(inputs, **kwargs)
904 self._check_variables(created_variables, tape.watched_variables())
905 return result
~\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\ops\array_ops.py in wrapper(*args, **kwargs)
208 # Note: convert_to_eager_tensor currently raises a ValueError, not a
209 # TypeError, when given unexpected types. So we need to catch both.
--> 210 result = dispatch(wrapper, args, kwargs)
211 if result is not OpDispatcher.NOT_SUPPORTED:
212 return result
TypeError: 'module' object is not callable
The dataset is classical dog-vs-cat dataset and I'm trying to achieve
one-class classification task.
Then the model should have only one output node because this is an example of Binary Classification. Your model has the last Dense layer with two output units and the softmax as an activation function, which is a possible solution, but for this kind of problem my suggestion is to specify on the last Dense layer only one output unit and the sigmoid as the activation function.
It seems you forgot to add the string dog in the classes list on the train_generator.
train_generator = train_datagen.flow_from_directory(
base_path + 'training_set/training_set/',
target_size = (SHAPE[0], SHAPE[1]),
batch_size = batch_size,
class_mode = 'categorical',
shuffle = True,
seed = 33,
classes = ['cats']
)
Now, regards on the main topic of your question.
from keras.models import load_model
loaded_model = load_model('my_custom_model.h5')
#or
loaded_model = tf.keras.models.load_model('my_custom_model.h5')
#or
loaded_model = load_model('my_custom_model') # as from folder
I think, but I'm not 100% sure about it, the problem is from this line of code from keras.models import load_model. Starting from Tensorflow 2.x, which includes also the Keras module, every single function, class etc. must be imported by specifying always tensorflow as the first name for import each Keras module. In short, you should write this line of code: from tensorflow.keras.models import load_model and it should work because it calls the function from the Keras module with the Tensorflow as the back engine.
I want to include a pre-trained XLNet (or possibly another state of the art transformer) in a model to fine-tune it.
However, it doesn't work when I include it with keras layers.
import tensorflow as tf
from transformers import AutoTokenizer, TFAutoModel
inputs = tf.keras.Input(shape=2000, dtype='int32')
x = inputs
xlnetPretrainedModel = TFAutoModel.from_pretrained("xlnet-base-cased")
x = xlnetPretrainedModel(x)
x = tf.keras.layers.GlobalAveragePooling1D()(x)
x = tf.keras.layers.Dense(32, activation='relu')(x)
x = tf.keras.layers.Dense(32, activation=None)(x)
model = tf.keras.Model(inputs=inputs, outputs=x)
model.compile(optimizer='adam',
loss='mean_squared_error')
model.summary()
The bug is
AttributeError: 'NoneType' object has no attribute 'shape'
at the line
x = xlnetPretrainedModel(x)
So when the model is used on the input layer.
The XLNet model works if used on a numpy array, but then I wouldn't be able to train it.
The full error message is:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-23-d543506f9697> in <module>
5 x = inputs
6 xlnetPretrainedModel = TFAutoModel.from_pretrained("xlnet-base-cased")
----> 7 x = xlnetPretrainedModel(x)
8 x = tf.keras.layers.GlobalAveragePooling1D()(x)
9 x = tf.keras.layers.Dense(32, activation='relu')(x)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
771 not base_layer_utils.is_in_eager_or_tf_function()):
772 with auto_control_deps.AutomaticControlDependencies() as acd:
--> 773 outputs = call_fn(cast_inputs, *args, **kwargs)
774 # Wrap Tensors in `outputs` in `tf.identity` to avoid
775 # circular dependencies.
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/autograph/impl/api.py in wrapper(*args, **kwargs)
235 except Exception as e: # pylint:disable=broad-except
236 if hasattr(e, 'ag_error_metadata'):
--> 237 raise e.ag_error_metadata.to_exception(e)
238 else:
239 raise
AttributeError: in converted code:
/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_xlnet.py:810 call *
outputs = self.transformer(inputs, **kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py:805 __call__
inputs, outputs, args, kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py:2014 _set_connectivity_metadata_
input_tensors=inputs, output_tensors=outputs, arguments=arguments)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py:2044 _add_inbound_node
arguments=arguments)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/node.py:110 __init__
self.output_shapes = nest.map_structure(backend.int_shape, output_tensors)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/nest.py:568 map_structure
structure[0], [func(*x) for x in entries],
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/nest.py:568 <listcomp>
structure[0], [func(*x) for x in entries],
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py:1172 int_shape
shape = x.shape
AttributeError: 'NoneType' object has no attribute 'shape'
or after trying a solution presented here https://github.com/huggingface/transformers/issues/1350 by decoring the call by a tf.function
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-16-c852fba5aa15> in <module>
8 xlnetPretrainedModel = TFAutoModel.from_pretrained("xlnet-base-cased")
9 xlnetPretrainedModel.call = tf.function(xlnetPretrainedModel.transformer.call)
---> 10 x = xlnetPretrainedModel(x)
11
12 x = tf.keras.layers.GlobalAveragePooling1D()(x)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
803 kwargs.pop('mask')
804 inputs, outputs = self._set_connectivity_metadata_(
--> 805 inputs, outputs, args, kwargs)
806 self._handle_activity_regularization(inputs, outputs)
807 self._set_mask_metadata(inputs, outputs, input_masks)
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in _set_connectivity_metadata_(self, inputs, outputs, args, kwargs)
2012 # This updates the layer history of the output tensor(s).
2013 self._add_inbound_node(
-> 2014 input_tensors=inputs, output_tensors=outputs, arguments=arguments)
2015 return inputs, outputs
2016
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py in _add_inbound_node(self, input_tensors, output_tensors, arguments)
2042 input_tensors=input_tensors,
2043 output_tensors=output_tensors,
-> 2044 arguments=arguments)
2045
2046 # Update tensor history metadata.
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/node.py in __init__(self, outbound_layer, inbound_layers, node_indices, tensor_indices, input_tensors, output_tensors, arguments)
108 self.input_shapes = nest.map_structure(backend.int_shape, input_tensors)
109 # Nested structure of shape tuples, shapes of output_tensors.
--> 110 self.output_shapes = nest.map_structure(backend.int_shape, output_tensors)
111
112 # Optional keyword arguments to layer's `call`.
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/nest.py in map_structure(func, *structure, **kwargs)
566
567 return pack_sequence_as(
--> 568 structure[0], [func(*x) for x in entries],
569 expand_composites=expand_composites)
570
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/util/nest.py in <listcomp>(.0)
566
567 return pack_sequence_as(
--> 568 structure[0], [func(*x) for x in entries],
569 expand_composites=expand_composites)
570
/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/backend.py in int_shape(x)
1170 """
1171 try:
-> 1172 shape = x.shape
1173 if not isinstance(shape, tuple):
1174 shape = tuple(shape.as_list())
AttributeError: 'NoneType' object has no attribute 'shape'
Please, can anyone help me fix this error?
Generally the model output differs from model to model in the Huggingface transformers library. Check the documentation for what values are returned from the XLNet's "call()" function.
The last hidden state can generally be accessed in the following way:
model= TFAutoModel.from_pretrained("xlnet-base-cased")
# 'last_hidden_state' seems to be common to most of the transformer models
# refer: https://huggingface.co/transformers/model_doc/bert.html#tfbertmodel
output = model(tokenizer_ouput).last_hidden_state
x = tf.keras.layers.Dense(32, activation='relu')(x)
# The rest of you model
Making this change should solve your issue.
The XLNet Model takes in input which has been passed through the XLNet tokenizer.
Given:
from transformers import XLNetTokenizer
xlnet_tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased")
For a single forward pass, you can then generate your x as something like:
x = xlnet_tokenizer('Hello world', padding=True, return_tensors="tf")
After that, you should be able to proceed with:
xlnet_output = xlnetPretrainedModel(x)
To use x = xlnetPretrainedModel(x) in the functional API definition of your model, the input x will need to be in a format identical to output from xlnet_tokenizer.
Which is:
{'input_ids': <Tensor>, 'token_type_ids':<Tensor>, 'attention_mask':<Tensor>}.
You can do that with the functional API like:
input_ids = tf.keras.layers.Input(shape=(max_length,), dtype=tf.int32, name="input_ids")
token_type_ids = tf.keras.layers.Input(shape=(max_length,), dtype=tf.int32, name="token_type_ids")
attention_masks = tf.keras.layers.Input(shape=(max_length,), dtype=tf.int32, name="attention_mask")
x = xlnetPretrainedModel({
'input_ids': input_ids,
'token_type_ids':token_type_ids,
'attention_mask':attention_mask})
...
You should be able to proceed like that without any more of such problems.
I was experimenting with the Proximal AdaGrad for science fair and I was not able to use it because it counts it as it not existing.
My code:
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import numpy as np
import time
start_time = time.time()
data = tf.keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = data.load_data()
class_names = ['T-shirt', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle Boot']
train_images = train_images/255.0
test_images = test_images/255.0
model = keras.Sequential([
keras.layers.Flatten(input_shape=(28, 28)),
keras.layers.Dense(100, activation="relu"),
keras.layers.Dense(10, activation="softmax")
])
model.compile(optimizer="Proximal AdaGrad", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
model.fit(train_images, train_labels, epochs=200)
test_loss, test_acc = model.evaluate(test_images, test_labels)
print("Test acc is:", test_acc)
print("--- %s seconds ---" % (time.time() - start_time))
The Error:
ValueError Traceback (most recent call last)
<ipython-input-2-2d12844ae498> in <module>()
24 ])
25
---> 26 model.compile(optimizer="Proximal AdaGrad", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
27
28 model.fit(train_images, train_labels, epochs=200)
6 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, distribute, **kwargs)
250 'experimental_run_tf_function', True)
251
--> 252 self._set_optimizer(optimizer)
253 is_any_optimizer_v1 = any(isinstance(opt, optimizers.Optimizer)
254 for opt in nest.flatten(self.optimizer))
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py in _set_optimizer(self, optimizer)
1451 self.optimizer = [optimizers.get(opt) for opt in optimizer]
1452 else:
-> 1453 self.optimizer = optimizers.get(optimizer)
1454
1455 if (self._dtype_policy.loss_scale is not None and
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/optimizers.py in get(identifier)
844 elif isinstance(identifier, six.string_types):
845 config = {'class_name': str(identifier), 'config': {}}
--> 846 return deserialize(config)
847 else:
848 raise ValueError('Could not interpret optimizer identifier:', identifier)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/optimizers.py in deserialize(config, custom_objects)
813 module_objects=all_classes,
814 custom_objects=custom_objects,
--> 815 printable_module_name='optimizer')
816
817
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
178 config = identifier
179 (cls, cls_config) = class_and_config_for_serialized_keras_object(
--> 180 config, module_objects, custom_objects, printable_module_name)
181
182 if hasattr(cls, 'from_config'):
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/utils/generic_utils.py in class_and_config_for_serialized_keras_object(config, module_objects, custom_objects, printable_module_name)
163 cls = module_objects.get(class_name)
164 if cls is None:
--> 165 raise ValueError('Unknown ' + printable_module_name + ': ' + class_name)
166 return (cls, config['config'])
167
ValueError: Unknown optimizer: Proximal AdaGrad
I tried naming it other things such as "ProximalAdaGrad" and "ProximalGrad" but non of those worked. The Activation Function looks to have no issues but the Optimizer itself seems to have a bug. I searched for a post on GitHub but did not find anyone posting an issue about this.
There is an open issue about this. The TensorFlow implementation exists (even in TensorFlow 2.x, as tf.compat.v1.train.ProximalAdagradOptimizer), but there is no corresponding Keras implementation at the moment. However, the Keras API is able to wrap an existing TensorFlow optimizer, so you should be able to do the following:
# This works both in recent 1.x and 2.0
optimizer = tf.compat.v1.train.ProximalAdagradOptimizer(0.001)
model.compile(optimizer=optimizer,
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
I designed my own loss function. However when trying to revert to the best model encountered during training with
model = load_model("lc_model.h5")
I got the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-105-9d09ef163b0a> in <module>
23
24 # revert to the best model encountered during training
---> 25 model = load_model("lc_model.h5")
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in load_model(filepath, custom_objects, compile)
417 f = h5dict(filepath, 'r')
418 try:
--> 419 model = _deserialize_model(f, custom_objects, compile)
420 finally:
421 if opened_new_file:
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in _deserialize_model(f, custom_objects, compile)
310 metrics=metrics,
311 loss_weights=loss_weights,
--> 312 sample_weight_mode=sample_weight_mode)
313
314 # Set optimizer weights.
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
137 loss_functions = [losses.get(l) for l in loss]
138 else:
--> 139 loss_function = losses.get(loss)
140 loss_functions = [loss_function for _ in range(len(self.outputs))]
141 self.loss_functions = loss_functions
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in get(identifier)
131 if isinstance(identifier, six.string_types):
132 identifier = str(identifier)
--> 133 return deserialize(identifier)
134 if isinstance(identifier, dict):
135 return deserialize(identifier)
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in deserialize(name, custom_objects)
112 module_objects=globals(),
113 custom_objects=custom_objects,
--> 114 printable_module_name='loss function')
115
116
C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
163 if fn is None:
164 raise ValueError('Unknown ' + printable_module_name +
--> 165 ':' + function_name)
166 return fn
167 else:
ValueError: Unknown loss function:focal_loss_fixed
Here is the neural network :
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
model = create_model(x_train.shape[1], y_train.shape[1])
epochs = 35
batch_sz = 64
print("Beginning model training with batch size {} and {} epochs".format(batch_sz, epochs))
checkpoint = ModelCheckpoint("lc_model.h5", monitor='val_acc', verbose=0, save_best_only=True, mode='auto', period=1)
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.constraints import maxnorm
def create_model(input_dim, output_dim):
print(output_dim)
# create model
model = Sequential()
# input layer
model.add(Dense(100, input_dim=input_dim, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# hidden layer
model.add(Dense(60, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# output layer
model.add(Dense(output_dim, activation='softmax'))
# Compile model
# model.compile(loss='categorical_crossentropy', loss_weights=None, optimizer='adam', metrics=['accuracy'])
model.compile(loss=focal_loss(alpha=1), loss_weights=None, optimizer='adam', metrics=['accuracy'])
return model
# train the model
history = model.fit(x_train.as_matrix(),
y_train.as_matrix(),
validation_split=0.2,
epochs=epochs,
batch_size=batch_sz, # Can I tweak the batch here to get evenly distributed data ?
verbose=2,
class_weight = weights, # class_weight tells the model to "pay more attention" to samples from an under-represented fraud class.
callbacks=[checkpoint])
# revert to the best model encountered during training
model = load_model("lc_model.h5")
And here is my loss function:
import tensorflow as tf
def focal_loss(gamma=2., alpha=4.):
gamma = float(gamma)
alpha = float(alpha)
def focal_loss_fixed(y_true, y_pred):
"""Focal loss for multi-classification
FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
Notice: y_pred is probability after softmax
gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
Focal Loss for Dense Object Detection
https://arxiv.org/abs/1708.02002
Arguments:
y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
y_pred {tensor} -- model's output, shape of [batch_size, num_cls]
Keyword Arguments:
gamma {float} -- (default: {2.0})
alpha {float} -- (default: {4.0})
Returns:
[tensor] -- loss.
"""
epsilon = 1.e-9
y_true = tf.convert_to_tensor(y_true, tf.float32)
y_pred = tf.convert_to_tensor(y_pred, tf.float32)
model_out = tf.add(y_pred, epsilon)
ce = tf.multiply(y_true, -tf.log(model_out))
weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
fl = tf.multiply(alpha, tf.multiply(weight, ce))
reduced_fl = tf.reduce_max(fl, axis=1)
return tf.reduce_mean(reduced_fl)
return focal_loss_fixed
# model.compile(loss=focal_loss(alpha=1), optimizer='nadam', metrics=['accuracy'])
# model.fit(X_train, y_train, epochs=3, batch_size=1000)
You have to load the custom_objects of focal_loss_fixed as shown below:
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
However, if you wish to just perform inference with your model and not further optimization or training your model, you can simply wish to ignore the loss function like this:
model = load_model("lc_model.h5", compile=False)
The answer by #Prasad is great, but I would like to add a little explanation and a little correction:
while mentioning your custom loss function in the custom_objects dictionary you don't have to call your loss function, as it can give some parameter missing errors.
# Instead of this
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
# ty this without calling your loss function
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss})
Also, the thing I would like to add here is that you have to use the name of your custom loss function as a key and your function object as its value in custom_objects. I know this is very basic but the thing to mention but I hope this will be helpful to someone.
I am trying to run a stacked regression when the n_jobs is 1 it runs fine however, whenever I set the n_jobs to 2 it crashes with the error below. I looked into similar issues but none actually solved my error.
The code:
from civismlext.stacking import StackedRegressor
from civismlext.nonnegative import NonNegativeLinearRegression
def create_model():
model = Sequential()
model.add(Dense(150, activation='softmax', kernel_initializer='VarianceScaling', input_dim=456, name='HL1'))
model.add(Dropout(0.25, name="Dropout1"))
model.add(Dense(150, kernel_initializer='VarianceScaling', activation='softmax', name='HL2'))
model.add(Dropout(0.25, name="Dropout2"))
model.add(Dense(1, name='Output_Layer'))
model.compile(optimizer='adam', loss='mae', metrics=['mae', 'mean_squared_error'])
return model
mlp_model = KerasRegressor(build_fn=create_model, epochs=50, batch_size=75, validation_split=0.2, verbose=True)
super_learner = StackedRegressor([
('pipe_mlp', mlp_model),
('rf', rf),
('xgb', gb),
('meta', NonNegativeLinearRegression())
], cv=5, n_jobs=2, verbose=5)
the error:
MaybeEncodingError Traceback (most recent call last)
<ipython-input-7-1d4b04377633> in <module>()
1 # fitting the model
----> 2 super_learner.fit(X_train[:50], y_train[:50])
~/anaconda3/lib/python3.6/site-packages/civismlext/stacking.py in fit(self, X, y, **fit_params)
163 self.meta_estimator.fit(Xmeta, ymeta, **meta_params)
164 # Now fit base estimators again, this time on full training set
--> 165 self._base_est_fit(X, y, **fit_params)
166
167 return self
~/anaconda3/lib/python3.6/site-packages/civismlext/stacking.py in _base_est_fit(self, X, y, **fit_params)
220 n_jobs=self.n_jobs,
221 verbose=self.verbose,
--> 222 pre_dispatch=self.pre_dispatch)(_jobs)
223
224 for name, _ in self.estimator_list[:-1]:
~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self, iterable)
787 # consumption.
788 self._iterating = False
--> 789 self.retrieve()
790 # Make sure that we get a last message telling us we are done
791 elapsed_time = time.time() - self._start_time
~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in retrieve(self)
697 try:
698 if getattr(self._backend, 'supports_timeout', False):
--> 699 self._output.extend(job.get(timeout=self.timeout))
700 else:
701 self._output.extend(job.get())
~/anaconda3/lib/python3.6/multiprocessing/pool.py in get(self, timeout)
642 return self._value
643 else:
--> 644 raise self._value
645
646 def _set(self, i, obj):
MaybeEncodingError: Error sending result: '[<keras.callbacks.History object at 0x7f93fe43c7b8>]'. Reason: 'TypeError("can't pickle _thread.lock objects",)'
It's because the Keras scikit-learn wrappers don't exactly follow the scikit-learn API.
In scikit-learn, calling fit() on an estimator returns a fitted estimator. In the Keras wrapper, the fit() call returns a callbacks.History object.