Related
I am facing an unknown issue while training my BERT-CRF model for NER. I am using keras.contrib for the CRF model.
Here are the imported libraries.
!pip install transformers
!pip install git+https://www.github.com/keras-team/keras-contrib.git
import pandas as pd
import numpy as np
from transformers import TFBertModel, BertTokenizer, BertConfig
import tensorflow as tf
from tensorflow import keras
from keras_contrib.layers import CRF
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tqdm import tqdm
Code for the model creation.
input_ids = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
token_type_ids = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
attention_mask = keras.layers.Input(shape=(MAX_LEN,), dtype=tf.int32)
bert_output = bert(
[input_ids,
attention_mask,
token_type_ids]
)[0]
bert_output = keras.layers.Dropout(0.3)(bert_output)
dense_layer_output = keras.layers.Dense(num_classes+1, activation='softmax', name='output')(bert_output)
crf = CRF(num_classes)
outputs = crf(dense_layer_output)
model = keras.Model(
inputs=[input_ids, token_type_ids, attention_mask],
outputs=[outputs],
)
model.compile(
loss=crf.loss_function,
metrics=[crf.accuracy],
optimizer=keras.optimizers.Adam(5e-5)
)
model.fit(
x_train,
y_train,
epochs=1,
verbose=1,
batch_size=32,
validation_data=(x_test, y_test)
)
While trying to train the model I am getting this error. I cannot understand from where it is originating and why.
WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).
WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-18-f369b38eb91d> in <module>()
5 verbose=1,
6 batch_size=32,
----> 7 validation_data=(x_test, y_test)
8 )
9 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
AttributeError: in user code:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
/usr/local/lib/python3.7/dist-packages/keras_contrib/losses/crf_losses.py:54 crf_loss *
crf, idx = y_pred._keras_history[:2]
AttributeError: 'Tensor' object has no attribute '_keras_history'
I have read on the internet that keras.contrib is depricated but I don't know any other way how to use a CRF layer on top of BERT. If there is a better way of doing it in keras then please suggest me.
I don't know whether this question is making sense or not but any help would be appreciated.
Thanks in advance!
The easiest way is to use the CRF layer of the TensorFlow addons. Then utilize the output of that to calculate the loss.
import tensorflow_addons as tfa
crf = tfa.layers.CRF(len(num_labels)+1)
Further, you can utilize it by creating your own Model class too for model creation.
from tensorflow_addons.text.crf import crf_log_likelihood
def unpack_data(data):
if len(data) == 2:
return data[0], data[1], None
elif len(data) == 3:
return data
else:
raise TypeError("Expected data to be a tuple of size 2 or 3.")
class ModelWithCRFLoss(tf.keras.Model):
"""Wrapper around the base model for custom training logic."""
def __init__(self, base_model):
super().__init__()
self.base_model = base_model
def call(self, inputs):
return self.base_model(inputs)
def compute_loss(self, x, y, sample_weight, training=False):
y_pred = self(x, training=training)
_, potentials, sequence_length, chain_kernel = y_pred
# we now add the CRF loss:
crf_loss = -crf_log_likelihood(potentials, y, sequence_length, chain_kernel)[0]
if sample_weight is not None:
crf_loss = crf_loss * sample_weight
return tf.reduce_mean(crf_loss), sum(self.losses)
def train_step(self, data):
x, y, sample_weight = unpack_data(data)
with tf.GradientTape() as tape:
crf_loss, internal_losses = self.compute_loss(
x, y, sample_weight, training=True
)
total_loss = crf_loss + internal_losses
gradients = tape.gradient(total_loss, self.trainable_variables)
self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))
return {"crf_loss": crf_loss, "internal_losses": internal_losses}
def test_step(self, data):
x, y, sample_weight = unpack_data(data)
crf_loss, internal_losses = self.compute_loss(x, y, sample_weight)
return {"crf_loss_val": crf_loss, "internal_losses_val": internal_losses}
You can write along these lines of code
decoded_sequence, potentials, sequence_length, chain_kernel = crf(dense_layer_output, mask=attention_mask)
base_model = tf.keras.Model(
inputs=[input_ids, attention_mask],
outputs=crf_layer_outputs,
)
model = ModelWithCRFLoss(base_model)
model.compile(
optimizer=tf.keras.optimizers.Adam(learning_rate=5e-3, epsilon=1e-08),
metrics=tf.metrics.SparseCategoricalAccuracy(),
)
I am trying to train a model using Sequential model of Keras, and then calibrate it using scikit-learn's CalibratedClassifierCV. For this I use the KerasClassifier wrapper. Here is the code I use:
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,
patience=5, min_lr=0.000001)
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)
def create_model():
model_1 = Sequential()
n_cols = X_train_1.shape[1]
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal",input_shape=(n_cols,)))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(10, activation="selu", kernel_initializer="lecun_normal"))
model_1.add(Dense(1, activation='sigmoid'))
opt = keras.optimizers.Nadam(lr=0.0001)
loss = tf.keras.losses.BinaryCrossentropy(reduction='sum')
model_1.compile(optimizer=opt, loss=loss)
return model_1
X_train_1, X_test_1, y_train_1, y_test_1, w_train_1, w_test_1 = train_test_split(scaled_X_1, y_1, w_1, test_size=0.35, random_state=42)
model_1 = KerasClassifier(build_fn=create_model, epochs=5, batch_size=5000, verbose=1)
history_1 = model_1.fit(X_train_1, y_train_1, callbacks=[reduce_lr, es], epochs=5, validation_split=0.35, batch_size=5000, sample_weight=w_train_1, verbose=1)
plt.plot(history_1.history['loss'])
plt.plot(history_1.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
calibrator_1 = CalibratedClassifierCV(model_1, cv='prefit')
calibrator_1.fit(X_test_1, y_test_1, sample_weight = w_test_1)
As you see, I clearly call the instance of CalibratedClassifierCV to calibrator_1 and then use fit(). Despite this I get this error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-21-1cab7be4f6d6> in <module>
46 calibrator_1 = CalibratedClassifierCV(model_1, cv='prefit')
47
---> 48 calibrator_1.fit(X_test_1, y_test_1, sample_weight = w_test_1)
~/.local/lib/python3.6/site-packages/sklearn/calibration.py in fit(self, X, y, sample_weight)
263 pred_method = _get_prediction_method(base_estimator)
264 n_classes = len(self.classes_)
--> 265 predictions = _compute_predictions(pred_method, X, n_classes)
266
267 calibrated_classifier = _fit_calibrator(
~/.local/lib/python3.6/site-packages/sklearn/calibration.py in _compute_predictions(pred_method, X, n_classes)
499 (X.shape[0], 1).
500 """
--> 501 predictions = pred_method(X=X)
502 if hasattr(pred_method, '__name__'):
503 method_name = pred_method.__name__
TypeError: predict_proba() missing 1 required positional argument: 'x'
Does anyone spot any errors here?
This seems like an issue with the newer version of scikit-learn (0.24). I installed the older 0.23 version and it runs well with the same exact code.
i am trying my first time to get a keras-tuner tuned deep learning model. My tuning code goes like below:
def build_model_test(hp):
model = models.Sequential()
model.add(layers.InputLayer(input_shape=(100,28)))
model.add(layers.Dense(28,activation = 'relu'))
model.add(BatchNormalization(momentum = 0.99))
model.add(Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.5)))
model.add(layers.Conv1D(filters=hp.Int(
'num_filters',
16, 128,
step=16
),kernel_size=3,strides=1,padding='same',activation='relu'))
model.add(BatchNormalization(momentum = 0.99))
model.add(Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.5)))
model.add(layers.Conv1D(filters=hp.Int(
'num_filters',
16, 128,
step=16
),kernel_size=3,strides=1,padding='same',activation='relu'))
model.add(BatchNormalization(momentum = 0.99))
model.add(Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.5)))
model.add(layers.Conv1D(filters=hp.Int(
'num_filters',
16, 128,
step=16
),kernel_size=3,strides=1,padding='same',activation='relu'))
model.add(BatchNormalization(momentum = 0.99))
model.add(Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.5)))
model.add(layers.Dense(units=hp.Int('units',min_value=16,max_value=512,step=32,default=128),activation = 'relu'))
model.add(Dropout(hp.Float('dropout', 0, 0.5, step=0.1, default=0.5)))
model.add(layers.Dense(1, activation = 'linear'))
model.compile(
optimizer='adam',
loss=['mean_squared_error'],
metrics=[tf.keras.metrics.RootMeanSquaredError()]
)
return model
tuner = RandomSearch(
build_model_test,
objective='mean_squared_error',
max_trials=20,
executions_per_trial=3,
directory='my_dir',
project_name='helloworld')
x_train,x_test=dataframes[0:734,:,:],dataframes[734:1100,:,:]
y_train,y_test=target_fx[0:734,:,:],target_fx[734:1100,:,:]
tuner.search(x_train, y_train,
epochs=20,
validation_data=(x_test, y_test))
models = tuner.get_best_models(num_models=1)
but as soon as the 20th epoch arrives it prints this error
ValueError Traceback (most recent call last)
<ipython-input-59-997de3dfa9e5> in <module>
52 tuner.search(x_train, y_train,
53 epochs=20,
---> 54 validation_data=(x_test, y_test))
55
56 models = tuner.get_best_models(num_models=1)
~\Anaconda3\envs\deeplearning\lib\site-packages\kerastuner\engine\base_tuner.py in search(self, *fit_args, **fit_kwargs)
128
129 self.on_trial_begin(trial)
--> 130 self.run_trial(trial, *fit_args, **fit_kwargs)
131 self.on_trial_end(trial)
132 self.on_search_end()
~\Anaconda3\envs\deeplearning\lib\site-packages\kerastuner\engine\multi_execution_tuner.py in run_trial(self, trial, *fit_args, **fit_kwargs)
107 averaged_metrics[metric] = np.mean(execution_values)
108 self.oracle.update_trial(
--> 109 trial.trial_id, metrics=averaged_metrics, step=self._reported_step)
110
111 def _configure_tensorboard_dir(self, callbacks, trial_id, execution=0):
~\Anaconda3\envs\deeplearning\lib\site-packages\kerastuner\engine\oracle.py in update_trial(self, trial_id, metrics, step)
182
183 trial = self.trials[trial_id]
--> 184 self._check_objective_found(metrics)
185 for metric_name, metric_value in metrics.items():
186 if not trial.metrics.exists(metric_name):
~\Anaconda3\envs\deeplearning\lib\site-packages\kerastuner\engine\oracle.py in _check_objective_found(self, metrics)
351 'Objective value missing in metrics reported to the '
352 'Oracle, expected: {}, found: {}'.format(
--> 353 objective_names, metrics.keys()))
354
355 def _get_trial_dir(self, trial_id):
ValueError: Objective value missing in metrics reported to the Oracle, expected: ['mean_squared_error'], found: dict_keys(['loss', 'root_mean_squared_error', 'val_loss', 'val_root_mean_squared_error'])
which i do not get since i have specified to the model as mean squared error to follow, do you know what commands should i change to get the result i want?
Also can i call early stopping in the keras-tuner?
In addition to what was given in the previous response, you have also inquired about the possibility of calling early stopping in the keras-tuner. This is indeed possible with an early stopping callback.
First assign the EarlyStopping callback to a variable with the correct value to monitor. In this case I use 'val_loss'.
This would look like:
stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)
Then change the line where you start the hyperparameter search like so:
tuner.search(x_train, y_train,
epochs=20,
validation_data=(x_test, y_test), callbacks=[stop_early])
Note the callbacks argument. Feel free to change any of the arguments you define the callback with to your liking/ application
you should use objective='root_mean_squared_error'
tuner = RandomSearch(
build_model_test,
objective='root_mean_squared_error',
max_trials=20,
executions_per_trial=3,
directory='my_dir',
project_name='helloworld')
I would rather use 'val_root_mean_squared_error' as most probably you are interested to decrease the error on the validation dataset.
I designed my own loss function. However when trying to revert to the best model encountered during training with
model = load_model("lc_model.h5")
I got the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-105-9d09ef163b0a> in <module>
23
24 # revert to the best model encountered during training
---> 25 model = load_model("lc_model.h5")
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in load_model(filepath, custom_objects, compile)
417 f = h5dict(filepath, 'r')
418 try:
--> 419 model = _deserialize_model(f, custom_objects, compile)
420 finally:
421 if opened_new_file:
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\saving.py in _deserialize_model(f, custom_objects, compile)
310 metrics=metrics,
311 loss_weights=loss_weights,
--> 312 sample_weight_mode=sample_weight_mode)
313
314 # Set optimizer weights.
C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
137 loss_functions = [losses.get(l) for l in loss]
138 else:
--> 139 loss_function = losses.get(loss)
140 loss_functions = [loss_function for _ in range(len(self.outputs))]
141 self.loss_functions = loss_functions
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in get(identifier)
131 if isinstance(identifier, six.string_types):
132 identifier = str(identifier)
--> 133 return deserialize(identifier)
134 if isinstance(identifier, dict):
135 return deserialize(identifier)
C:\ProgramData\Anaconda3\lib\site-packages\keras\losses.py in deserialize(name, custom_objects)
112 module_objects=globals(),
113 custom_objects=custom_objects,
--> 114 printable_module_name='loss function')
115
116
C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
163 if fn is None:
164 raise ValueError('Unknown ' + printable_module_name +
--> 165 ':' + function_name)
166 return fn
167 else:
ValueError: Unknown loss function:focal_loss_fixed
Here is the neural network :
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
model = create_model(x_train.shape[1], y_train.shape[1])
epochs = 35
batch_sz = 64
print("Beginning model training with batch size {} and {} epochs".format(batch_sz, epochs))
checkpoint = ModelCheckpoint("lc_model.h5", monitor='val_acc', verbose=0, save_best_only=True, mode='auto', period=1)
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.constraints import maxnorm
def create_model(input_dim, output_dim):
print(output_dim)
# create model
model = Sequential()
# input layer
model.add(Dense(100, input_dim=input_dim, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# hidden layer
model.add(Dense(60, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
# output layer
model.add(Dense(output_dim, activation='softmax'))
# Compile model
# model.compile(loss='categorical_crossentropy', loss_weights=None, optimizer='adam', metrics=['accuracy'])
model.compile(loss=focal_loss(alpha=1), loss_weights=None, optimizer='adam', metrics=['accuracy'])
return model
# train the model
history = model.fit(x_train.as_matrix(),
y_train.as_matrix(),
validation_split=0.2,
epochs=epochs,
batch_size=batch_sz, # Can I tweak the batch here to get evenly distributed data ?
verbose=2,
class_weight = weights, # class_weight tells the model to "pay more attention" to samples from an under-represented fraud class.
callbacks=[checkpoint])
# revert to the best model encountered during training
model = load_model("lc_model.h5")
And here is my loss function:
import tensorflow as tf
def focal_loss(gamma=2., alpha=4.):
gamma = float(gamma)
alpha = float(alpha)
def focal_loss_fixed(y_true, y_pred):
"""Focal loss for multi-classification
FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
Notice: y_pred is probability after softmax
gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
Focal Loss for Dense Object Detection
https://arxiv.org/abs/1708.02002
Arguments:
y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
y_pred {tensor} -- model's output, shape of [batch_size, num_cls]
Keyword Arguments:
gamma {float} -- (default: {2.0})
alpha {float} -- (default: {4.0})
Returns:
[tensor] -- loss.
"""
epsilon = 1.e-9
y_true = tf.convert_to_tensor(y_true, tf.float32)
y_pred = tf.convert_to_tensor(y_pred, tf.float32)
model_out = tf.add(y_pred, epsilon)
ce = tf.multiply(y_true, -tf.log(model_out))
weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
fl = tf.multiply(alpha, tf.multiply(weight, ce))
reduced_fl = tf.reduce_max(fl, axis=1)
return tf.reduce_mean(reduced_fl)
return focal_loss_fixed
# model.compile(loss=focal_loss(alpha=1), optimizer='nadam', metrics=['accuracy'])
# model.fit(X_train, y_train, epochs=3, batch_size=1000)
You have to load the custom_objects of focal_loss_fixed as shown below:
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
However, if you wish to just perform inference with your model and not further optimization or training your model, you can simply wish to ignore the loss function like this:
model = load_model("lc_model.h5", compile=False)
The answer by #Prasad is great, but I would like to add a little explanation and a little correction:
while mentioning your custom loss function in the custom_objects dictionary you don't have to call your loss function, as it can give some parameter missing errors.
# Instead of this
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss()})
# ty this without calling your loss function
model = load_model("lc_model.h5", custom_objects={'focal_loss_fixed': focal_loss})
Also, the thing I would like to add here is that you have to use the name of your custom loss function as a key and your function object as its value in custom_objects. I know this is very basic but the thing to mention but I hope this will be helpful to someone.
I have been attempting to use the hyperas wrapper to search for optimal hyperparameters. Right now I am keeping it relatively basic to get it working first. Unfortunately, I am getting an "invalid argument" error. I am working in jupyter notebook.
I have read that I might have to not use jupyter notebook but I would like to continue using it if possible.
import keras
# prevents memory allocation errors when using GPU
from keras import backend as K
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
from keras.models import Sequential
from keras.layers import Dropout, Dense, Activation
from keras.regularizers import l2
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam, sgd
from keras.activations import relu
from keras.losses import categorical_crossentropy
from keras import metrics
import pandas as pd
import numpy as np
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
#categorical_labels = to_categorical(int_labels, num_classes=None)
last_layer = 1
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
from hyperopt import Trials, STATUS_OK, tpe
from keras.utils import np_utils
from hyperas import optim
from keras.models import model_from_json
from hyperas.distributions import choice, uniform, conditional
# Data()
def data():
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
def model(x_train, y_train, x_val, y_val, layers=[20, 20, 4],
kernel_init ='he_uniform', bias_init ='he_uniform',
batch_norm=True, dropout=True):
model = Sequential()
# layer 1
model.add(Dense({{choice([10, 20, 30, 50])}},
input_dim=x_train.shape[1],
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout({{uniform(0, 0.5)}}))
# layer 2+
for layer in range(0, len(layers)-1):
model.add(Dense({{choice([10, 20, 30, 50])}},
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout(params['dropout']))
# Last layer
model.add(Dense(layers[-1], activation=params['last_activation'],
kernel_initializer=kernel_init,
bias_initializer=bias_init))
model.compile(loss=params['losses'],
optimizer=keras.optimizers.adam(lr=params['lr']),
metrics=['accuracy'])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size={{choice([5, 10, 30, 60])}},
epochs=params['epochs'],
verbose=1)
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
history_dict = history.history
return {'model':model, 'status': STATUS_OK, 'history_dict':history_dict, 'loss':-acc}
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=2,
trials=Trials())
x_train, y_train, x_val, y_val = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(x_val, y_val))
print("Best performing model chosen hyper-parameters:")
print(best_run)
Error:
OSError Traceback (most recent call last)
<ipython-input-7-7cf7a7c755ab> in <module>()
64 algo=tpe.suggest,
65 max_evals=2,
---> 66 trials=Trials())
67
68 x_train, y_train, x_val, y_val = data()
~\Anaconda3\lib\site-packages\hyperas\optim.py in minimize(model, data, algo, max_evals, trials, functions, rseed, notebook_name, verbose, eval_space, return_space)
65 full_model_string=None,
66 notebook_name=notebook_name,
---> 67 verbose=verbose)
68
69 best_model = None
~\Anaconda3\lib\site-packages\hyperas\optim.py in base_minimizer(model, data, functions, algo, max_evals, trials, rseed, full_model_string, notebook_name, verbose, stack)
94 model_str = full_model_string
95 else:
---> 96 model_str = get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
97 temp_file = './temp_model.py'
98 write_temp_files(model_str, temp_file)
~\Anaconda3\lib\site-packages\hyperas\optim.py in get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
176 else:
177 calling_script_file = os.path.abspath(inspect.stack()[stack][1])
--> 178 with open(calling_script_file, 'r') as f:
179 source = f.read()
180
OSError: [Errno 22] Invalid argument: 'C:\\Users\\Michael\\Desktop\\CSV data-20180807T164633Z-001\\CSV data\\<ipython-input-7-7cf7a7c755ab>'
Jeez I need to read more. I think my question is answered on the github site
notebook adjustment
If you find error like "No such file or directory" or OSError, Err22, you may need add notebook_name='simple_notebook'(assume your current notebook name is simple_notebook) in optim.minimize function like this:
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials(),
notebook_name='simple_notebook')
except I needed to put
notebook='keras_glioma_hyperopt')
the name of my notebook is keras_glioma_hyperopt
Still running into errors but it could be in my set up with my code. I will update as a go along but the above code helped me progressed.
EDIT
finally got it to run.
Problems I ran into:
you really need to put all the data loading into data():
I changed mine to
def data():
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
last_layer = 1
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
I also ran into the error
TypeError: 'generator' object is not subscriptable
if you follow the hyperas github the fix is
pip install networkx==1.11