I am trying to use hyperas to optimize my keras model but I keep getting NameError: processing (function_name) is not defined. I have already looked at this and this example from hyperas and done exactly that. It doesn't seem to work for me.
This is my code:
def processing():
df = pd.read_json('balanced_all.json')
def label (df):
if df['rating'] < 3:
return 0
if df['rating'] > 3:
return 1
df['label'] = df.apply (lambda df: label(df), axis=1)
df = df[['review_text', 'label']]
maxlen = 100
max_words = 2000
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(df['review_text'].values)
sequences = tokenizer.texts_to_sequences(df['review_text'].values)
word_index = tokenizer.word_index
sequences = pad_sequences(sequences, maxlen=maxlen)
labels = pd.get_dummies(df['label']).values
glove_dir = '/home/uttam/Documents/Thesis/Glove'
embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), 'r', encoding='utf-8')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
if i < max_words:
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return sequences, labels, embedding_matrix
def data():
sequences = processing()[0]
labels = processing()[1]
x_train, x_test, y_train, y_test = train_test_split(sequences,labels, test_size = 0.33, random_state = 42)
return x_train, y_train, x_test, y_test
def create_model(x_train, y_train, x_test, y_test):
embedding_dim = 100
max_words = 2000
embedding_matrix = processing()[2]
model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=100))
model.add(LSTM(128))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(2, activation='sigmoid'))
model.layers[0].set_weights([embedding_matrix])
model.layers[0].trainable = False
model.compile(optimizer={{choice(['rmsprop', 'adam', 'sgd'])}}, loss='binary_crossentropy',metrics=['acc'])
result = model.fit(x_train, y_train, epochs=20, batch_size={{choice([64, 128])}}, validation_split=0.2)
model.save('pre_trained_glove_model.h5')
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
if __name__ == '__main__':
best_run, best_model = optim.minimize(model=create_model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials())
x_train, y_train, x_test, y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(x_test, y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
I don't even need the intermediate function, I had to create it because hyperas didn't find the global variable. for e.g. if I had a variable x outside the hyperas function say create_model(), It would say NameError: x is not defined
I need this because as you can see I am using pre-trained glove embedding. I can't put everything in either data() or create_model(). For e.g. data()needs the variable sequences and label and create_model needs the variable embedding_matrix, so there is no way (as far as I know) to split everything in two functions.
Only way this worked for me was by putting everything in both data() and create_model() functions, which definitely is not efficient and not the way to do.
A little bit late but for future reference, you are right hyperas doesn't recognize global variables. You can pass the function in a list of functions in minimize:
best_run, best_model = optim.minimize(model=create_model,
data=data,
functions=[processing], # <<
algo=tpe.suggest,
max_evals=5,
trials=Trials())
As you mentioned if you need to pass a global variable in hyperas. You can choose one of these options:
Using data():
def data():
## ... my code ...
return x_train, y_train, x_test, y_test, foo
def create_model(x_train, y_train, x_test, y_test, foo):
or defining a new function and pass it in the list of functions:
def my_funct():
return foo
def data():
return x_train, y_train, x_test, y_test
def create_model(x_train, y_train, x_test, y_test):
foo = my_funct()
best_run, best_model = optim.minimize(model=create_model,
data=data,
functions=[my_funct], # << foo
algo=tpe.suggest,
max_evals=5,
trials=Trials())
Related
I'm using Python. How can the GridSearchCV or RandomSearchCV be easily integrated in the network training?
I receive the following error code:
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'
Does anyone have a simple idea to get around this error?
def train_test_splitting(df):
x_train,x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2, random_state = 123, shuffle = False)
return [x_train,x_test, y_train, y_test]
def preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features):
TimeseriesGenerator(features, target, length = 6 , sampling_rate = 1, batch_size =1)[0]
win_length = time_period_window #Wie viele Datenpunkte
batch_size = num_oberservations_per_batch # wie viele Beobachtungswertee pro Iteration
num_features = number_training_features
nn_train_generator= TimeseriesGenerator(x_train, y_train, length = win_length , sampling_rate = 1, batch_size = batch_size)
nn_test_generator= TimeseriesGenerator(x_test, y_test, length = win_length , sampling_rate = 1, batch_size = batch_size)
return [nn_train_generator, nn_test_generator]
#Applying Functions:
x_train,x_test, y_train, y_test = train_test_splitting(df)
nn_train_generator,nn_test_generator = preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features)
def create_LSTM_network(optimizer='adam'):
model = Sequential()
model.add(LSTM(units=120, return_sequences=True, input_shape=(time_period_window,number_training_features)))
model.add(Dropout(0.5))
model.add(LSTM(units=120, return_sequences=False))
model.add(Dense(1, activation = "relu"))
model.compile(loss='mse', optimizer=optimizer,
metrics=['mae'])
return model
#Wrapper für Scikit Learn zur Benutztung von Randomised CV
LSTM_neural_network = KerasClassifier(build_fn=create_LSTM_network, verbose=1)
#Create hyperparameter space
epochs = sp_randInt(10,500)
batches = sp_randInt(10,100)
optimizers = ['rmsprop','adam','sgd']
hyperparameters = dict(optimizer = optimizers, epochs=epochs, batch_size = batches)
#RandomSearchCV - Creation
grid = RandomizedSearchCV(estimator = LSTM_neural_network, param_distributions=hyperparameters,
cv= 2, n_iter = 5, n_jobs =-1)
grid_result = grid.fit(nn_train_generator, epochs = 50,
validation_data = nn_test_generator,
shuffle=False,
callbacks=[early_stopping])
print(); print(grid_result.best_params_)
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'
I am currently training my neural network. Unfortunately I forgot to reserve several samples for validation. How can I incorporate this?
I have a dok matrix that creates a sparisty matrix and then converts the data with get_train_sampels(). How can I now incorporate the validation sampelsfor my code?
Example from https://www.tensorflow.org/guide/keras/train_and_evaluate:
x_val = x_train[-10000:]
y_val = y_train[-10000:]
x_train = x_train[:-10000]
y_train = y_train[:-10000]
.
.
.
print("Fit model on training data")
history = model.fit(
x_train,
y_train,
batch_size=64,
epochs=2,
# We pass some validation for
# monitoring validation loss and metrics
# at the end of each epoch
validation_data=(x_val, y_val),
)
My Code:
def get_train_samples(train_mat, num_negatives):
user_input, item_input, labels = [], [], []
num_user, num_item = train_mat.shape
for (u, i) in train_mat.keys():
user_input.append(u)
item_input.append(i)
labels.append(1)
# negative instances
for t in range(num_negatives):
j = np.random.randint(num_item)
while (u, j) in train_mat.keys():
j = np.random.randint(num_item)
user_input.append(u)
item_input.append(j)
labels.append(0)
return user_input, item_input, labels
.
.
.
train_mat = sp.load_npz('matrix.npz')
num_users, num_items = train_mat.shape
.
.
.
model = get_model(num_users, num_items, latent_dim, dense_layers, reg_layers, reg_mf[0])
model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
user_input, item_input, labels = get_train_samples(train_mat, num_negatives)
hist = model.fit([np.array(user_input), np.array(item_input)], np.array(labels)
, epochs=epochs, verbose=verbose, shuffle=True, batch_size = batch_size)
Edit
Try this:
train_mat = sp.load_npz('matrix.npz')
val_mat = train_mat[-10000:]
train_mat = train_mat[:-10000]
num_users, num_items = train_mat.shape
.
.
.
model = get_model(num_users, num_items, latent_dim, dense_layers, reg_layers, reg_mf[0])
model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['accuracy'])
user_input, item_input, labels = get_train_samples(train_mat, num_negatives)
val_user_input, val_item_input, val_labels = get_train_samples(train_mat, num_negatives)
hist = model.fit([np.array(user_input), np.array(item_input)],
np.array(labels),
epochs=epochs,
verbose=verbose,
validation_data=([np.array(val_user_input), np.array(val_item_input)],
np.array(val_labels)),
shuffle=True, batch_size = batch_size)
You have to restart training. Otherwise your valuation loss may not show overfitting.
When trying to build my data set an error of "TypeError: 'set' object is not subscriptable" is received.
dataDir = '/content/drive/My Drive/Colab Notebooks/HW 3/' # Directory with input files
trainFile = 'q2train.csv' # Training examples
labelFile = 'q2label.csv' # Test label
validFile = 'q2valid.csv' # Valid Files
train = pd.read_csv(dataDir+trainFile)
valid = pd.read_csv(dataDir+validFile)
label = pd.read_csv(dataDir+labelFile)
data_sets = {
'train',
'label',
'valid'}
def get_data(data_set_name, test_prop=0.2, seed=2019):
"""returns data for training, testing, and data characteristics"""
data = data_sets[data_set_name]
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=test_prop,
random_state=seed)
nF = X.shape[1] # number of features
nC = len(np.unique(y)) # number of classes
nTrain, nTest = len(y_train), len(y_test)
print("\nData set: %s" %data_set_name)
print("\tNumber of features %d" %nF)
print("\tNumber of output classes = %d" %(nC))
print("\tNumber of training examples = %d" %(nTrain))
print("\tNumber of testing examples = %d" %(nTest))
return X_train, X_test, y_train, y_test, nF, nC, nTrain, nTest
for name in data_set:
X_train, X_test, y_train, y_test, nF, nC, nTrain, nTest = get_data(name)
Any help would be appreciated, thanks in advance.
Use a dictionary:
train = pd.read_csv(dataDir+trainFile)
valid = pd.read_csv(dataDir+validFile)
label = pd.read_csv(dataDir+labelFile)
data_sets = {
'train': train,
'label': label,
'valid': valid
}
Then data_sets[data_set_name] will retrieve the dataset you want.
Hello i have problem with GridSearchCV it works perfectly on mnist_dataset but not on my own data, and i don't know why.
# df = pd.read_csv('bank-full.csv',sep=';')
# print(df.head())
#
# print(df.shape)
#
# print(df.columns)
# print(df.info)
# df.columns = [col.replace('"', '') for col in df.columns]
#
#
# df.drop(columns=['day', 'poutcome'], axis =1 , inplace=True)
#
#
# print(df.head())
# print(df.shape)
#
# le = preprocessing.LabelEncoder()
# df.job = le.fit_transform(df.job)
# df.education = le.fit_transform(df.education)
# df.housing = le.fit_transform(df.housing)
# df.loan = le.fit_transform(df.loan)
# #df.poutcome = le.fit_transform(df.poutcome)
# df.month = le.fit_transform(df.month)
# df.contact = le.fit_transform(df.contact)
# df.marital = le.fit_transform(df.marital)
# df.default = le.fit_transform(df.default)
# df.y = le.fit_transform(df.y)
#
#
#
# print(df.head())
#
# X = df.iloc[:, 0:14]
# y = df.iloc[:, 14]
# X = np.array(X, dtype="float64")
# y = np.array(y,dtype="float64")
#
# scaler = Normalizer()
# X = scaler.fit_transform(X)
#
#
#
#
# x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=0)
# model = LogisticRegression(penalty='l2', max_iter=1000)
# model.fit(x_train, y_train)
# prediction = model.predict(x_test)
# from sklearn.metrics import accuracy_score
# print("ACC: {} ".format(accuracy_score(y_test, prediction)))
#
#
# print(x_train.shape)
#
# nn = Sequential()
# nn.add(Dense(120,input_dim = 14, activation='relu'))
# nn.add(Dense(240,activation='relu'))
#
#
# nn.add(Dense(1))
# nn.add(Activation('sigmoid'))
#
# nn.compile(loss=keras.losses.binary_crossentropy,
# optimizer='sgd',
# metrics=['accuracy'])
#
# nn.fit(x_train, y_train,
# batch_size=10,
# epochs=10,
# verbose=1,
#
# validation_data=(x_test, y_test))
#
# loss_acc = nn.evaluate(x_test, y_test, verbose=0)
# print('Test loss:', loss_acc[0])
# print('Test accuracy:', loss_acc[1])
data = bm.load_data('bank-full.csv')
data = bm.preprocess_data(data)
X,y = bm.split_data(data)
scaler = Normalizer()
X = scaler.fit_transform(X)
x_train, x_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.1, random_state=0)
start = time()
model = KerasClassifier(build_fn=nnmodel.create_model())
optimizers = ['rmsprop', 'adam']
init = ['glorot_uniform', 'normal', 'uniform']
epochs = np.array([50, 100, 150])
batches = np.array([5, 10, 20])
param_grid = dict(optimizer=optimizers, nb_epoch=epochs, batch_size=batches, init=init)
grid = GridSearchCV(estimator=model, param_grid=param_grid)
grid_result = grid.fit(x_train, y_train)
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
for params, mean_score, scores in grid_result.grid_scores_:
print("%f (%f) with: %r" % (scores.mean(), scores.std(), params))
print("total time:", time() - start)
this commented section, is just a simple keras model, that works perfectrly but below if i try gridSearchCV on this model, it gives me this errors:
https://pastebin.com/mhJLSXAS , for example if i run this program https://www.kaggle.com/shujunge/gridsearchcv-with-keras it works perferctly byt on my data it's not, does somebody know why ?
Scikitlearn builds each time new model. Script has to build a classifier with specific parameters inside the grid search method. So you have to send the method name as an argument, not the result of it.
Probably nnmodel.create_model is your function which creates a new model based on parameters. So try to change:
build_fn=nnmodel.create_model()
To:
build_fn=nnmodel.create_model
I have been attempting to use the hyperas wrapper to search for optimal hyperparameters. Right now I am keeping it relatively basic to get it working first. Unfortunately, I am getting an "invalid argument" error. I am working in jupyter notebook.
I have read that I might have to not use jupyter notebook but I would like to continue using it if possible.
import keras
# prevents memory allocation errors when using GPU
from keras import backend as K
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
from keras.models import Sequential
from keras.layers import Dropout, Dense, Activation
from keras.regularizers import l2
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam, sgd
from keras.activations import relu
from keras.losses import categorical_crossentropy
from keras import metrics
import pandas as pd
import numpy as np
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
#categorical_labels = to_categorical(int_labels, num_classes=None)
last_layer = 1
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
from hyperopt import Trials, STATUS_OK, tpe
from keras.utils import np_utils
from hyperas import optim
from keras.models import model_from_json
from hyperas.distributions import choice, uniform, conditional
# Data()
def data():
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
def model(x_train, y_train, x_val, y_val, layers=[20, 20, 4],
kernel_init ='he_uniform', bias_init ='he_uniform',
batch_norm=True, dropout=True):
model = Sequential()
# layer 1
model.add(Dense({{choice([10, 20, 30, 50])}},
input_dim=x_train.shape[1],
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout({{uniform(0, 0.5)}}))
# layer 2+
for layer in range(0, len(layers)-1):
model.add(Dense({{choice([10, 20, 30, 50])}},
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout(params['dropout']))
# Last layer
model.add(Dense(layers[-1], activation=params['last_activation'],
kernel_initializer=kernel_init,
bias_initializer=bias_init))
model.compile(loss=params['losses'],
optimizer=keras.optimizers.adam(lr=params['lr']),
metrics=['accuracy'])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size={{choice([5, 10, 30, 60])}},
epochs=params['epochs'],
verbose=1)
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
history_dict = history.history
return {'model':model, 'status': STATUS_OK, 'history_dict':history_dict, 'loss':-acc}
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=2,
trials=Trials())
x_train, y_train, x_val, y_val = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(x_val, y_val))
print("Best performing model chosen hyper-parameters:")
print(best_run)
Error:
OSError Traceback (most recent call last)
<ipython-input-7-7cf7a7c755ab> in <module>()
64 algo=tpe.suggest,
65 max_evals=2,
---> 66 trials=Trials())
67
68 x_train, y_train, x_val, y_val = data()
~\Anaconda3\lib\site-packages\hyperas\optim.py in minimize(model, data, algo, max_evals, trials, functions, rseed, notebook_name, verbose, eval_space, return_space)
65 full_model_string=None,
66 notebook_name=notebook_name,
---> 67 verbose=verbose)
68
69 best_model = None
~\Anaconda3\lib\site-packages\hyperas\optim.py in base_minimizer(model, data, functions, algo, max_evals, trials, rseed, full_model_string, notebook_name, verbose, stack)
94 model_str = full_model_string
95 else:
---> 96 model_str = get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
97 temp_file = './temp_model.py'
98 write_temp_files(model_str, temp_file)
~\Anaconda3\lib\site-packages\hyperas\optim.py in get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
176 else:
177 calling_script_file = os.path.abspath(inspect.stack()[stack][1])
--> 178 with open(calling_script_file, 'r') as f:
179 source = f.read()
180
OSError: [Errno 22] Invalid argument: 'C:\\Users\\Michael\\Desktop\\CSV data-20180807T164633Z-001\\CSV data\\<ipython-input-7-7cf7a7c755ab>'
Jeez I need to read more. I think my question is answered on the github site
notebook adjustment
If you find error like "No such file or directory" or OSError, Err22, you may need add notebook_name='simple_notebook'(assume your current notebook name is simple_notebook) in optim.minimize function like this:
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials(),
notebook_name='simple_notebook')
except I needed to put
notebook='keras_glioma_hyperopt')
the name of my notebook is keras_glioma_hyperopt
Still running into errors but it could be in my set up with my code. I will update as a go along but the above code helped me progressed.
EDIT
finally got it to run.
Problems I ran into:
you really need to put all the data loading into data():
I changed mine to
def data():
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
last_layer = 1
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
I also ran into the error
TypeError: 'generator' object is not subscriptable
if you follow the hyperas github the fix is
pip install networkx==1.11