Develop Reference

How to optimize hyperparameters with RandomSearchCV or GridSearchCV in neural network for beginners?

I'm using Python. How can the GridSearchCV or RandomSearchCV be easily integrated in the network training?
I receive the following error code:
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'
Does anyone have a simple idea to get around this error?
def train_test_splitting(df):
x_train,x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2, random_state = 123, shuffle = False)
return [x_train,x_test, y_train, y_test]
def preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features):
TimeseriesGenerator(features, target, length = 6 , sampling_rate = 1, batch_size =1)[0]
win_length = time_period_window #Wie viele Datenpunkte
batch_size = num_oberservations_per_batch # wie viele Beobachtungswertee pro Iteration
num_features = number_training_features
nn_train_generator= TimeseriesGenerator(x_train, y_train, length = win_length , sampling_rate = 1, batch_size = batch_size)
nn_test_generator= TimeseriesGenerator(x_test, y_test, length = win_length , sampling_rate = 1, batch_size = batch_size)
return [nn_train_generator, nn_test_generator]
#Applying Functions:
x_train,x_test, y_train, y_test = train_test_splitting(df)
nn_train_generator,nn_test_generator = preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features)
def create_LSTM_network(optimizer='adam'):
model = Sequential()
model.add(LSTM(units=120, return_sequences=True, input_shape=(time_period_window,number_training_features)))
model.add(Dropout(0.5))
model.add(LSTM(units=120, return_sequences=False))
model.add(Dense(1, activation = "relu"))
model.compile(loss='mse', optimizer=optimizer,
metrics=['mae'])
return model
#Wrapper für Scikit Learn zur Benutztung von Randomised CV
LSTM_neural_network = KerasClassifier(build_fn=create_LSTM_network, verbose=1)
#Create hyperparameter space
epochs = sp_randInt(10,500)
batches = sp_randInt(10,100)
optimizers = ['rmsprop','adam','sgd']
hyperparameters = dict(optimizer = optimizers, epochs=epochs, batch_size = batches)
#RandomSearchCV - Creation
grid = RandomizedSearchCV(estimator = LSTM_neural_network, param_distributions=hyperparameters,
cv= 2, n_iter = 5, n_jobs =-1)
grid_result = grid.fit(nn_train_generator, epochs = 50,
validation_data = nn_test_generator,
shuffle=False,
callbacks=[early_stopping])
print(); print(grid_result.best_params_)
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'

"ValueError: Data cardinality is ambiguous" Having trouble copying tensor flow tutorial for basic classifier. Using Python

I'm following along with Laurence Moroney at tensorflow, coding as I go. Here's the video:
https://www.youtube.com/watch?v=Y_hzMnRXjhI
I'm getting this error message:
Traceback (most recent call last):
File "tensorTest.py", line 66, in <module>
validation_data=(testing_padded, testing_labels), verbose=2)
File "/Users/elliot/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 108, in _method_wrapper
return method(self, *args, **kwargs)
File "/Users/elliot/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 1063, in fit
steps_per_execution=self._steps_per_execution)
File "/Users/elliot/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 1117, in __init__
model=model)
File "/Users/elliot/opt/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/data_adapter.py", line 282, in __init__
raise ValueError(msg)
ValueError: Data cardinality is ambiguous:
x sizes: 28618
y sizes: 14309
Please provide data which shares the same first dimension.
I can get the model summary just fine, but when it tries to fit I get the error. Full code below:
import tensorflow as tf from tensorflow import keras from tensorflow.keras.preprocessing.text import Tokenizer from tensorflow.keras.preprocessing.sequence import pad_sequences import pandas as pd
data = pd.read_json("Sarcasm_Headlines_Dataset_v2.json", lines=True)
sentences = data['headline'].to_list() labels = data['is_sarcastic'].to_list() urls = data['article_link'].to_list()
vocab_size = 10000 embedding_dim = 16 max_length = 100 trunc_type='post' padding_type='post' oov_tok = "<OOV>" training_size
= 14309
training_sentences = sentences[0:training_size] testing_sentences = sentences[:training_size] training_labels = labels[0:training_size] testing_labels = labels[training_size:]
tokenizer = Tokenizer(oov_token = oov_tok) tokenizer.fit_on_texts(training_sentences)
word_index = tokenizer.word_index
training_sequences = tokenizer.texts_to_sequences(sentences) training_padded = pad_sequences(training_sequences, padding=padding_type, maxlen=max_length, truncating=trunc_type)
testing_sequences = tokenizer.texts_to_sequences(testing_sentences) testing_padded = pad_sequences(testing_sequences, padding=padding_type, maxlen=max_length, truncating=trunc_type)
import numpy as np training_padded = np.array(training_padded) training_labels = np.array(training_labels) testing_padded = np.array(testing_padded) testing_labels = np.array(testing_labels)
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D(),
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid') ])
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary()
num_epochs = 30
history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=2)
print(history)
THANKS IN ADVANCE!

How to do kfold cross-validation for multi-input models

The model is as below:
inputs_1 = keras.Input(shape=(10081,1))
layer1 = Conv1D(64,14)(inputs_1)
layer2 = layers.MaxPool1D(5)(layer1)
layer3 = Conv1D(64, 14)(layer2)
layer4 = layers.GlobalMaxPooling1D()(layer3)
inputs_2 = keras.Input(shape=(85,))
layer5 = layers.concatenate([layer4, inputs_2])
layer6 = Dense(128, activation='relu')(layer5)
layer7 = Dense(2, activation='softmax')(layer6)
model_2 = keras.models.Model(inputs = [inputs_1, inputs_2], output = [layer7])
X_train, X_test, y_train, y_test = train_test_split(df.iloc[:,0:10166], df[['Result_cat','Result_cat1']].values, test_size=0.2)
X_train = X_train.to_numpy()
X_train = X_train.reshape([X_train.shape[0], X_train.shape[1], 1])
X_train_1 = X_train[:,0:10081,:]
X_train_2 = X_train[:,10081:10166,:].reshape(736,85)
X_test = X_test.to_numpy()
X_test = X_test.reshape([X_test.shape[0], X_test.shape[1], 1])
X_test_1 = X_test[:,0:10081,:]
X_test_2 = X_test[:,10081:10166,:].reshape(185,85)
adam = keras.optimizers.Adam(lr = 0.0005)
model_2.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['acc'])
history = model_2.fit([X_train_1,X_train_2], y_train, epochs = 120, batch_size = 256, validation_split = 0.2, callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)])
Questions:
1) The data is 921rows x 10166columns. Each row is an observation(first 10080 columns being a time series with remaining columns being other statistics features). According to the model, is the input data split into inputs_1 and inputs_2 randomly?
2) I am thinking about doing a kfold cross-validation and splitting the input data into inputs_1 and inputs_2. What is a good way to do this? Thanks

By splitting only indexes.
num_folds = 5
kfold = KFold(n_splits=num_folds, shuffle=False)
ID_Inp = np.array(range(nSamples))
ID_Out = np.array(range(nSamples))
Inputs = [Input1,Input2]
for IDs_Train, IDs_Test in kfold.split(ID_Inp, ID_Out):
Fold_Train_Input1, Fold_Train_Input2 = Input1[IDs_Train], Input2[IDs_Train]
Fold_Train_OutPut = Output[IDs_Train]
Fold_Test_Input1, Fold_Test_Input2 = Input1[IDs_Test], Input2[IDs_Test]
Fold_Test_OutPut = Output[IDs_Test]
####################

Prediction using Keras NLP

I am a beginner in the field of Neural Networks.
I am trying to implement an LSTM model for predicting the secondary structure of a protein from a given primary sequence. The base kernel on which my program is based can be found on Kaggle - https://www.kaggle.com/helmehelmuto/secondary-structure-prediction-with-keras
I successfully trained the model, saved the model to a pickle file, and I am able to use the pickle file to load weights to my model and make predictions as well. However, these predictions are on the test set which was created by the test_train_split function in Keras.
I aim to feed in a String containing the sequence of a protein and get its secondary structure prediction.
The code uses Tokenizer to convert the data (the protein sequences) from the dataset into a numpy.ndarray which is used for making predictions.
The part where I am stuck is, taking a string as input(some protein sequence) and converting it into the same class and then making predictions on it.
I have tried using the same method the Kaggle Kernel author used for converting the data from the .csv file to a numpy.ndarray but I get an error - 'numpy.ndarray' object has no attribute 'lower'.
I would be grateful if someone could guide me here, converting strings to the objects which are used for prediction in the code.
Code (works fine, predictions made directly on the test set generated by test_train_split):
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle
df = pd.read_csv(r'C:\Users\Viktor\Desktop\2018-06-06-ss.cleaned.csv')
df.len.hist(bins=100)
print(df.shape)
def seq2ngrams(seqs, n=3):
return np.array([[seq[i:i+n] for i in range(len(seq))] for seq in seqs])
maxlen_seq = 128
input_seqs, target_seqs = df[['seq', 'sst3']][(df.len <= maxlen_seq) & (~df.has_nonstd_aa)].values.T
input_grams = seq2ngrams(input_seqs)
print(len(input_seqs))
from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.utils import to_categorical
tokenizer_encoder = Tokenizer()
tokenizer_encoder.fit_on_texts(input_grams)
input_data = tokenizer_encoder.texts_to_sequences(input_grams)
input_data = sequence.pad_sequences(input_data, maxlen=maxlen_seq, padding='post')
tokenizer_decoder = Tokenizer(char_level=True)
tokenizer_decoder.fit_on_texts(target_seqs)
target_data = tokenizer_decoder.texts_to_sequences(target_seqs)
target_data = sequence.pad_sequences(target_data, maxlen=maxlen_seq, padding='post')
target_data = to_categorical(target_data)
input_data.shape, target_data.shape
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Bidirectional
n_words = len(tokenizer_encoder.word_index) + 1
n_tags = len(tokenizer_decoder.word_index) + 1
print(n_words, n_tags)
input = Input(shape=(maxlen_seq,))
x = Embedding(input_dim=n_words, output_dim=128, input_length=maxlen_seq)(input)
x = Bidirectional(LSTM(units=64, return_sequences=True, recurrent_dropout=0.1))(x)
y = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input, y)
model.summary()
from sklearn.model_selection import train_test_split
from keras.metrics import categorical_accuracy
from keras import backend as K
import tensorflow as tf
def q3_acc(y_true, y_pred):
y = tf.argmax(y_true, axis=-1)
y_ = tf.argmax(y_pred, axis=-1)
mask = tf.greater(y, 0)
return K.cast(K.equal(tf.boolean_mask(y, mask), tf.boolean_mask(y_, mask)), K.floatx())
model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy", q3_acc])
X_train, X_test, y_train, y_test = train_test_split(input_data, target_data, test_size=.4, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs, target_seqs, test_size=.4, random_state=0)
#model.fit(X_train, y_train, batch_size=128, epochs=5, validation_data=(X_test, y_test), verbose=1)
def onehot_to_seq(oh_seq, index):
s = ''
for o in oh_seq:
i = np.argmax(o)
if i != 0:
s += index[i]
else:
break
return s
def plot_results(x, y, y_):
print("---")
print("Input: " + str(x))
print("Target: " + str(onehot_to_seq(y, revsere_decoder_index).upper()))
print("Result: " + str(onehot_to_seq(y_, revsere_decoder_index).upper()))
fig = plt.figure(figsize=(10,2))
plt.imshow(y.T, cmap='Blues')
plt.imshow(y_.T, cmap='Reds', alpha=.5)
plt.yticks(range(4), [' '] + [revsere_decoder_index[i+1].upper() for i in range(3)])
plt.show()
revsere_decoder_index = {value:key for key,value in tokenizer_decoder.word_index.items()}
revsere_encoder_index = {value:key for key,value in tokenizer_encoder.word_index.items()}
#N=3
#y_train_pred = model.predict(X_train[:N])
#y_test_pred = model.predict(X_test[:N])
#print('training')
#for i in range(N):
# plot_results(seq_train[i], y_train[i], y_train_pred[i])
#print('testing')
#for i in range(N):
# plot_results(seq_test[i], y_test[i], y_test_pred[i])
loaded_model = pickle.load(open( "save.p", "rb" ))
N=3
y_train_pred = loaded_model.predict(X_train[:N])
y_test_pred = loaded_model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
#print(type(target_seqs))
CODE WHICH DOES NOT WORK AS EXPECTED:
In this, 'xf' is the csv file which has the sequences for which I require the predictions.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle
df = pd.read_csv(r'C:\Users\Viktor\Desktop\2018-06-06-ss.cleaned.csv')
xf = pd.read_csv(r'C:\Users\Viktor\Desktop\sequence.csv')
df.len.hist(bins=100)
print(df.shape)
def seq2ngrams(seqs, n=3):
return np.array([[seq[i:i+n] for i in range(len(seq))] for seq in seqs])
maxlen_seq = 128
input_seqs, target_seqs = df[['seq', 'sst3']][(df.len <= maxlen_seq) & (~df.has_nonstd_aa)].values.T
input_grams = seq2ngrams(input_seqs)
print(len(input_seqs))
from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.utils import to_categorical
tokenizer_encoder = Tokenizer()
tokenizer_encoder.fit_on_texts(input_grams)
input_data = tokenizer_encoder.texts_to_sequences(input_grams)
input_data = sequence.pad_sequences(input_data, maxlen=maxlen_seq, padding='post')
tokenizer_decoder = Tokenizer(char_level=True)
tokenizer_decoder.fit_on_texts(target_seqs)
target_data = tokenizer_decoder.texts_to_sequences(target_seqs)
target_data = sequence.pad_sequences(target_data, maxlen=maxlen_seq, padding='post')
target_data = to_categorical(target_data)
input_data.shape, target_data.shape
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Bidirectional
n_words = len(tokenizer_encoder.word_index) + 1
n_tags = len(tokenizer_decoder.word_index) + 1
print(n_words, n_tags)
input = Input(shape=(maxlen_seq,))
x = Embedding(input_dim=n_words, output_dim=128, input_length=maxlen_seq)(input)
x = Bidirectional(LSTM(units=64, return_sequences=True, recurrent_dropout=0.1))(x)
y = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input, y)
model.summary()
from sklearn.model_selection import train_test_split
from keras.metrics import categorical_accuracy
from keras import backend as K
import tensorflow as tf
model.compile(optimizer="rmsprop", loss="categorical_crossentropy")
X_train, X_test, y_train, y_test = train_test_split(input_data, target_data, test_size=.4, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs, target_seqs, test_size=.4, random_state=0)
#model.fit(X_train, y_train, batch_size=128, epochs=1, validation_data=(X_test, y_test), verbose=1)
def onehot_to_seq(oh_seq, index):
s = ''
for o in oh_seq:
i = np.argmax(o)
if i != 0:
s += index[i]
else:
break
return s
def plot_results(x, y, y_):
print("---")
print("Input: " + str(x))
print("Target: " + str(onehot_to_seq(y, revsere_decoder_index).upper()))
print("Result: " + str(onehot_to_seq(y_, revsere_decoder_index).upper()))
fig = plt.figure(figsize=(10,2))
plt.imshow(y.T, cmap='Blues')
plt.imshow(y_.T, cmap='Reds', alpha=.5)
plt.yticks(range(4), [' '] + [revsere_decoder_index[i+1].upper() for i in range(3)])
plt.show()
revsere_decoder_index = {value:key for key,value in tokenizer_decoder.word_index.items()}
revsere_encoder_index = {value:key for key,value in tokenizer_encoder.word_index.items()}
N=3
y_train_pred = model.predict(X_train[:N])
y_test_pred = model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
loaded_model = pickle.load(open( "save.p", "rb" ))
N=3
y_train_pred = loaded_model.predict(X_train[:N])
y_test_pred = loaded_model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
print("-----")
print(X_test[:3])
print("-----")
xf.len.hist(bins=100)
input_seqs1, target_seqs1 = xf[['seq', 'sst3']][(xf.len <= maxlen_seq) & (~xf.has_nonstd_aa)].values.T
input_grams1 = seq2ngrams(input_seqs1)
tokenizer_encoder1 = Tokenizer()
tokenizer_encoder1.fit_on_texts(input_grams1)
input_data1 = tokenizer_encoder1.texts_to_sequences(input_grams1)
input_data1 = sequence.pad_sequences(input_data1, maxlen=maxlen_seq, padding='post')
tokenizer_decoder1 = Tokenizer(char_level=True)
tokenizer_decoder1.fit_on_texts(target_seqs1)
target_data1 = tokenizer_decoder1.texts_to_sequences(target_seqs1)
target_data1 = sequence.pad_sequences(target_data1, maxlen=maxlen_seq, padding='post')
target_data1 = to_categorical(target_data1)
input_data1.shape, target_data1.shape
X_train, X_test, y_train, y_test = train_test_split(input_data1, target_data1, test_size=1, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs1, target_seqs1, test_size=1, random_state=0)
y_train_pred1 = loaded_model.predict(X_train)
y_test_pred1 = loaded_model.predict(X_test)
plot_results(seq_train, y_train, y_train_pred)
plot_results(seq_test, y_test, y_test_pred)
#print(input_data1[0])
##y_train_pred = loaded_model.predict(input_data)
#y_test_pred1 = loaded_model.predict(input_data1[0])
##plot_results(seq_train, y_train, y_train_pred)
#plot_results(input_seqs1, target_data1, y_test_pred1)
TRACEBACK:
Traceback (most recent call last):
File "<ipython-input-38-e8f27dda0841>", line 1, in <module>
runfile('C:/Users/Viktor/Desktop/rost_nocl.py', wdir='C:/Users/Viktor/Desktop')
File "D:\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 827, in runfile
execfile(filename, namespace)
File "D:\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 110, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/Viktor/Desktop/rost_nocl.py", line 116, in <module>
tokenizer_encoder1.fit_on_texts(input_grams1)
File "D:\Anaconda3\lib\site-packages\keras_preprocessing\text.py", line 223, in fit_on_texts
self.split)
File "D:\Anaconda3\lib\site-packages\keras_preprocessing\text.py", line 43, in text_to_word_sequence
text = text.lower()
AttributeError: 'numpy.ndarray' object has no attribute 'lower'

Invalid agrument error using hyperas in keras

I have been attempting to use the hyperas wrapper to search for optimal hyperparameters. Right now I am keeping it relatively basic to get it working first. Unfortunately, I am getting an "invalid argument" error. I am working in jupyter notebook.
I have read that I might have to not use jupyter notebook but I would like to continue using it if possible.
import keras
# prevents memory allocation errors when using GPU
from keras import backend as K
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
from keras.models import Sequential
from keras.layers import Dropout, Dense, Activation
from keras.regularizers import l2
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam, sgd
from keras.activations import relu
from keras.losses import categorical_crossentropy
from keras import metrics
import pandas as pd
import numpy as np
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
#categorical_labels = to_categorical(int_labels, num_classes=None)
last_layer = 1
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
from hyperopt import Trials, STATUS_OK, tpe
from keras.utils import np_utils
from hyperas import optim
from keras.models import model_from_json
from hyperas.distributions import choice, uniform, conditional
# Data()
def data():
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
def model(x_train, y_train, x_val, y_val, layers=[20, 20, 4],
kernel_init ='he_uniform', bias_init ='he_uniform',
batch_norm=True, dropout=True):
model = Sequential()
# layer 1
model.add(Dense({{choice([10, 20, 30, 50])}},
input_dim=x_train.shape[1],
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout({{uniform(0, 0.5)}}))
# layer 2+
for layer in range(0, len(layers)-1):
model.add(Dense({{choice([10, 20, 30, 50])}},
W_regularizer=l2(l2_reg),
kernel_initializer=kernel_init,
bias_initializer=bias_init))
if batch_norm == True:
model.add(BatchNormalization(axis=-1, momentum=momentum, center=True))
model.add(Activation(params['activation']))
if dropout == True:
model.add(Dropout(params['dropout']))
# Last layer
model.add(Dense(layers[-1], activation=params['last_activation'],
kernel_initializer=kernel_init,
bias_initializer=bias_init))
model.compile(loss=params['losses'],
optimizer=keras.optimizers.adam(lr=params['lr']),
metrics=['accuracy'])
history = model.fit(x_train, y_train,
validation_data=[x_val, y_val],
batch_size={{choice([5, 10, 30, 60])}},
epochs=params['epochs'],
verbose=1)
score, acc = model.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
history_dict = history.history
return {'model':model, 'status': STATUS_OK, 'history_dict':history_dict, 'loss':-acc}
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=2,
trials=Trials())
x_train, y_train, x_val, y_val = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(x_val, y_val))
print("Best performing model chosen hyper-parameters:")
print(best_run)
Error:
OSError Traceback (most recent call last)
<ipython-input-7-7cf7a7c755ab> in <module>()
64 algo=tpe.suggest,
65 max_evals=2,
---> 66 trials=Trials())
67
68 x_train, y_train, x_val, y_val = data()
~\Anaconda3\lib\site-packages\hyperas\optim.py in minimize(model, data, algo, max_evals, trials, functions, rseed, notebook_name, verbose, eval_space, return_space)
65 full_model_string=None,
66 notebook_name=notebook_name,
---> 67 verbose=verbose)
68
69 best_model = None
~\Anaconda3\lib\site-packages\hyperas\optim.py in base_minimizer(model, data, functions, algo, max_evals, trials, rseed, full_model_string, notebook_name, verbose, stack)
94 model_str = full_model_string
95 else:
---> 96 model_str = get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
97 temp_file = './temp_model.py'
98 write_temp_files(model_str, temp_file)
~\Anaconda3\lib\site-packages\hyperas\optim.py in get_hyperopt_model_string(model, data, functions, notebook_name, verbose, stack)
176 else:
177 calling_script_file = os.path.abspath(inspect.stack()[stack][1])
--> 178 with open(calling_script_file, 'r') as f:
179 source = f.read()
180
OSError: [Errno 22] Invalid argument: 'C:\\Users\\Michael\\Desktop\\CSV data-20180807T164633Z-001\\CSV data\\<ipython-input-7-7cf7a7c755ab>'

Jeez I need to read more. I think my question is answered on the github site
notebook adjustment
If you find error like "No such file or directory" or OSError, Err22, you may need add notebook_name='simple_notebook'(assume your current notebook name is simple_notebook) in optim.minimize function like this:
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials(),
notebook_name='simple_notebook')
except I needed to put
notebook='keras_glioma_hyperopt')
the name of my notebook is keras_glioma_hyperopt
Still running into errors but it could be in my set up with my code. I will update as a go along but the above code helped me progressed.
EDIT
finally got it to run.
Problems I ran into:
you really need to put all the data loading into data():
I changed mine to
def data():
# load data
x_main = pd.read_csv("glioma DB X.csv")
y_main = pd.read_csv("glioma DB Y.csv")
# fill with median (will have to improve later, not done yet)
fill_median =['Surgery_SBRT','df','Dose','Ki67','KPS','BMI','tumor_size']
x_main[fill_median] = x_main[fill_median].fillna(x_main[fill_median].median())
x_main['Neurofc'] = x_main['Neurofc'].fillna(2)
x_main['comorbid'] = x_main['comorbid'].fillna(int(x_main['comorbid'].median()))
# drop surgery
x_main = x_main.drop(['Surgery'], axis=1)
# normalize all x
x_main_normalized = x_main.apply(lambda x: (x-np.mean(x))/(np.std(x)+1e-10))
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x_main, y_main, test_size=0.3)
x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=0.5)
last_layer = 1
params = {'lr': 0.0001,
'batch_size': 30,
'epochs': 100,
'dropout': 0.2,
'weight_regulizer':['l2'],
'optimizer': 'sgd',
'losses': 'categorical_crossentropy',
'activation':'relu',
'last_activation': 'softmax'}
from keras.utils.np_utils import to_categorical
if params['losses']=='categorical_crossentropy':
y_train = to_categorical(y_train,num_classes=4)
y_val = to_categorical(y_val,num_classes=4)
y_test = to_categorical(y_test,num_classes=4)
last_layer=4
x_train = x_train
x_val = x_val
y_train = y_train
y_val = y_val
return x_train, y_train, x_val, y_val
I also ran into the error
TypeError: 'generator' object is not subscriptable
if you follow the hyperas github the fix is
pip install networkx==1.11