Develop Reference

How to optimize hyperparameters with RandomSearchCV or GridSearchCV in neural network for beginners?

I'm using Python. How can the GridSearchCV or RandomSearchCV be easily integrated in the network training?
I receive the following error code:
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'
Does anyone have a simple idea to get around this error?
def train_test_splitting(df):
x_train,x_test, y_train, y_test = train_test_split(features, target, test_size = 0.2, random_state = 123, shuffle = False)
return [x_train,x_test, y_train, y_test]
def preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features):
TimeseriesGenerator(features, target, length = 6 , sampling_rate = 1, batch_size =1)[0]
win_length = time_period_window #Wie viele Datenpunkte
batch_size = num_oberservations_per_batch # wie viele Beobachtungswertee pro Iteration
num_features = number_training_features
nn_train_generator= TimeseriesGenerator(x_train, y_train, length = win_length , sampling_rate = 1, batch_size = batch_size)
nn_test_generator= TimeseriesGenerator(x_test, y_test, length = win_length , sampling_rate = 1, batch_size = batch_size)
return [nn_train_generator, nn_test_generator]
#Applying Functions:
x_train,x_test, y_train, y_test = train_test_splitting(df)
nn_train_generator,nn_test_generator = preprocessing_nn(df,time_period_window, num_oberservations_per_batch ,number_training_features)
def create_LSTM_network(optimizer='adam'):
model = Sequential()
model.add(LSTM(units=120, return_sequences=True, input_shape=(time_period_window,number_training_features)))
model.add(Dropout(0.5))
model.add(LSTM(units=120, return_sequences=False))
model.add(Dense(1, activation = "relu"))
model.compile(loss='mse', optimizer=optimizer,
metrics=['mae'])
return model
#Wrapper für Scikit Learn zur Benutztung von Randomised CV
LSTM_neural_network = KerasClassifier(build_fn=create_LSTM_network, verbose=1)
#Create hyperparameter space
epochs = sp_randInt(10,500)
batches = sp_randInt(10,100)
optimizers = ['rmsprop','adam','sgd']
hyperparameters = dict(optimizer = optimizers, epochs=epochs, batch_size = batches)
#RandomSearchCV - Creation
grid = RandomizedSearchCV(estimator = LSTM_neural_network, param_distributions=hyperparameters,
cv= 2, n_iter = 5, n_jobs =-1)
grid_result = grid.fit(nn_train_generator, epochs = 50,
validation_data = nn_test_generator,
shuffle=False,
callbacks=[early_stopping])
print(); print(grid_result.best_params_)
AttributeError: 'RandomizedSearchCV' object has no attribute 'fit_generator'

GridSearchCV results are not reproducable

Im new to Keras and I need your professional help.
I have used GridSearchCV to optmize my regression network. When i try to use the results, the newly created network is far worse in regards to the mean squared error than the one calculated by GridSearch.
The GridSearchCV code:
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from time import time
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from tensorflow import keras
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Flatten, Input, Dropout, LeakyReLU
from keras.utils import plot_model
from keras.optimizers import SGD, rmsprop, adam
from keras.wrappers.scikit_learn import KerasClassifier, KerasRegressor
from keras.initializers import uniform, normal, glorot_uniform
from keras.losses import MAPE
#Data preprocessing
def get_data():
data = pd.read_csv("test.csv", sep=";", usecols=["rHsubLS","b","lowerSetpoint"])
test = data.loc[:,['rHsubLS','b']]
target = data.loc[:,'lowerSetpoint']
print(test.shape)
print(target.shape)
return test.astype(float), target.astype(float)
def split_data(test, target):
X_train, X_test, y_train, y_test = train_test_split(test, target)
X_train = np.array(X_train)
X_test = np.array(X_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
stdsc1 = StandardScaler()
train_data_std = stdsc1.fit_transform(X_train)
test_data_std = stdsc1.fit_transform(X_test)
y_train_1 = np.reshape(y_train, (-1, 1))
y_test_1 = np.reshape(y_test, (-1, 1))
train_target_std = stdsc1.fit_transform(y_train_1)
test_target_std = stdsc1.fit_transform(y_test_1)
return train_data_std, test_data_std, train_target_std, test_target_std
#Network Creation
def create_NN(optimizer='rmsprop', init='glorot_uniform', alpha=0.15, activation_func='tanh'):
NN_model = Sequential()
#input layer
NN_model.add(Dense(128, kernel_initializer=init, input_dim=2, activation=activation_func))
#hidden layers
NN_model.add(LeakyReLU(alpha=alpha))
NN_model.add(Dense(256, kernel_initializer=init, activation='relu'))
#output layer
NN_model.add(Dense(1, kernel_initializer=init, activation='linear'))
NN_model.compile(loss='mean_squared_error', optimizer=optimizer, metrics=["mse", "mape"])
NN_model.summary()
return NN_model
#GridSearchCV
def train_NN(NN_model, train_data, train_target):
seed = 4
np.random.seed(seed)
model = KerasRegressor(build_fn=create_NN, verbose=1)
optimizers = ['rmsprop', 'adam', 'SGD']
inits = ['glorot_uniform', 'normal', 'uniform', 'he_uniform']
activation_funcs = ['tanh','relu','softmax']
epochs = [50, 100, 150]
batches = [50, 100, 500]
alphas = [0.15, 0.45, 0.3]
grid_parameter = dict(optimizer=optimizers, epochs=epochs, batch_size=batches, init=inits, alpha=alphas, activation_func=activation_funcs)#, dropout_rate=dropout)
if __name__ == '__main__':
grid = GridSearchCV(estimator=model, scoring='neg_mean_squared_error' , param_grid=grid_parameter, verbose=1, cv=3)
grid_results = grid.fit(train_data, train_target, use_multiprocessing=True, shuffle=True, workers=8)
print("Best: %f using %s" % (grid_results.best_score_, grid_results.best_params_))
means = grid_results.cv_results_['mean_test_score']
stds = grid_results.cv_results_['std_test_score']
params = grid_results.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
try:
test, target = get_data()
train_data, test_data, train_target, test_target = split_data(test, target)
print("Data split\n")
NN_model = create_NN()
train_NN(NN_model, train_data, train_target)
except (KeyboardInterrupt, SystemExit):
raise
The results of the GridSearch:
Best: -0.000064 using {'activation_func': 'relu', 'alpha': 0.3, 'batch_size': 50, 'epochs': 150, 'init': 'he_uniform', 'optimizer': 'adam'}
When I try to reproduce this network with this code:
import os
import numpy as np
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
from time import time
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from tensorflow import keras
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Flatten, Input, Dropout, PReLU, LeakyReLU
from keras.utils import plot_model
from keras.optimizers import SGD
from keras.losses import MeanAbsolutePercentageError
def get_data():
data = pd.read_csv("test.csv", sep=";", usecols=["rHsubLS","b","lowerSetpoint"])
test = data.loc[:,['rHsubLS','b']]
target = data.loc[:,'lowerSetpoint']
print(test.shape)
print(target.shape)
return test.astype(float), target.astype(float)
def split_data(test, target):
X_train, X_test, y_train, y_test = train_test_split(test, target)
X_train = np.array(X_train)
X_test = np.array(X_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
stdsc1 = StandardScaler()
train_data_std = stdsc1.fit_transform(X_train)
test_data_std = stdsc1.fit_transform(X_test)
y_train_1 = np.reshape(y_train, (-1, 1))
y_test_1 = np.reshape(y_test, (-1, 1))
train_target_std = stdsc1.fit_transform(y_train_1)
test_target_std = stdsc1.fit_transform(y_test_1)
return train_data_std, test_data_std, train_target_std, test_target_std
def create_NN():
NN_model = Sequential()
#input layer
NN_model.add(Dense(128, input_dim=2, kernel_initializer='he_uniform', activation='relu'))
#hidden layers
NN_model.add(LeakyReLU(0.3))
NN_model.add(Dense(256, kernel_initializer='he_uniform', activation='relu'))
#output layer
NN_model.add(Dense(1, activation='linear'))
keras.backend.set_epsilon(1)
NN_model.compile(loss='mean_squared_error', optimizer='adam', metrics=['mse','mape'])
NN_model.summary()
return NN_model
def train_NN(NN_model, train_data, train_target, test_data, test_target):
history = NN_model.fit(train_data, train_target, epochs=150, shuffle=True, batch_size=50, verbose=1, use_multiprocessing=True)
return history
def test_NN(NN_model, test_data, test_target, train_data, train_target):
mean_test = NN_model.evaluate(test_data, test_target, verbose=1)
mean_train = NN_model.evaluate(train_data, train_target, verbose=1)
return mean_test, mean_train
try:
seed = 4
np.random.seed(seed)
test, target = get_data()
train_data, test_data, train_target, test_target = split_data(test, target)
print("Data split\n")
NN_model = create_NN()
print("Neural Network created\n")
history = train_NN(NN_model, train_data, train_target, test_data, test_target)
mean_test, mean_train = test_NN(NN_model, test_data, test_target, train_data, train_target)
print("Durchschnittliche Abweichung Training: ", mean_train)
print("Durchschnittliche Abweichung Test: ", mean_test)
print(NN_model.metrics_names)
NN_model.save('Regelung_v1.h5')
print("Neural Network saved")
except (KeyboardInterrupt, SystemExit):
raise
I get this result:
mse loss training data: 0.028168134637475015;
mse loss test data: 0.028960488473176955
The mean average percentage error is at about 9%. This result is not what i expected.
Where is my mistake?
Thank you for your help in advance
Have a nice day!
PC Specs:
Intel i5 4570
16GB RAM + 16 GB page file
Nvidia GTX 1070
3 TB HDD
Software:
Windows 10
Geforce Game ready driver 451.48
Tensorflow 2.2.0
Keras 2.3.1
Sklearn 0.23.1
Cuda 10.1
Python 3.7.7
Edit: Here are a few lines of the test.csv
TIMESTAMP;rHsubLS;b;lowerSetpoint
20200714091423000.00000000000;2.28878288783;-0.74361743617;-0.27947195702
20200714091423000.00000000000;0.13274132741;-0.94552945529;-0.32351276857
20200714091423000.00000000000;1.85753857539;0.77844778448;0.22244954249
20200714091423000.00000000000;1.31896318963;0.44518445184;0.33573301999
20200714091423000.00000000000;2.55885558856;-0.77792777928;-0.28837806344

The Output layer had its initialization weight missing:
NN_model.add(Dense(1, kernel_initializer='he_uniform', activation='linear'))

Prediction using Keras NLP

I am a beginner in the field of Neural Networks.
I am trying to implement an LSTM model for predicting the secondary structure of a protein from a given primary sequence. The base kernel on which my program is based can be found on Kaggle - https://www.kaggle.com/helmehelmuto/secondary-structure-prediction-with-keras
I successfully trained the model, saved the model to a pickle file, and I am able to use the pickle file to load weights to my model and make predictions as well. However, these predictions are on the test set which was created by the test_train_split function in Keras.
I aim to feed in a String containing the sequence of a protein and get its secondary structure prediction.
The code uses Tokenizer to convert the data (the protein sequences) from the dataset into a numpy.ndarray which is used for making predictions.
The part where I am stuck is, taking a string as input(some protein sequence) and converting it into the same class and then making predictions on it.
I have tried using the same method the Kaggle Kernel author used for converting the data from the .csv file to a numpy.ndarray but I get an error - 'numpy.ndarray' object has no attribute 'lower'.
I would be grateful if someone could guide me here, converting strings to the objects which are used for prediction in the code.
Code (works fine, predictions made directly on the test set generated by test_train_split):
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle
df = pd.read_csv(r'C:\Users\Viktor\Desktop\2018-06-06-ss.cleaned.csv')
df.len.hist(bins=100)
print(df.shape)
def seq2ngrams(seqs, n=3):
return np.array([[seq[i:i+n] for i in range(len(seq))] for seq in seqs])
maxlen_seq = 128
input_seqs, target_seqs = df[['seq', 'sst3']][(df.len <= maxlen_seq) & (~df.has_nonstd_aa)].values.T
input_grams = seq2ngrams(input_seqs)
print(len(input_seqs))
from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.utils import to_categorical
tokenizer_encoder = Tokenizer()
tokenizer_encoder.fit_on_texts(input_grams)
input_data = tokenizer_encoder.texts_to_sequences(input_grams)
input_data = sequence.pad_sequences(input_data, maxlen=maxlen_seq, padding='post')
tokenizer_decoder = Tokenizer(char_level=True)
tokenizer_decoder.fit_on_texts(target_seqs)
target_data = tokenizer_decoder.texts_to_sequences(target_seqs)
target_data = sequence.pad_sequences(target_data, maxlen=maxlen_seq, padding='post')
target_data = to_categorical(target_data)
input_data.shape, target_data.shape
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Bidirectional
n_words = len(tokenizer_encoder.word_index) + 1
n_tags = len(tokenizer_decoder.word_index) + 1
print(n_words, n_tags)
input = Input(shape=(maxlen_seq,))
x = Embedding(input_dim=n_words, output_dim=128, input_length=maxlen_seq)(input)
x = Bidirectional(LSTM(units=64, return_sequences=True, recurrent_dropout=0.1))(x)
y = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input, y)
model.summary()
from sklearn.model_selection import train_test_split
from keras.metrics import categorical_accuracy
from keras import backend as K
import tensorflow as tf
def q3_acc(y_true, y_pred):
y = tf.argmax(y_true, axis=-1)
y_ = tf.argmax(y_pred, axis=-1)
mask = tf.greater(y, 0)
return K.cast(K.equal(tf.boolean_mask(y, mask), tf.boolean_mask(y_, mask)), K.floatx())
model.compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy", q3_acc])
X_train, X_test, y_train, y_test = train_test_split(input_data, target_data, test_size=.4, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs, target_seqs, test_size=.4, random_state=0)
#model.fit(X_train, y_train, batch_size=128, epochs=5, validation_data=(X_test, y_test), verbose=1)
def onehot_to_seq(oh_seq, index):
s = ''
for o in oh_seq:
i = np.argmax(o)
if i != 0:
s += index[i]
else:
break
return s
def plot_results(x, y, y_):
print("---")
print("Input: " + str(x))
print("Target: " + str(onehot_to_seq(y, revsere_decoder_index).upper()))
print("Result: " + str(onehot_to_seq(y_, revsere_decoder_index).upper()))
fig = plt.figure(figsize=(10,2))
plt.imshow(y.T, cmap='Blues')
plt.imshow(y_.T, cmap='Reds', alpha=.5)
plt.yticks(range(4), [' '] + [revsere_decoder_index[i+1].upper() for i in range(3)])
plt.show()
revsere_decoder_index = {value:key for key,value in tokenizer_decoder.word_index.items()}
revsere_encoder_index = {value:key for key,value in tokenizer_encoder.word_index.items()}
#N=3
#y_train_pred = model.predict(X_train[:N])
#y_test_pred = model.predict(X_test[:N])
#print('training')
#for i in range(N):
# plot_results(seq_train[i], y_train[i], y_train_pred[i])
#print('testing')
#for i in range(N):
# plot_results(seq_test[i], y_test[i], y_test_pred[i])
loaded_model = pickle.load(open( "save.p", "rb" ))
N=3
y_train_pred = loaded_model.predict(X_train[:N])
y_test_pred = loaded_model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
#print(type(target_seqs))
CODE WHICH DOES NOT WORK AS EXPECTED:
In this, 'xf' is the csv file which has the sequences for which I require the predictions.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pickle
df = pd.read_csv(r'C:\Users\Viktor\Desktop\2018-06-06-ss.cleaned.csv')
xf = pd.read_csv(r'C:\Users\Viktor\Desktop\sequence.csv')
df.len.hist(bins=100)
print(df.shape)
def seq2ngrams(seqs, n=3):
return np.array([[seq[i:i+n] for i in range(len(seq))] for seq in seqs])
maxlen_seq = 128
input_seqs, target_seqs = df[['seq', 'sst3']][(df.len <= maxlen_seq) & (~df.has_nonstd_aa)].values.T
input_grams = seq2ngrams(input_seqs)
print(len(input_seqs))
from keras.preprocessing import text, sequence
from keras.preprocessing.text import Tokenizer
from keras.utils import to_categorical
tokenizer_encoder = Tokenizer()
tokenizer_encoder.fit_on_texts(input_grams)
input_data = tokenizer_encoder.texts_to_sequences(input_grams)
input_data = sequence.pad_sequences(input_data, maxlen=maxlen_seq, padding='post')
tokenizer_decoder = Tokenizer(char_level=True)
tokenizer_decoder.fit_on_texts(target_seqs)
target_data = tokenizer_decoder.texts_to_sequences(target_seqs)
target_data = sequence.pad_sequences(target_data, maxlen=maxlen_seq, padding='post')
target_data = to_categorical(target_data)
input_data.shape, target_data.shape
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Bidirectional
n_words = len(tokenizer_encoder.word_index) + 1
n_tags = len(tokenizer_decoder.word_index) + 1
print(n_words, n_tags)
input = Input(shape=(maxlen_seq,))
x = Embedding(input_dim=n_words, output_dim=128, input_length=maxlen_seq)(input)
x = Bidirectional(LSTM(units=64, return_sequences=True, recurrent_dropout=0.1))(x)
y = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input, y)
model.summary()
from sklearn.model_selection import train_test_split
from keras.metrics import categorical_accuracy
from keras import backend as K
import tensorflow as tf
model.compile(optimizer="rmsprop", loss="categorical_crossentropy")
X_train, X_test, y_train, y_test = train_test_split(input_data, target_data, test_size=.4, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs, target_seqs, test_size=.4, random_state=0)
#model.fit(X_train, y_train, batch_size=128, epochs=1, validation_data=(X_test, y_test), verbose=1)
def onehot_to_seq(oh_seq, index):
s = ''
for o in oh_seq:
i = np.argmax(o)
if i != 0:
s += index[i]
else:
break
return s
def plot_results(x, y, y_):
print("---")
print("Input: " + str(x))
print("Target: " + str(onehot_to_seq(y, revsere_decoder_index).upper()))
print("Result: " + str(onehot_to_seq(y_, revsere_decoder_index).upper()))
fig = plt.figure(figsize=(10,2))
plt.imshow(y.T, cmap='Blues')
plt.imshow(y_.T, cmap='Reds', alpha=.5)
plt.yticks(range(4), [' '] + [revsere_decoder_index[i+1].upper() for i in range(3)])
plt.show()
revsere_decoder_index = {value:key for key,value in tokenizer_decoder.word_index.items()}
revsere_encoder_index = {value:key for key,value in tokenizer_encoder.word_index.items()}
N=3
y_train_pred = model.predict(X_train[:N])
y_test_pred = model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
loaded_model = pickle.load(open( "save.p", "rb" ))
N=3
y_train_pred = loaded_model.predict(X_train[:N])
y_test_pred = loaded_model.predict(X_test[:N])
print('training')
for i in range(N):
plot_results(seq_train[i], y_train[i], y_train_pred[i])
print('testing')
for i in range(N):
plot_results(seq_test[i], y_test[i], y_test_pred[i])
print("-----")
print(X_test[:3])
print("-----")
xf.len.hist(bins=100)
input_seqs1, target_seqs1 = xf[['seq', 'sst3']][(xf.len <= maxlen_seq) & (~xf.has_nonstd_aa)].values.T
input_grams1 = seq2ngrams(input_seqs1)
tokenizer_encoder1 = Tokenizer()
tokenizer_encoder1.fit_on_texts(input_grams1)
input_data1 = tokenizer_encoder1.texts_to_sequences(input_grams1)
input_data1 = sequence.pad_sequences(input_data1, maxlen=maxlen_seq, padding='post')
tokenizer_decoder1 = Tokenizer(char_level=True)
tokenizer_decoder1.fit_on_texts(target_seqs1)
target_data1 = tokenizer_decoder1.texts_to_sequences(target_seqs1)
target_data1 = sequence.pad_sequences(target_data1, maxlen=maxlen_seq, padding='post')
target_data1 = to_categorical(target_data1)
input_data1.shape, target_data1.shape
X_train, X_test, y_train, y_test = train_test_split(input_data1, target_data1, test_size=1, random_state=0)
seq_train, seq_test, target_train, target_test = train_test_split(input_seqs1, target_seqs1, test_size=1, random_state=0)
y_train_pred1 = loaded_model.predict(X_train)
y_test_pred1 = loaded_model.predict(X_test)
plot_results(seq_train, y_train, y_train_pred)
plot_results(seq_test, y_test, y_test_pred)
#print(input_data1[0])
##y_train_pred = loaded_model.predict(input_data)
#y_test_pred1 = loaded_model.predict(input_data1[0])
##plot_results(seq_train, y_train, y_train_pred)
#plot_results(input_seqs1, target_data1, y_test_pred1)
TRACEBACK:
Traceback (most recent call last):
File "<ipython-input-38-e8f27dda0841>", line 1, in <module>
runfile('C:/Users/Viktor/Desktop/rost_nocl.py', wdir='C:/Users/Viktor/Desktop')
File "D:\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 827, in runfile
execfile(filename, namespace)
File "D:\Anaconda3\lib\site-packages\spyder_kernels\customize\spydercustomize.py", line 110, in execfile
exec(compile(f.read(), filename, 'exec'), namespace)
File "C:/Users/Viktor/Desktop/rost_nocl.py", line 116, in <module>
tokenizer_encoder1.fit_on_texts(input_grams1)
File "D:\Anaconda3\lib\site-packages\keras_preprocessing\text.py", line 223, in fit_on_texts
self.split)
File "D:\Anaconda3\lib\site-packages\keras_preprocessing\text.py", line 43, in text_to_word_sequence
text = text.lower()
AttributeError: 'numpy.ndarray' object has no attribute 'lower'

Can't use intermediate function in hyperas

I am trying to use hyperas to optimize my keras model but I keep getting NameError: processing (function_name) is not defined. I have already looked at this and this example from hyperas and done exactly that. It doesn't seem to work for me.
This is my code:
def processing():
df = pd.read_json('balanced_all.json')
def label (df):
if df['rating'] < 3:
return 0
if df['rating'] > 3:
return 1
df['label'] = df.apply (lambda df: label(df), axis=1)
df = df[['review_text', 'label']]
maxlen = 100
max_words = 2000
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(df['review_text'].values)
sequences = tokenizer.texts_to_sequences(df['review_text'].values)
word_index = tokenizer.word_index
sequences = pad_sequences(sequences, maxlen=maxlen)
labels = pd.get_dummies(df['label']).values
glove_dir = '/home/uttam/Documents/Thesis/Glove'
embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), 'r', encoding='utf-8')
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
if i < max_words:
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return sequences, labels, embedding_matrix
def data():
sequences = processing()[0]
labels = processing()[1]
x_train, x_test, y_train, y_test = train_test_split(sequences,labels, test_size = 0.33, random_state = 42)
return x_train, y_train, x_test, y_test
def create_model(x_train, y_train, x_test, y_test):
embedding_dim = 100
max_words = 2000
embedding_matrix = processing()[2]
model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=100))
model.add(LSTM(128))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(2, activation='sigmoid'))
model.layers[0].set_weights([embedding_matrix])
model.layers[0].trainable = False
model.compile(optimizer={{choice(['rmsprop', 'adam', 'sgd'])}}, loss='binary_crossentropy',metrics=['acc'])
result = model.fit(x_train, y_train, epochs=20, batch_size={{choice([64, 128])}}, validation_split=0.2)
model.save('pre_trained_glove_model.h5')
validation_acc = np.amax(result.history['val_acc'])
print('Best validation acc of epoch:', validation_acc)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
if __name__ == '__main__':
best_run, best_model = optim.minimize(model=create_model,
data=data,
algo=tpe.suggest,
max_evals=5,
trials=Trials())
x_train, y_train, x_test, y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(x_test, y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
I don't even need the intermediate function, I had to create it because hyperas didn't find the global variable. for e.g. if I had a variable x outside the hyperas function say create_model(), It would say NameError: x is not defined
I need this because as you can see I am using pre-trained glove embedding. I can't put everything in either data() or create_model(). For e.g. data()needs the variable sequences and label and create_model needs the variable embedding_matrix, so there is no way (as far as I know) to split everything in two functions.
Only way this worked for me was by putting everything in both data() and create_model() functions, which definitely is not efficient and not the way to do.

A little bit late but for future reference, you are right hyperas doesn't recognize global variables. You can pass the function in a list of functions in minimize:
best_run, best_model = optim.minimize(model=create_model,
data=data,
functions=[processing], # <<
algo=tpe.suggest,
max_evals=5,
trials=Trials())
As you mentioned if you need to pass a global variable in hyperas. You can choose one of these options:
Using data():
def data():
## ... my code ...
return x_train, y_train, x_test, y_test, foo
def create_model(x_train, y_train, x_test, y_test, foo):
or defining a new function and pass it in the list of functions:
def my_funct():
return foo
def data():
return x_train, y_train, x_test, y_test
def create_model(x_train, y_train, x_test, y_test):
foo = my_funct()
best_run, best_model = optim.minimize(model=create_model,
data=data,
functions=[my_funct], # << foo
algo=tpe.suggest,
max_evals=5,
trials=Trials())

Keras Index out of Bounds Error for csv database

this is my first time posting on StackOverflow after quite some time using the site.
I have been trying to predict the last column of a practice machine learning database from this link
http://archive.ics.uci.edu/ml/datasets/Diabetes+130-US+hospitals+for+years+1999-2008#
I ran the code below and received this error:
Traceback (most recent call last):
File "", line 1, in
runfile('/Users/ric4711/diabetes_tensorflow', wdir='/Users/ric4711')
File "/Users/ric4711/anaconda/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 880, in runfile
execfile(filename, namespace)
File "/Users/ric4711/anaconda/lib/python2.7/site-packages/spyder/utils/site/sitecustomize.py", line 94, in execfile
builtins.execfile(filename, *where)
File "/Users/ric4711/diabetes_tensorflow", line 60, in
y_train = to_categorical(y_train, num_classes = num_classes)
File "/Users/ric4711/anaconda/lib/python2.7/site-packages/keras/utils/np_utils.py", line 25, in to_categorical
categorical[np.arange(n), y] = 1
IndexError: index 3 is out of bounds for axis 1 with size 3
I suspect there might be an issue in my dimensions of the y axis or how I'm managing categories for this. Any help would be greatly appreciated.
from pandas import read_csv
import numpy
from sklearn.model_selection import train_test_split
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder
from keras.layers import Dense, Input
from keras.models import Model
dataset = read_csv(r"/Users/ric4711/Documents/dataset_diabetes/diabetic_data.csv", header=None)
#Column 2, 5, 10, 11, 18, 19, 20 all have "?"
#(101767, 50) size of dataset
#PROBLEM COLUMNS WITH NUMBER OF "?"
#2 2273
#5 98569
#10 40256
#11 49949
#18 21
#19 358
#20 1423
le=LabelEncoder()
dataset[[2,5,10,11,18,19,20]] = dataset[[2,5,10,11,18,19,20]].replace("?", numpy.NaN)
dataset = dataset.drop(dataset.columns[[0, 1, 5, 10, 11]], axis=1)
dataset.dropna(inplace=True)
y = dataset[[49]]
X = dataset.drop(dataset.columns[[44]], 1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
for col in X_test.columns.values:
if X_test[col].dtypes=='object':
data=X_train[col].append(X_test[col])
le.fit(data.values)
X_train[col]=le.transform(X_train[col])
X_test[col]=le.transform(X_test[col])
for col in y_test.columns.values:
if y_test[col].dtypes=='object':
data=y_train[col].append(y_test[col])
le.fit(data.values)
y_train[col]=le.transform(y_train[col])
y_test[col]=le.transform(y_test[col])
batch_size = 500
num_epochs = 300
hidden_size = 250
num_test = X_test.shape[0]
num_training = X_train.shape[0]
height, width, depth = 1, X_train.shape[1], 1
num_classes = 3
y_train = y_train.as_matrix()
y_test = y_test.as_matrix()
y_train = to_categorical(y_train, num_classes = num_classes)
y_test = to_categorical(y_test, num_classes = num_classes)
inp = Input(shape=(height * width,))
hidden_1 = Dense(hidden_size, activation='tanh')(inp)
hidden_2 = Dense(hidden_size, activation='tanh')(hidden_1)
hidden_3 = Dense(hidden_size, activation='tanh')(hidden_2)
hidden_4 = Dense(hidden_size, activation='tanh')(hidden_3)
hidden_5 = Dense(hidden_size, activation='tanh')(hidden_4)
hidden_6 = Dense(hidden_size, activation='tanh')(hidden_5)
hidden_7 = Dense(hidden_size, activation='tanh')(hidden_6)
hidden_8 = Dense(hidden_size, activation='tanh')(hidden_7)
hidden_9 = Dense(hidden_size, activation='tanh')(hidden_8)
hidden_10 = Dense(hidden_size, activation='tanh')(hidden_9)
hidden_11 = Dense(hidden_size, activation='tanh')(hidden_10)
out = Dense(num_classes, activation='softmax')(hidden_11)
model = Model(inputs=inp, outputs=out)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X_train, y_train, batch_size = batch_size,epochs = num_epochs, validation_split = 0.1, shuffle = True)
model.evaluate(X_test, y_test, verbose=1)

I fixed this by changing the num_classes to 4, and in the .fit method applying numpy.array(X_train), numpy.array(y_train)