I have the question on how to convert a Pytorch Neural Network to a Tensorflow Neural Network. The TF class as its written below does not work and I assume its die to the difference between nn.Sequential and tf.keras.Sequential.
class FullyConnected(nn.Sequential):
"""
Fully connected multi-layer network with ELU activations.
"""
def __init__(self, sizes, final_activation=None):
layers = []
for in_size, out_size in zip(sizes, sizes[1:]):
layers.append(nn.Linear(in_size, out_size))
layers.append(nn.ELU())
layers.pop(-1)
if final_activation is not None:
layers.append(final_activation)
super().__init__(*layers) here
class FullyConnected(tf.keras.Sequential):
"""
Fully connected multi-layer network with ELU activations.
"""
def __init__(self, sizes, final_activation=None):
layers = []
for out_size in sizes[1:-1]:
layers.append(Dense(units=out_size, activation='elu'))
if final_activation is not None:
layers.append(Dense(units=sizes[-1], activation='elu'))
else:
layers.append(Dense(units=sizes[-1]))
super().__init__(*layers)
If I try to initialize the network say by self.fc = FullyConnected(sizes=(sizes[:-1] + [self.dim * 2])) with sizes = [1, 128, 128, 128, 1] I get the Error: TypeError: object.__init__() takes exactly one argument (the instance to initialize) when using the TF network.
Can somebody help?
Many thanks in advance!!
I built a 4 layer keras network outputting a binary classification. tanh seemed to perform better.
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from sklearn.metrics import classification_report,confusion_matrix
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
batch_size=673
data = np.random.rand(batch_size, 25)
#for item in data:
# print(item)
label = np.random.randint(0,2,(batch_size,1))
#print(data)
df=pd.DataFrame(data)
df2=pd.DataFrame(label,columns=["Target"])
df=pd.concat([df,df2],axis=1)
#print(df)
columns=[x for x in df.columns if x!="Target"]
X=df[columns]
y=df["Target"]
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.3, random_state=42)
scaler = StandardScaler()
scaler.fit(X_train)
X_train=scaler.transform(X_train)
X_test=scaler.transform(X_test)
model= Sequential()
model.add(Dense(25, input_shape=(25,),activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
#model.compile(optimizer=Adam(0.01),loss='binary_crossentropy')
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.001), metrics=['accuracy'])
model.summary()
history=model.fit(X_train, y_train,epochs = 100,verbose=0)
model.evaluate(X_test, y_test)
plt.plot(history.history['accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
Related
For the sake of reproducibility, the training and validations datasets I am using are shared here
The validation_dataset.csv is the ground truth of training_dataset.csv.
What I am doing below is feeding the datasets into a simple CNN layer that extracts the useful features of the images and feed that as 1D into the LSTM network for classification.
from keras.models import Sequential
from keras.layers import Dense, Flatten, Activation
from keras.layers.convolutional import Conv1D
from keras.layers import LSTM
from keras.layers.convolutional import MaxPooling1D
from keras.layers import TimeDistributed
from keras.layers import Dropout
from keras import optimizers
from keras.callbacks import EarlyStopping
import pandas as pd
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score
from confusion_matrix import plot_confusion_matrix
import scikitplot as skplt
from numpy import genfromtxt
train_set = genfromtxt('data/train/training_dataset.csv', delimiter=',')
validation_set = genfromtxt('data/validation/validation_dataset.csv', delimiter=',')
train_set = train_set[..., None]
validation_set = validation_set[..., None]
X_train, X_test, y_train, y_test = train_test_split(train_set, validation_set, test_size=0.30, random_state=0)
batch_size=16
epochs=5
# Create the model
model = Sequential()
model.add(Conv1D(filters=5, kernel_size=3, activation='relu', padding='same'))
model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(50, return_sequences=True))
model.add(Dropout(0.5))
model.add(LSTM(10))
model.add(Dense(1,kernel_initializer='random_normal'))
model.add(Activation('relu'))
adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0)
sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=adam, loss='mean_squared_error', metrics=['mae', 'mape', 'mean_squared_error', 'acc'])
model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs)
print(model.summary())
# Evaluate the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
skplt.metrics.plot_confusion_matrix(y_test, scores, x_tick_rotation=50, title=' ', normalize=True)
Finally, I want to plot the confusion matrix of the model using
skplt.metrics.plot_confusion_matrix(y_test, scores, x_tick_rotation=50, title=' ', normalize=True)
However, it is raising an error ValueError: Found input variables with inconsistent numbers of samples: [5394, 5].
How can we fix this error?
The second argument to skplt.metrics.plot_confusion_matrix must be the predicted labels (see https://scikit-plot.readthedocs.io/en/stable/metrics.html). But, you pass scores, which does not contain the predicted labels.
The fix would be to do:
y_pred = model.predict(X_test)
skplt.metrics.plot_confusion_matrix(y_test,
y_pred,
x_tick_rotation=50,
title=' ',
normalize=True)
I was working on SVM few days ago and when i tried to plot confusion matrix the following lines of code worked for me.
predicted=model.predict(X_test) #predicted output
cm=metrics.confusion_matrix(y_test, predicted)
df_cm = pd.DataFrame(cm, range(2), range(2))
sns.set(font_scale=1.4)
sns.heatmap(df_cm, annot=True, annot_kws={"size": 16})
plt.title('CONFUSION MATRIX ',fontdict={'fontsize': 14, 'fontweight': 'bold'})
plt.show()
I have to perform a simple FSGM attack to a convolutional neural network. The code for the CNN works correctly, and the model is saved without a problem, but when i try to perform the attack an error is shown.
HERE'S THE CODE FOR THE CNN
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, MaxPooling2D
import matplotlib.pyplot as plt
from keras.datasets import mnist
from keras.utils import to_categorical
import json
import tensorflow as tf
#Using TensorFlow backend.
#download mnist data and split into train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
#plot the first image in the dataset
plt.imshow(X_train[0])
#check image shape
X_train[0].shape
#reshape data to fit model
X_train = X_train.reshape(60000,28,28,1)
X_test = X_test.reshape(10000,28,28,1)
#one-hot encode target column
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
y_train[0]
#create model
model = Sequential()
#add model layers
model.add(Conv2D(32, kernel_size=(5,5), activation='relu', input_shape= (28,28,1)))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Conv2D(64, kernel_size=(5,5), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(10, activation='softmax'))
#compile model using accuracy as a measure of model performance
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics= ['accuracy'])
#train model
model.fit(X_train, y_train,validation_data=(X_test, y_test), epochs=5)
json.dump({'model':model.to_json()},open("model.json", "w"))
model.save_weights("model_weights.h5")
THEN I TRY TO PERFORM THE ATTACK WITH THE FOLLOWING CODE:
import json
import foolbox
import keras
import numpy as np
from keras import backend
from keras.models import load_model
from keras.datasets import mnist
from keras.utils import np_utils
from foolbox.attacks import FGSM
from foolbox.criteria import Misclassification
from foolbox.distances import MeanSquaredDistance
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
import numpy as np
import tensorflow as tf
from keras.models import model_from_json
import os
############## Loading the model and preprocessing #####################
backend.set_learning_phase(False)
model = tf.keras.models.model_from_json(json.load(open("model.json"))["model"],custom_objects={})
model.load_weights("model_weights.h5")
fmodel = foolbox.models.KerasModel(model, bounds=(0,1))
_,(images, labels) = mnist.load_data()
images = images.reshape(10000,28,28)
images= images.astype('float32')
images /= 255
######################### Attacking the model ##########################
attack=foolbox.attacks.FGSM(fmodel, criterion=Misclassification())
adversarial=attack(images[12],labels[12]) # for single image
adversarial_all=attack(images,labels) # for all the images
adversarial =adversarial.reshape(1,28,28,1) #reshaping it for model prediction
model_predictions = model.predict(adversarial)
print(model_predictions)
########################## Visualization ################################
images=images.reshape(10000,28,28)
adversarial =adversarial.reshape(28,28)
plt.figure()
plt.subplot(1,3,1)
plt.title('Original')
plt.imshow(images[12])
plt.axis('off')
plt.subplot(1, 3, 2)
plt.title('Adversarial')
plt.imshow(adversarial)
plt.axis('off')
plt.subplot(1, 3, 3)
plt.title('Difference')
difference = adversarial - images[124]
plt.imshow(difference / abs(difference).max() * 0.2 + 0.5)
plt.axis('off')
plt.show()
this error is shown when the adversarial examples are generated:
c_api.TF_GetCode(self.status.status))
InvalidArgumentError: Matrix size-incompatible: In[0]: [1,639232], In[1]: [1024,10]
[[{{node dense_4_5/MatMul}}]]
[[{{node dense_4_5/BiasAdd}}]]
What could it be?
here is my solution.
First of all modify the model code as follows
import tensorflow as tf
import json
# download mnist data and split into train and test sets
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
# reshape data to fit model
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
X_train, X_test = X_train/255, X_test/255
# one-hot encode target column
y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)
# create model
model = tf.keras.models.Sequential()
# add model layers
model.add(tf.keras.layers.Conv2D(32, kernel_size=(5, 5),
activation='relu', input_shape=(28, 28, 1)))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Conv2D(64, kernel_size=(5, 5), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(10, activation='softmax'))
# compile model using accuracy as a measure of model performance
model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
# train model
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=5)
json.dump({'model': model.to_json()}, open("model.json", "w"))
model.save_weights("model_weights.h5")
You just forgot to divide each pixel by the maximum value of RGB (255)
As for the attacker code
import json
import foolbox
from foolbox.attacks import FGSM
from foolbox.criteria import Misclassification
import numpy as np
import tensorflow as tf
############## Loading the model and preprocessing #####################
tf.enable_eager_execution()
tf.keras.backend.set_learning_phase(False)
model = tf.keras.models.model_from_json(
json.load(open("model.json"))["model"], custom_objects={})
model.load_weights("model_weights.h5")
model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
_, (images, labels) = tf.keras.datasets.mnist.load_data()
images = images.reshape(images.shape[0], 28, 28, 1)
images = images/255
images = images.astype(np.float32)
fmodel = foolbox.models.TensorFlowEagerModel(model, bounds=(0, 1))
######################### Attacking the model ##########################
attack = foolbox.attacks.FGSM(fmodel, criterion=Misclassification())
adversarial = np.array([attack(images[0], label=labels[0])])
model_predictions = model.predict(adversarial)
print('real label: {}, label prediction; {}'.format(
labels[0], np.argmax(model_predictions)))
I used TensorFlowEagerModel instead of KerasModel for simplicty. The error you were encountering was due to the fact that model.predict expects a 4d matrix while you were passing a 3d matrix, so I just wrapped up the attack to the image example into a numpy array to make it 4d.
Hope it helps
As an example of network, I used the first example here
I would like to use tensorboard with this network. After read this documentation about how to use TensorBoard, I added these commands to the code:
from keras.callbacks import TensorBoard
TensorBoard("Directory path that contains the log files")
The output sounds correct:
Out[3]: <keras.callbacks.TensorBoard at 0x7f14730e79b0>
But there is nothing in the directory...
What I did wrong ?
Here is the complete code:
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.optimizers import SGD
from keras.callbacks import TensorBoard
# Generate dummy data
import numpy as np
x_train = np.random.random((1000, 20))
y_train = keras.utils.to_categorical(np.random.randint(10, size=(1000, 1)), num_classes=10)
x_test = np.random.random((100, 20))
y_test = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10)
model = Sequential()
# Dense(64) is a fully-connected layer with 64 hidden units.
# in the first layer, you must specify the expected input data shape:
# here, 20-dimensional vectors.
model.add(Dense(64, activation='relu', input_dim=20))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
model.fit(x_train, y_train,
epochs=20,
batch_size=128)
score = model.evaluate(x_test, y_test, batch_size=128)
TensorBoard("Directory path that contains the log files")
You need to pass the callback to model.fit:
tb = TensorBoard('log_dir')
model.fit(x_train, y_train,
epochs=20,
batch_size=128,
callbacks=[tb])
This is a regression problem. Below is my code
import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.cross_validation import cross_val_score, KFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
os.chdir(r'C:\Users\Swapnil\Desktop\RP TD\first\Changes')
## Load the dataset
dataset1 = pd.read_csv("Main Lane Plaza 1.csv")
X_train = dataset1.iloc[:,0:11].values
Y_train = dataset1.iloc[:,11].values
dataset2 = pd.read_csv("Main Lane Plaza 1_070416010117.csv")
X_test = dataset2.iloc[:,0:11].values
Y_test = dataset2.iloc[:,11].values
##Define base model
def base_model():
model = Sequential()
model.add(Dense(11, input_dim=11, kernel_initializer='normal',
activation='sigmoid'))
model.add(Dense(7, kernel_initializer='normal', activation='sigmoid'))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_squared_error', optimizer = 'adam')
return model
seed = 7
np.random.seed(seed)
clf = KerasRegressor(build_fn=base_model, nb_epoch=100,
batch_size=5,verbose=0)
clf.fit(X_train, Y_train)
res = clf.predict(X_train)
##Result
clf.score(X_test, Y_test)
Not sure if the score should be negative??
Kindly advise if i am doing something wrong.
Thanks in advance.
I am not able to figure it out can this be problem due to feature scaling as I did feature scaling using R and saved the csv files to use in python.
When you get a negative score for regression problem, it usually means that your the model you choose can't fit your data well.
You have layer 1 activation as sigmoid, layer 2 also as sigmoid and then final layer as 1 output.
change the activations to relu, as sigmoid would be squashing the values between 0 to 1. Making the numbers really small, causing the vanishing gradient problem over the 2 hidden layer.
def base_model():
model = Sequential()
model.add(Dense(11, input_dim=11, kernel_initializer='normal', activation='relu'))
model.add(Dense(7, kernel_initializer='normal', activation='relu'))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mean_squared_error', optimizer='adam')
return model
I want to build a non linear regression model using keras to predict a +ve continuous variable.
For the below model how do I select the following hyperparameters?
Number of Hidden layers and Neurons
Dropout ratio
Use BatchNormalization or not
Activation function out of linear, relu, tanh, sigmoid
Best optimizer to use among adam, rmsprog, sgd
Code
def dnn_reg():
model = Sequential()
#layer 1
model.add(Dense(40, input_dim=13, kernel_initializer='normal'))
model.add(Activation('tanh'))
model.add(Dropout(0.2))
#layer 2
model.add(Dense(30, kernel_initializer='normal'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.4))
#layer 3
model.add(Dense(5, kernel_initializer='normal'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(1, kernel_initializer='normal'))
model.add(Activation('relu'))
# Compile model
model.compile(loss='mean_squared_error', optimizer='adam')
return model
I have considered random gridsearch but instead want to use hyperopt which I believe will be faster. I initially implemented the tuning using https://github.com/maxpumperla/hyperas. Hyperas is not working with latest version of keras. I suspect that keras is evolving fast and it's difficult for the maintainer to make it compatible. So I think using hyperopt directly will be a better option.
PS: I am new to bayesian optimization for hyper parameter tuning and hyperopt.
I've had a lot of success with Hyperas. The following are the things I've learned to make it work.
1) Run it as a python script from the terminal (not from an Ipython notebook)
2) Make sure that you do not have any comments in your code (Hyperas doesn't like comments!)
3) Encapsulate your data and model in a function as described in the hyperas readme.
Below is an example of a Hyperas script that worked for me (following the instructions above).
from __future__ import print_function
from hyperopt import Trials, STATUS_OK, tpe
from keras.datasets import mnist
from keras.layers.core import Dense, Dropout, Activation
from keras.models import Sequential
from keras.utils import np_utils
import numpy as np
from hyperas import optim
from keras.models import model_from_json
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD , Adam
import tensorflow as tf
from hyperas.distributions import choice, uniform, conditional
__author__ = 'JOnathan Hilgart'
def data():
"""
Data providing function:
This function is separated from model() so that hyperopt
won't reload data for each evaluation run.
"""
import numpy as np
x = np.load('training_x.npy')
y = np.load('training_y.npy')
x_train = x[:15000,:]
y_train = y[:15000,:]
x_test = x[15000:,:]
y_test = y[15000:,:]
return x_train, y_train, x_test, y_test
def model(x_train, y_train, x_test, y_test):
"""
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
"""
model_mlp = Sequential()
model_mlp.add(Dense({{choice([32, 64,126, 256, 512, 1024])}},
activation='relu', input_shape= (2,)))
model_mlp.add(Dropout({{uniform(0, .5)}}))
model_mlp.add(Dense({{choice([32, 64, 126, 256, 512, 1024])}}))
model_mlp.add(Activation({{choice(['relu', 'sigmoid'])}}))
model_mlp.add(Dropout({{uniform(0, .5)}}))
model_mlp.add(Dense({{choice([32, 64, 126, 256, 512, 1024])}}))
model_mlp.add(Activation({{choice(['relu', 'sigmoid'])}}))
model_mlp.add(Dropout({{uniform(0, .5)}}))
model_mlp.add(Dense({{choice([32, 64, 126, 256, 512, 1024])}}))
model_mlp.add(Activation({{choice(['relu', 'sigmoid'])}}))
model_mlp.add(Dropout({{uniform(0, .5)}}))
model_mlp.add(Dense(9))
model_mlp.add(Activation({{choice(['softmax','linear'])}}))
model_mlp.compile(loss={{choice(['categorical_crossentropy','mse'])}}, metrics=['accuracy'],
optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})
model_mlp.fit(x_train, y_train,
batch_size={{choice([16, 32, 64, 128])}},
epochs=50,
verbose=2,
validation_data=(x_test, y_test))
score, acc = model_mlp.evaluate(x_test, y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model_mlp}
enter code here
if __name__ == '__main__':
import gc; gc.collect()
with K.get_session(): ## TF session
best_run, best_model = optim.minimize(model=model,
data=data,
algo=tpe.suggest,
max_evals=2,
trials=Trials())
X_train, Y_train, X_test, Y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
it induced by different gc sequence, if python collect session first , the program will exit successfully, if python collect swig memory(tf_session) first, the program exit with failure.
you can force python to del session by:
del session
or if you are using keras, you cant get the session instance, you can run following code at end of your code:
import gc; gc.collect()
This can be also another approach:
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
from sklearn.metrics import roc_auc_score
import sys
X = []
y = []
X_val = []
y_val = []
space = {'choice': hp.choice('num_layers',
[ {'layers':'two', },
{'layers':'three',
'units3': hp.uniform('units3', 64,1024),
'dropout3': hp.uniform('dropout3', .25,.75)}
]),
'units1': hp.uniform('units1', 64,1024),
'units2': hp.uniform('units2', 64,1024),
'dropout1': hp.uniform('dropout1', .25,.75),
'dropout2': hp.uniform('dropout2', .25,.75),
'batch_size' : hp.uniform('batch_size', 28,128),
'nb_epochs' : 100,
'optimizer': hp.choice('optimizer',['adadelta','adam','rmsprop']),
'activation': 'relu'
}
def f_nn(params):
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import Adadelta, Adam, rmsprop
print ('Params testing: ', params)
model = Sequential()
model.add(Dense(output_dim=params['units1'], input_dim = X.shape[1]))
model.add(Activation(params['activation']))
model.add(Dropout(params['dropout1']))
model.add(Dense(output_dim=params['units2'], init = "glorot_uniform"))
model.add(Activation(params['activation']))
model.add(Dropout(params['dropout2']))
if params['choice']['layers']== 'three':
model.add(Dense(output_dim=params['choice']['units3'], init = "glorot_uniform"))
model.add(Activation(params['activation']))
model.add(Dropout(params['choice']['dropout3']))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=params['optimizer'])
model.fit(X, y, nb_epoch=params['nb_epochs'], batch_size=params['batch_size'], verbose = 0)
pred_auc =model.predict_proba(X_val, batch_size = 128, verbose = 0)
acc = roc_auc_score(y_val, pred_auc)
print('AUC:', acc)
sys.stdout.flush()
return {'loss': -acc, 'status': STATUS_OK}
trials = Trials()
best = fmin(f_nn, space, algo=tpe.suggest, max_evals=50, trials=trials)
print('best: ', best)
Source