I want to check my loss values using MSE during the training process, how to fetching the loss values using MSE at each of iteration?., thank you.
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import mean_absolute_error
dataset = open_dataset("forex.csv")
dataset_vector = [float(i[-1]) for i in dataset]
normalized_dataset_vector = normalize_vector(dataset_vector)
training_vector, validation_vector, testing_vector = split_dataset(training_size, validation_size, testing_size, normalized_dataset_vector)
training_features = get_features(training_vector)
training_fact = get_fact(training_vector)
validation_features = get_features(validation_vector)
validation_fact = get_fact(validation_vector)
model = MLPRegressor(activation=activation, alpha=alpha, hidden_layer_sizes=(neural_net_structure[1],), max_iter=number_of_iteration, random_state=seed)
model.fit(training_features, training_fact)
pred = model.predict(training_features)
err = mean_absolute_error(pred, validation_fact)
print(err)
There's no callbacks object like there is in Keras so you'll have to loop over the fitting process to get it for each iteration. Something like the below will work for you
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import mean_absolute_error
# create some toy data
X = np.random.random((100, 5))
y = np.random.choice([0, 1], 100)
max_iter = 500
mlp = MLPClassifier(hidden_layer_sizes=(10, 10, 10), max_iter=max_iter)
errors = []
for i in range(max_iter):
mlp.partial_fit(X, y, classes=[0, 1])
pred = mlp.predict(X)
errors.append(mean_absolute_error(y, pred))
Which throws an annoying DeprecationWarning at the moment, but that can be ignored. The only problem with using this method is that you have to manually keep track of whether or not your model has converged. Personally I would suggest using Keras instead of sklearn if you want to work with neural networks.
Related
I am trying to use the GaussianProcessRegressor in scikit-learn with some graph kernels computed by the grakel software. Below is my code for a 5-fold cross-validation on 100 graph data. For the sake of testing convenience, I have commented out all graph-related lines and use random kernel matrices and y values instead.
from sklearn.model_selection import KFold
from sklearn.utils import check_random_state
from sklearn.gaussian_process import GaussianProcessRegressor as GPR
from sklearn.metrics import mean_squared_error
#from grakel.kernels import WeisfeilerLehman
import numpy as np
def Kfold_CV_GPR(Gs, y, n_iter=4, n_splits=5, random_state=None):
random_state = check_random_state(random_state)
kf = KFold(n_splits=n_splits, random_state=random_state, shuffle=True)
errors = []
for train_idxs, test_idxs in kf.split(y):
# gk = WeisfeilerLehman(n_iter=n_iter, normalize=True)
# K_train = gk.fit_transform(Gs[train_idxs])
# K_test = gk.transform(Gs[test_idxs])
K_train = np.random.randn(80, 80)
K_test = np.random.randn(20, 80)
gpr = GPR(kernel='precomputed')
gpr.fit(K_train, y[train_idxs])
y_pred = gpr.predict(K_test)
rmse = mean_squared_error(y[test_idxs], y_pred, squared=False)
errors.append(rmse)
return -np.mean(errors)
score = Kfold_CV_GPR(Gs=None, y=np.random.randn(100, ), n_iter=4, n_splits=5)
print(score)
However, I am getting the following error
TypeError: Cannot clone object ''precomputed'' (type <class 'str'>): it does not seem to be a scikit-learn
estimator as it does not implement a 'get_params' method.
When I change sklearn.gaussian_process.GaussianProcessRegressor to sklearn.svm.SVR (support vector regression), my code doesn't throw any error, but it will run forever for some reason. I also tested classifers like sklearn.svm.SVC and my code is working fine.
Anyone know how to use precomputed kernel in a scikit-learn's GaussianProcessRegressor?
I am using a Semi-Supervised approach for Support Vector Machine in Python for the image classification from PASCAL VOC 2007 data.
I have tried with the default parameters from the libraries and also tuned them but it get extremely bad accuracy of about only ~ 2%.
Below is my code:
import pandas as pd
import numpy as np
from sklearn import decomposition
from sklearn.model_selection import train_test_split
from numpy import concatenate
import numpy as np
from sklearn import datasets
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn import decomposition
import warnings
warnings.filterwarnings("ignore")
color_layout_features = pd.read_pickle("color_layout_descriptor.pkl")
bow_surf = pd.read_pickle("bow_surf.pkl")
color_hist_features = pd.read_pickle("hist.pkl")
labels = pd.read_pickle("labels.pkl")
# Feat. Scaling
def scale(X, x_min, x_max):
nom = (X-X.min(axis=0))*(x_max-x_min)
denom = X.max(axis=0) - X.min(axis=0)
denom[denom==0] = 1
return x_min + nom/denom
# normalization
def normalize(x):
return (x - np.min(x))/(np.max(x) - np.min(x))
color_layout_features_scaled = scale(color_layout_features, 0, 1)
color_hist_features_scaled = scale(color_hist_features, 0, 1)
bow_surf_scaled = scale(bow_surf, 0, 1)
features = np.hstack([color_layout_features_scaled, color_hist_features_scaled, bow_surf_scaled])
# define dataset
X, Y = features, labels
X = normalize(X)
pca = decomposition.PCA(n_components=100)
pca.fit(X)
X = pca.transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.30, random_state=1, stratify=Y)
# split train into labeled and unlabeled
X_train_lab, X_test_unlab, y_train_lab, y_test_unlab = train_test_split(X_train, y_train, test_size=0.30, random_state=1, stratify=y_train)
# create the training dataset input
X_train_mixed = concatenate((X_train_lab, X_test_unlab))
# create "no label" for unlabeled data
nolabel = [-1 for _ in range(len(y_test_unlab))]
# recombine training dataset labels
y_train_mixed = concatenate((y_train_lab, nolabel))
from semisupervised import S3VM
model = S3VM(kernel="Linear", C = 1e-2, gamma = 0.5, lamU = 1.0, probability=True)
#model.fit(X_train_mixed, _train_mixed)
model.fit(np.vstack((X_train_lab, X_test_unlab)), np.append(y_train_lab, nolabel))
#model.fit(np.vstack((label_X_train, unlabel_X_train)), np.append(label_y_train, unlabel_y))
# predict
predict = model.predict(X_test)
acc = metrics.accuracy_score(y_test, predict)
# metric
print("accuracy", acc*100)
accuracy 2.6692291266282298
I am using a Transductive version of SVM (TSVM) from the semisupervised library. But not sure what am I doing wrong so that even after tweaking the parameters I still get the same result. Any inputs would be helpful.
I refer https://github.com/rosefun/SemiSupervised/blob/master/semisupervised/TSVM.py to make the implementation. Any inputs would be helpful.
Please consider that according to link Documentation "The unlabeled samples should be labeled as -1".
I'm trying to implement batch training using sklearn's MLPClassifier leveraging partial_fit() function, but I get the following error:
AttributeError: 'MLPClassifier' object has no attribute '_label_binarizer'.
I have consulted some issues related to this (partial_fit Sklearn's MLPClassifier). This is the piece of code that I have used to reproduce the error (from the attached reference):
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:500,:]
train_target = output[0:500]
test_input= input[500:N,:]
test_target = output[500:N]
#Creating and training the Neural Net
# 1. Disable verbose (verbose is annoying with partial_fit)
clf = MLPClassifier(activation='tanh', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= False,
max_iter=1, warm_start=False)
# 2. Set what the classes are
clf.classes_ = [0,1]
for j in range(0,100):
for i in range(0,train_input.shape[0]):
input_inst = train_input[[i]]
target_inst= train_target[[i]]
clf=clf.partial_fit(input_inst,target_inst)
# 3. Monitor progress
print("Score on training set: %0.8f" % clf.score(train_input, train_target))
#Testing the Neural Net
y_pred = clf.predict(test_input)
print(y_pred)
# 4. Compute score on testing set
print(clf.score(test_input, test_target))
I have also modified multilayer_perceptron.py code at line 895 to replace this, as mentioned here:
self.label_binarizer_.fit(y)
With this:
if not incremental:
self.label_binarizer_.fit(y)
else:
self.label_binarizer_.fit(self.classes_)
And still doesn't work. Any help is really appreciated.
Thanks!
This would work:
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:500,:]
train_target = output[0:500]
test_input= input[500:N,:]
test_target = output[500:N]
#Creating and training the Neural Net
# 1. Disable verbose (verbose is annoying with partial_fit)
clf = MLPClassifier(activation='tanh', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= False,
max_iter=1, warm_start=False)
for j in range(0,100):
for i in range(0,train_input.shape[0]):
input_inst = train_input[[i]]
target_inst= train_target[[i]]
clf.partial_fit(input_inst,target_inst,[0,1])
# 3. Monitor progress
print("Score on training set: %0.8f" % clf.score(train_input, train_target))
#Testing the Neural Net
y_pred = clf.predict(test_input)
print(y_pred)
# 4. Compute score on testing set
print(clf.score(test_input, test_target))
This line was causing error:
# 2. Set what the classes are
clf.classes_ = [0,1]
And you have to pass classes here:
clf.partial_fit(input_inst,target_inst,[0,1])
I'm used to using sklearn, whose documentation I find very easy to follow. However, I now need to learn to use OpenCV - in particular, I need to be able to use an MLP classifier, and to update its weights as new training data comes in.
In sklearn, this can be done using the partial_fit method. According to the OpenCV documentation, there is an UPDATE_WEIGHTS flag that can be set, but I can't figure out how to include it in my code.
Here's a MCVE of what I have so far:
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
import numpy as np
import cv2
from sklearn.neural_network import MLPClassifier
def softmax(x):
softmaxes = np.zeros(x.shape)
for i in range(x.shape[1]):
softmaxes[:, i] = np.exp(x)[:, i]/np.sum(np.exp(x), axis=1)
return softmaxes
data = load_breast_cancer()
X = data.data
y = data.target.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1729)
y2 = np.zeros((y_train.shape[0], 2))
y2[:,0] = np.where(y_train==0, 1, 0)
y2[:,1] = np.where(y_train==1, 1, 0)
ann = cv2.ml.ANN_MLP_create()
ann.setLayerSizes(np.array([X.shape[1], y2.shape[1]]))
ann.setActivationFunction(cv2.ml.ANN_MLP_SIGMOID_SYM)
ann.train(np.float32(X_train), cv2.ml.ROW_SAMPLE, np.float32(y2))
mlp = MLPClassifier()
mlp.fit(X_train, y_train)
preds_proba = softmax(ann.predict(np.float32(X_test))[1])
print(roc_auc_score(y_test, preds_proba[:,1]))
print(roc_auc_score(y_test, mlp.predict_proba(X_test)[:,1]))
As the score between the OpenCV classifier and the sklearn learn one are comparable, I'm pretty confident it's implemented correctly.
How can I modify this code, so that when a new training sample comes in, I can update the weights based on that sample alone, rather than retraining on the entire train set?
The equivalent in sklearn would be:
mlp.partial_fit(X_new_sample, y_new_sample).
Figured out the answer. The syntax is the following:
ann.train(cv2.ml.TrainData_create(np.float32(X), 0, np.float32(y)), flags=1)
I've been trying to use Sklearn's neural network MLPClassifier. I have a dataset that is of size 1000 instances (with binary outputs) and I want to apply a basic Neural Net with 1 hidden layer to it.
The issue is that my data instances are not available all at the same time. At any point in time, I only have access to 1 data instance. I thought that partial_fit method of MLPClassifier can be used for this so I simulated the problem with an imaginary dataset of 1000 inputs and looped over the inputs one at a time and partial_fit to each instance but when I run the code, the neural net learns nothing and the predicted output is all zeros.
I am clueless as to what might be causing the problem. Any thought is hugely appreciated.
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:N/2,:]
train_target = output[0:N/2]
test_input= input[N/2:N,:]
test_target = output[N/2:N]
#Creating and training the Neural Net
clf = MLPClassifier(activation='tanh', algorithm='sgd', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= True,
max_iter=1, warm_start=True)
classes=[0,1]
for j in xrange(0,100):
for i in xrange(0,train_input.shape[0]):
input_inst = [train_input[i,:]]
input_inst = np.asarray(input_inst)
target_inst= [train_target[i]]
target_inst = np.asarray(target_inst)
clf=clf.partial_fit(input_inst,target_inst,classes)
#Testing the Neural Net
y_pred = clf.predict(test_input)
print y_pred
Explanation of the problem
The problem is with self.label_binarizer_.fit(y) in line 895 in multilayer_perceptron.py.
Whenever you call clf.partial_fit(input_inst,target_inst,classes), you call self.label_binarizer_.fit(y) where y has only one sample corresponding to one class, in this case. Therefore, if the last sample is of class 0, then your clf will classify everything as class 0.
Solution
As a temporary fix, you can edit multilayer_perceptron.py at line 895.
It is found in a directory similar to this python2.7/site-packages/sklearn/neural_network/
At line 895, change,
self.label_binarizer_.fit(y)
to
if not incremental:
self.label_binarizer_.fit(y)
else:
self.label_binarizer_.fit(self.classes_)
That way, if you are using partial_fit, then self.label_binarizer_ fits on the classes rather than on the individual sample.
Further, the code you posted can be changed to the following to make it work,
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:N/2,:]
train_target = output[0:N/2]
test_input= input[N/2:N,:]
test_target = output[N/2:N]
#Creating and training the Neural Net
# 1. Disable verbose (verbose is annoying with partial_fit)
clf = MLPClassifier(activation='tanh', algorithm='sgd', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= False,
max_iter=1, warm_start=True)
# 2. Set what the classes are
clf.classes_ = [0,1]
for j in xrange(0,100):
for i in xrange(0,train_input.shape[0]):
input_inst = train_input[[i]]
target_inst= train_target[[i]]
clf=clf.partial_fit(input_inst,target_inst)
# 3. Monitor progress
print "Score on training set: %0.8f" % clf.score(train_input, train_target)
#Testing the Neural Net
y_pred = clf.predict(test_input)
print y_pred
# 4. Compute score on testing set
print clf.score(test_input, test_target)
There are 4 main changes in the code. This should give you a good prediction on both the training and the testing set!
Cheers.