Develop Reference

How to solve Feature Shape Mismatch error in XGBoost Feature Selection?

I'm trying to use the following code to get feature importance for feature selection in XGBOOST. But I keep getting an error saying that there's a "Feature shape mismatch, expected: 91, got 78".
"select_X_train" has 78 features (columns) as does "select_X_test" so I think the issue is that the thresholds array must still contain the 91 original variables? But, I can't figure out a way to fix it. Can you please advise?
Here is the code:
from numpy import loadtxt
from numpy import sort
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectFromModel
from sklearn.metrics import classification_report
# load data
dataset = df_train
# split data into X and y
X_train = df[df.columns.difference(['Date','IsDeceased','IsTotal','Deceased','Sick','Injured','Displaced','Homeless','MissingPeople','Other','Total'])]
y_train = df['IsDeceased'].values
X_test = df_test[df_test.columns.difference(['Date','IsDeceased','IsTotal','Deceased','Sick','Injured','Displaced','Homeless','MissingPeople','Other','Total'])]
y_test = df_test['IsDeceased'].values
# fit model on all training data
model = XGBClassifier()
model.fit(X_train, y_train)
# make predictions for test data and evaluate
print("Accuracy: %.2f%%" % (accuracy * 100.0))
# Fit model using each importance as a threshold
thresholds = sort(model.feature_importances_)
for thresh in thresholds:
# select features using threshold
selection = SelectFromModel(model, threshold=thresh, prefit=True)
select_X_train = selection.transform(X_train)
# train model
selection_model = XGBClassifier()
selection_model.fit(select_X_train, y_train)
print(thresh)
# eval model
select_X_test = selection.transform(X_test)
y_pred = model.predict(select_X_test)
report = classification_report(y_test,y_pred)
print("Thresh= {} , n= {}\n {}" .format(thresh,select_X_train.shape[1], report))
cm = confusion_matrix(y_test, y_pred)
print(cm)```

Use GaussianProcessRegressor with precomputed kernel in scikit-learn

I am trying to use the GaussianProcessRegressor in scikit-learn with some graph kernels computed by the grakel software. Below is my code for a 5-fold cross-validation on 100 graph data. For the sake of testing convenience, I have commented out all graph-related lines and use random kernel matrices and y values instead.
from sklearn.model_selection import KFold
from sklearn.utils import check_random_state
from sklearn.gaussian_process import GaussianProcessRegressor as GPR
from sklearn.metrics import mean_squared_error
#from grakel.kernels import WeisfeilerLehman
import numpy as np
def Kfold_CV_GPR(Gs, y, n_iter=4, n_splits=5, random_state=None):
random_state = check_random_state(random_state)
kf = KFold(n_splits=n_splits, random_state=random_state, shuffle=True)
errors = []
for train_idxs, test_idxs in kf.split(y):
# gk = WeisfeilerLehman(n_iter=n_iter, normalize=True)
# K_train = gk.fit_transform(Gs[train_idxs])
# K_test = gk.transform(Gs[test_idxs])
K_train = np.random.randn(80, 80)
K_test = np.random.randn(20, 80)
gpr = GPR(kernel='precomputed')
gpr.fit(K_train, y[train_idxs])
y_pred = gpr.predict(K_test)
rmse = mean_squared_error(y[test_idxs], y_pred, squared=False)
errors.append(rmse)
return -np.mean(errors)
score = Kfold_CV_GPR(Gs=None, y=np.random.randn(100, ), n_iter=4, n_splits=5)
print(score)
However, I am getting the following error
TypeError: Cannot clone object ''precomputed'' (type <class 'str'>): it does not seem to be a scikit-learn
estimator as it does not implement a 'get_params' method.
When I change sklearn.gaussian_process.GaussianProcessRegressor to sklearn.svm.SVR (support vector regression), my code doesn't throw any error, but it will run forever for some reason. I also tested classifers like sklearn.svm.SVC and my code is working fine.
Anyone know how to use precomputed kernel in a scikit-learn's GaussianProcessRegressor?

Poor accuarcy score for Semi-Supervised Support Vector machine

I am using a Semi-Supervised approach for Support Vector Machine in Python for the image classification from PASCAL VOC 2007 data.
I have tried with the default parameters from the libraries and also tuned them but it get extremely bad accuracy of about only ~ 2%.
Below is my code:
import pandas as pd
import numpy as np
from sklearn import decomposition
from sklearn.model_selection import train_test_split
from numpy import concatenate
import numpy as np
from sklearn import datasets
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn import svm
from sklearn import decomposition
import warnings
warnings.filterwarnings("ignore")
color_layout_features = pd.read_pickle("color_layout_descriptor.pkl")
bow_surf = pd.read_pickle("bow_surf.pkl")
color_hist_features = pd.read_pickle("hist.pkl")
labels = pd.read_pickle("labels.pkl")
# Feat. Scaling
def scale(X, x_min, x_max):
nom = (X-X.min(axis=0))*(x_max-x_min)
denom = X.max(axis=0) - X.min(axis=0)
denom[denom==0] = 1
return x_min + nom/denom
# normalization
def normalize(x):
return (x - np.min(x))/(np.max(x) - np.min(x))
color_layout_features_scaled = scale(color_layout_features, 0, 1)
color_hist_features_scaled = scale(color_hist_features, 0, 1)
bow_surf_scaled = scale(bow_surf, 0, 1)
features = np.hstack([color_layout_features_scaled, color_hist_features_scaled, bow_surf_scaled])
# define dataset
X, Y = features, labels
X = normalize(X)
pca = decomposition.PCA(n_components=100)
pca.fit(X)
X = pca.transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.30, random_state=1, stratify=Y)
# split train into labeled and unlabeled
X_train_lab, X_test_unlab, y_train_lab, y_test_unlab = train_test_split(X_train, y_train, test_size=0.30, random_state=1, stratify=y_train)
# create the training dataset input
X_train_mixed = concatenate((X_train_lab, X_test_unlab))
# create "no label" for unlabeled data
nolabel = [-1 for _ in range(len(y_test_unlab))]
# recombine training dataset labels
y_train_mixed = concatenate((y_train_lab, nolabel))
from semisupervised import S3VM
model = S3VM(kernel="Linear", C = 1e-2, gamma = 0.5, lamU = 1.0, probability=True)
#model.fit(X_train_mixed, _train_mixed)
model.fit(np.vstack((X_train_lab, X_test_unlab)), np.append(y_train_lab, nolabel))
#model.fit(np.vstack((label_X_train, unlabel_X_train)), np.append(label_y_train, unlabel_y))
# predict
predict = model.predict(X_test)
acc = metrics.accuracy_score(y_test, predict)
# metric
print("accuracy", acc*100)
accuracy 2.6692291266282298
I am using a Transductive version of SVM (TSVM) from the semisupervised library. But not sure what am I doing wrong so that even after tweaking the parameters I still get the same result. Any inputs would be helpful.
I refer https://github.com/rosefun/SemiSupervised/blob/master/semisupervised/TSVM.py to make the implementation. Any inputs would be helpful.

Please consider that according to link Documentation "The unlabeled samples should be labeled as -1".

Fetching the loss values (MAE) per iteration using sklearn MLPRegressor

I want to check my loss values using MSE during the training process, how to fetching the loss values using MSE at each of iteration?., thank you.
from sklearn.neural_network import MLPRegressor
from sklearn.metrics import mean_absolute_error
dataset = open_dataset("forex.csv")
dataset_vector = [float(i[-1]) for i in dataset]
normalized_dataset_vector = normalize_vector(dataset_vector)
training_vector, validation_vector, testing_vector = split_dataset(training_size, validation_size, testing_size, normalized_dataset_vector)
training_features = get_features(training_vector)
training_fact = get_fact(training_vector)
validation_features = get_features(validation_vector)
validation_fact = get_fact(validation_vector)
model = MLPRegressor(activation=activation, alpha=alpha, hidden_layer_sizes=(neural_net_structure[1],), max_iter=number_of_iteration, random_state=seed)
model.fit(training_features, training_fact)
pred = model.predict(training_features)
err = mean_absolute_error(pred, validation_fact)
print(err)

There's no callbacks object like there is in Keras so you'll have to loop over the fitting process to get it for each iteration. Something like the below will work for you
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import mean_absolute_error
# create some toy data
X = np.random.random((100, 5))
y = np.random.choice([0, 1], 100)
max_iter = 500
mlp = MLPClassifier(hidden_layer_sizes=(10, 10, 10), max_iter=max_iter)
errors = []
for i in range(max_iter):
mlp.partial_fit(X, y, classes=[0, 1])
pred = mlp.predict(X)
errors.append(mean_absolute_error(y, pred))
Which throws an annoying DeprecationWarning at the moment, but that can be ignored. The only problem with using this method is that you have to manually keep track of whether or not your model has converged. Personally I would suggest using Keras instead of sklearn if you want to work with neural networks.

partial_fit Sklearn's MLPClassifier

I've been trying to use Sklearn's neural network MLPClassifier. I have a dataset that is of size 1000 instances (with binary outputs) and I want to apply a basic Neural Net with 1 hidden layer to it.
The issue is that my data instances are not available all at the same time. At any point in time, I only have access to 1 data instance. I thought that partial_fit method of MLPClassifier can be used for this so I simulated the problem with an imaginary dataset of 1000 inputs and looped over the inputs one at a time and partial_fit to each instance but when I run the code, the neural net learns nothing and the predicted output is all zeros.
I am clueless as to what might be causing the problem. Any thought is hugely appreciated.
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:N/2,:]
train_target = output[0:N/2]
test_input= input[N/2:N,:]
test_target = output[N/2:N]
#Creating and training the Neural Net
clf = MLPClassifier(activation='tanh', algorithm='sgd', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= True,
max_iter=1, warm_start=True)
classes=[0,1]
for j in xrange(0,100):
for i in xrange(0,train_input.shape[0]):
input_inst = [train_input[i,:]]
input_inst = np.asarray(input_inst)
target_inst= [train_target[i]]
target_inst = np.asarray(target_inst)
clf=clf.partial_fit(input_inst,target_inst,classes)
#Testing the Neural Net
y_pred = clf.predict(test_input)
print y_pred

Explanation of the problem
The problem is with self.label_binarizer_.fit(y) in line 895 in multilayer_perceptron.py.
Whenever you call clf.partial_fit(input_inst,target_inst,classes), you call self.label_binarizer_.fit(y) where y has only one sample corresponding to one class, in this case. Therefore, if the last sample is of class 0, then your clf will classify everything as class 0.
Solution
As a temporary fix, you can edit multilayer_perceptron.py at line 895.
It is found in a directory similar to this python2.7/site-packages/sklearn/neural_network/
At line 895, change,
self.label_binarizer_.fit(y)
to
if not incremental:
self.label_binarizer_.fit(y)
else:
self.label_binarizer_.fit(self.classes_)
That way, if you are using partial_fit, then self.label_binarizer_ fits on the classes rather than on the individual sample.
Further, the code you posted can be changed to the following to make it work,
from __future__ import division
import numpy as np
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
#Creating an imaginary dataset
input, output = make_classification(1000, 30, n_informative=10, n_classes=2)
input= input / input.max(axis=0)
N = input.shape[0]
train_input = input[0:N/2,:]
train_target = output[0:N/2]
test_input= input[N/2:N,:]
test_target = output[N/2:N]
#Creating and training the Neural Net
# 1. Disable verbose (verbose is annoying with partial_fit)
clf = MLPClassifier(activation='tanh', algorithm='sgd', learning_rate='constant',
alpha=1e-4, hidden_layer_sizes=(15,), random_state=1, batch_size=1,verbose= False,
max_iter=1, warm_start=True)
# 2. Set what the classes are
clf.classes_ = [0,1]
for j in xrange(0,100):
for i in xrange(0,train_input.shape[0]):
input_inst = train_input[[i]]
target_inst= train_target[[i]]
clf=clf.partial_fit(input_inst,target_inst)
# 3. Monitor progress
print "Score on training set: %0.8f" % clf.score(train_input, train_target)
#Testing the Neural Net
y_pred = clf.predict(test_input)
print y_pred
# 4. Compute score on testing set
print clf.score(test_input, test_target)
There are 4 main changes in the code. This should give you a good prediction on both the training and the testing set!
Cheers.