Logistic regression: objects are not aligned - python

I am trying to do logistic regression on this dataset from A Ng's machihne learning class in coursera.
The idea is that we have a cost function, which we need to minimize to find the parameters theta.
import numpy as np
from scipy.optimize import fmin_bfgs
data = np.loadtxt('ex2data1.txt',delimiter=",")
m,n = data.shape
X = np.array(np.column_stack((np.ones(m),data[:,:-1])))
y = np.array(data[:,2].reshape(m,1))
theta = np.array(np.zeros(n).reshape(n,1))
def sigmoid(z):
return 1/(1+np.exp(-z))
def hypothesis(X,theta):
return sigmoid( X.dot(theta) )
def cost(theta):
print theta.shape
h = hypothesis(X,theta)
cost = (-y.T.dot(np.log(h))-(1-y).T.dot(np.log(1-h)))/m
return cost
def gradient(theta):
h = hypothesis(X,theta)
grad = ((h-y).T.dot(X)).T/m
return grad.flatten()
def fmin():
initial_theta=np.zeros(n).reshape(n,1)
theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
return theta
print fmin()
I am getting ValueError: Objects are not aligned but I have checked the shapes of all entities and still can't figure it out. Here is the traceback:
---> 32 theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
33
/usr/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc in fmin_bfgs(f, x0, fprime, args, gtol, norm, epsilon, maxiter, full_output, disp, retall, callback)
775 'return_all': retall}
776
--> 777 res = _minimize_bfgs(f, x0, args, fprime, callback=callback, **opts)
778
779 if full_output:
/usr/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc in _minimize_bfgs(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, **unknown_options)
844 gnorm = vecnorm(gfk, ord=norm)
845 while (gnorm > gtol) and (k < maxiter):
--> 846 pk = -numpy.dot(Hk, gfk)
847 try:
848 alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
ValueError: objects are not aligned


I modified your code, it can get the same result as LogisticRegression in sklearn with c=inf:
import numpy as np
from scipy.optimize import fmin_bfgs
import io
data = np.loadtxt('ex2data1.txt',delimiter=",")
m,n = data.shape
X = np.array(np.column_stack((np.ones(m),data[:,:-1])))
y = np.array(data[:,2].reshape(m,1))
theta = np.array(np.zeros(n).reshape(n,1))
def sigmoid(z):
return 1/(1+np.exp(-z))
def hypothesis(X,theta):
return sigmoid( X.dot(theta) )
def cost(theta):
h = hypothesis(X,theta)
cost = (-y.T.dot(np.log(h))-(1-y).T.dot(np.log(1-h)))/m
r = cost[0]
if np.isnan(r):
return np.inf
return r
def gradient(theta):
theta = theta.reshape(-1, 1)
h = hypothesis(X,theta)
grad = ((h-y).T.dot(X)).T/m
return grad.flatten()
def fmin():
initial_theta=np.zeros(n)
theta=fmin_bfgs(cost,initial_theta,fprime=gradient)
return theta
theta = fmin()

Related

ValueError: too many values to unpack, while using LMfit with solve_ivp

I am facing ValueError: too many values to unpack (expected 2) while optimizing parameters of a system of ODEs using solve_ivp. In fact I get the same error when I tried to use solve_ivp instead of odeint in this SO answer, which you may use as a minimal working example since it has the same problem as far as I am concerned. The only changes I made to that code is swap positions of y, t in arguments for f and similarly while solving it using solve_ivp like so: x = solve_ivp(f, t, x0, args=(paras,)) instead of using odeint in g
Here's the full code for the sake of convenience:
# import libraries
import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import odeint, solve_ivp
from lmfit import minimize, Parameters, Parameter, report_fit
def f(t, y, paras):
"""
Your system of differential equations
"""
x1 = y[0]
x2 = y[1]
x3 = y[2]
try:
k0 = paras['k0'].value
k1 = paras['k1'].value
except KeyError:
k0, k1 = paras
# the model equations
f0 = -k0 * x1
f1 = k0 * x1 - k1 * x2
f2 = k1 * x2
return [f0, f1, f2]
def g(t, x0, paras):
"""
Solution to the ODE x'(t) = f(t,x,k) with initial condition x(0) = x0
"""
x = solve_ivp(f, t, x0, args=(paras,))
return x
def residual(paras, t, data):
"""
compute the residual between actual data and fitted data
"""
x0 = paras['x10'].value, paras['x20'].value, paras['x30'].value
model = g(t, x0, paras)
# you only have data for one of your variables
x2_model = model[:, 1]
return (x2_model - data).ravel()
# initial conditions
x10 = 5.
x20 = 0
x30 = 0
y0 = [x10, x20, x30]
# measured data
t_measured = np.linspace(0, 9, 10)
x2_measured = np.array([0.000, 0.416, 0.489, 0.595, 0.506, 0.493, 0.458, 0.394, 0.335, 0.309])
plt.figure()
plt.scatter(t_measured, x2_measured, marker='o', color='b', label='measured data', s=75)
# set parameters including bounds; you can also fix parameters (use vary=False)
params = Parameters()
params.add('x10', value=x10, vary=False)
params.add('x20', value=x20, vary=False)
params.add('x30', value=x30, vary=False)
params.add('k0', value=0.2, min=0.0001, max=2.)
params.add('k1', value=0.3, min=0.0001, max=2.)
# fit model
result = minimize(residual, params, args=(t_measured, x2_measured), method='leastsq') # leastsq nelder
# check results of the fit
data_fitted = g(np.linspace(0., 9., 100), y0, result.params)
# plot fitted data
plt.plot(np.linspace(0., 9., 100), data_fitted[:, 1], '-', linewidth=2, color='red', label='fitted data')
plt.legend()
plt.xlim([0, max(t_measured)])
plt.ylim([0, 1.1 * max(data_fitted[:, 1])])
# display fitted statistics
report_fit(result)
plt.show()
Here's the error traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/home/swami/work/scrap/lmfit_example1.ipynb Cell 4 in <cell line: 67>()
64 params.add('k1', value=0.3, min=0.0001, max=2.)
66 # fit model
---> 67 result = minimize(residual, params, args=(t_measured, x2_measured), method='leastsq') # leastsq nelder
68 # check results of the fit
69 data_fitted = g(np.linspace(0., 9., 100), y0, result.params)
File ~/miniconda3/envs/dynamical/lib/python3.10/site-packages/lmfit/minimizer.py:2600, in minimize(fcn, params, method, args, kws, iter_cb, scale_covar, nan_policy, reduce_fcn, calc_covar, max_nfev, **fit_kws)
2460 """Perform the minimization of the objective function.
2461
2462 The minimize function takes an objective function to be minimized,
(...)
2594
2595 """
2596 fitter = Minimizer(fcn, params, fcn_args=args, fcn_kws=kws,
2597 iter_cb=iter_cb, scale_covar=scale_covar,
2598 nan_policy=nan_policy, reduce_fcn=reduce_fcn,
2599 calc_covar=calc_covar, max_nfev=max_nfev, **fit_kws)
-> 2600 return fitter.minimize(method=method)
File ~/miniconda3/envs/dynamical/lib/python3.10/site-packages/lmfit/minimizer.py:2369, in Minimizer.minimize(self, method, params, **kws)
2366 if (key.lower().startswith(user_method) or
2367 val.lower().startswith(user_method)):
2368 kwargs['method'] = val
-> 2369 return function(**kwargs)
File ~/miniconda3/envs/dynamical/lib/python3.10/site-packages/lmfit/minimizer.py:1693, in Minimizer.leastsq(self, params, max_nfev, **kws)
1691 result.call_kws = lskws
1692 try:
-> 1693 lsout = scipy_leastsq(self.__residual, variables, **lskws)
1694 except AbortFitException:
1695 pass
File ~/.local/lib/python3.10/site-packages/scipy/optimize/_minpack_py.py:410, in leastsq(func, x0, args, Dfun, full_output, col_deriv, ftol, xtol, gtol, maxfev, epsfcn, factor, diag)
408 if not isinstance(args, tuple):
409 args = (args,)
--> 410 shape, dtype = _check_func('leastsq', 'func', func, x0, args, n)
411 m = shape[0]
413 if n > m:
File ~/.local/lib/python3.10/site-packages/scipy/optimize/_minpack_py.py:24, in _check_func(checker, argname, thefunc, x0, args, numinputs, output_shape)
22 def _check_func(checker, argname, thefunc, x0, args, numinputs,
23 output_shape=None):
---> 24 res = atleast_1d(thefunc(*((x0[:numinputs],) + args)))
25 if (output_shape is not None) and (shape(res) != output_shape):
26 if (output_shape[0] != 1):
File ~/miniconda3/envs/dynamical/lib/python3.10/site-packages/lmfit/minimizer.py:586, in Minimizer.__residual(self, fvars, apply_bounds_transformation)
583 self.result.success = False
584 raise AbortFitException(f"fit aborted: too many function evaluations {self.max_nfev}")
--> 586 out = self.userfcn(params, *self.userargs, **self.userkws)
588 if callable(self.iter_cb):
589 abort = self.iter_cb(params, self.result.nfev, out,
590 *self.userargs, **self.userkws)
/home/swami/work/scrap/lmfit_example1.ipynb Cell 4 in residual(paras, t, data)
33 """
34 compute the residual between actual data and fitted data
35 """
37 x0 = paras['x10'].value, paras['x20'].value, paras['x30'].value
---> 38 model = g(t, x0, paras)
40 # you only have data for one of your variables
41 x2_model = model[:, 1]
/home/swami/work/scrap/lmfit_example1.ipynb Cell 4 in g(t, x0, paras)
23 def g(t, x0, paras):
24 """
25 Solution to the ODE x'(t) = f(t,x,k) with initial condition x(0) = x0
26 """
---> 27 x = solve_ivp(f, t, x0, args=(paras,))
28 return x
File ~/.local/lib/python3.10/site-packages/scipy/integrate/_ivp/ivp.py:512, in solve_ivp(fun, t_span, y0, method, t_eval, dense_output, events, vectorized, args, **options)
507 if method not in METHODS and not (
508 inspect.isclass(method) and issubclass(method, OdeSolver)):
509 raise ValueError("`method` must be one of {} or OdeSolver class."
510 .format(METHODS))
--> 512 t0, tf = map(float, t_span)
514 if args is not None:
515 # Wrap the user's fun (and jac, if given) in lambdas to hide the
516 # additional parameters. Pass in the original fun as a keyword
517 # argument to keep it in the scope of the lambda.
518 try:
ValueError: too many values to unpack (expected 2)
Any idea what the problem might be?

dimension error with scipy optimize.minimize

I want to minimise the objective function: objective_function_2() subject to the inequality constraints 69 < T(x) < 71. The objective of the optimisation is to find the set of Fourier series parameters [c0, c1, c2, wavelength] that minimises the objective function (curve fitting problem). fx() is the function that calculates the residuals between the data and the estimated values of the Fourier series. T(x) is the function calculate_tightness() that calculates the range of the fourier series estimated at each iteration.
The problem is that I don't understand why I get this error message: ValueError: all the input arrays must have same number of dimensions, but the array at index 0 has 2 dimension(s) and the array at index 1 has 1 dimension(s)
the objective_function_2() produce an n-vector of residuals and calculate_tightness() one scalar value.
Does anyone have any idea why this is happening?
import numpy as np
from scipy.optimize import LinearConstraint, NonlinearConstraint, BFGS, minimize
ref_fld = np.array([-479.9024323 , -469.80142114, -459.70040999, -449.59939883,
-439.49838768, -429.39737652, -419.29636536, -409.19535421,
-399.09434305, -388.9933319 , -378.89232074, -368.79130958,
-358.69029843, -348.58928727, -338.48827611, -328.38726496,
-318.2862538 , -308.18524265, -298.08423149, -287.98322033,
-277.88220918, -267.78119802, -257.68018686, -247.57917571,
-237.47816455, -227.3771534 , -217.27614224, -207.17513108,
-197.07411993, -186.97310877, -176.87209762, -166.77108646,
-156.6700753 , -146.56906415, -136.46805299, -126.36704183,
-116.26603068, -106.16501952, -96.06400837, -85.96299721,
-75.86198605, -65.7609749 , -55.65996374, -45.55895258,
-35.45794143, -25.35693027, -15.25591912, -5.15490796,
4.9461032 , 15.04711435, 25.14812551, 35.24913667,
45.35014782, 55.45115898, 65.55217013, 75.65318129,
85.75419245, 95.8552036 , 105.95621476, 116.05722591,
126.15823707, 136.25924823, 146.36025938, 156.46127054,
166.5622817 , 176.66329285, 186.76430401, 196.86531516,
206.96632632, 217.06733748, 227.16834863, 237.26935979,
247.37037095, 257.4713821 , 267.57239326, 277.67340441,
287.77441557, 297.87542673, 307.97643788, 318.07744904,
328.1784602 , 338.27947135, 348.38048251, 358.48149366,
368.58250482, 378.68351598, 388.78452713, 398.88553829,
408.98654944, 419.0875606 , 429.18857176, 439.28958291,
449.39059407, 459.49160523, 469.59261638, 479.69362754,
489.79463869, 499.89564985, 509.99666101, 520.09767216])
fld = np.array([-300.41522506, -120.9280477 , -274.77413647, 494.45656622,
-44.00495929, -479.90233432, 58.55913797, -326.056248 ,
84.20018256, 443.17449743])
flr = np.array([-13.20752855, 38.56985419, 44.28484794, -51.64708478,
-10.50558888, -49.95878419, -53.88137785, -12.73304144,
-54.2792669 , -7.59544309])
def fourier_series(x, c0, c1, c2, w):
"""
Parameters
----------
x
c0
c1
c2
w
Returns
-------
"""
v = np.array(x.astype(float))
# v.fill(c0)
v = c0 + c1 * np.cos(2 * np.pi / w * x) + c2 * np.sin(2 * np.pi / w * x)
return np.rad2deg(np.arctan(v))
def calculate_splot(ref_fold_frame, popt):
return np.rad2deg(np.arctan(fourier_series(ref_fold_frame, *popt)))
def calculate_tightness(theta):
curve = calculate_splot(ref_fld, theta)
amax = np.arctan(np.deg2rad(curve.max()))
amin = np.arctan(np.deg2rad(curve.min()))
return 180 - np.rad2deg(2*np.tan((amax - amin) / 2))
def fx(theta, x, y):
# function to calculate residuals for optimisation (least squares)
return np.tan(np.deg2rad(y)) - fourier_series(x, *theta)
def objective_function_2(theta):
x = fld
y = flr
return fx(theta, x, y) + calculate_tightness(theta)
Cx = NonlinearConstraint(calculate_tightness, 69, 71, jac='2-point', hess=BFGS())
x0 = [0, 1, 1, 500]
res = minimize(objective_function_2, x0, constraints=[Cx], method='trust-constr',
options={'verbose': 1}, )
Error message
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_91/1210121354.py in <module>
1 Cx = NonlinearConstraint(calculate_tightness, 69, 71, jac='2-point', hess=BFGS()) #setup_optimisation_constraints(constraints_matrix, [60, -1e-2], [71, 1e-4], linear=False)
2 x0 = theta[recovery<1][0]
----> 3 res = minimize(objective_function_2, x0, constraints=[Cx], method='trust-constr',
4
5 options={'verbose': 1}, )
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
632 constraints, callback=callback, **options)
633 elif meth == 'trust-constr':
--> 634 return _minimize_trustregion_constr(fun, x0, args, jac, hess, hessp,
635 bounds, constraints,
636 callback=callback, **options)
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/scipy/optimize/_trustregion_constr/minimize_trustregion_constr.py in _minimize_trustregion_constr(fun, x0, args, grad, hess, hessp, bounds, constraints, xtol, gtol, barrier_tol, sparse_jacobian, callback, maxiter, verbose, finite_diff_rel_step, initial_constr_penalty, initial_tr_radius, initial_barrier_parameter, initial_barrier_tolerance, factorization_method, disp)
507
508 elif method == 'tr_interior_point':
--> 509 _, result = tr_interior_point(
510 objective.fun, objective.grad, lagrangian_hess,
511 n_vars, canonical.n_ineq, canonical.n_eq,
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/scipy/optimize/_trustregion_constr/tr_interior_point.py in tr_interior_point(fun, grad, lagr_hess, n_vars, n_ineq, n_eq, constr, jac, x0, fun0, grad0, constr_ineq0, jac_ineq0, constr_eq0, jac_eq0, stop_criteria, enforce_feasibility, xtol, state, initial_barrier_parameter, initial_tolerance, initial_penalty, initial_trust_radius, factorization_method)
302 s0 = np.maximum(-1.5*constr_ineq0, np.ones(n_ineq))
303 # Define barrier subproblem
--> 304 subprob = BarrierSubproblem(
305 x0, s0, fun, grad, lagr_hess, n_vars, n_ineq, n_eq, constr, jac,
306 barrier_parameter, tolerance, enforce_feasibility,
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/scipy/optimize/_trustregion_constr/tr_interior_point.py in __init__(self, x0, s0, fun, grad, lagr_hess, n_vars, n_ineq, n_eq, constr, jac, barrier_parameter, tolerance, enforce_feasibility, global_stop_criteria, xtol, fun0, grad0, constr_ineq0, jac_ineq0, constr_eq0, jac_eq0)
51 self.xtol = xtol
52 self.fun0 = self._compute_function(fun0, constr_ineq0, s0)
---> 53 self.grad0 = self._compute_gradient(grad0)
54 self.constr0 = self._compute_constr(constr_ineq0, constr_eq0, s0)
55 self.jac0 = self._compute_jacobian(jac_eq0, jac_ineq0, s0)
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/scipy/optimize/_trustregion_constr/tr_interior_point.py in _compute_gradient(self, g)
137
138 def _compute_gradient(self, g):
--> 139 return np.hstack((g, -self.barrier_parameter*np.ones(self.n_ineq)))
140
141 def _compute_jacobian(self, J_eq, J_ineq, s):
<__array_function__ internals> in hstack(*args, **kwargs)
/mnt/c/Users/rcha0044/LoopPhD/LOOP_ENV/env_18/lib/python3.8/site-packages/numpy/core/shape_base.py in hstack(tup)
343 return _nx.concatenate(arrs, 0)
344 else:
--> 345 return _nx.concatenate(arrs, 1)
346
347
<__array_function__ internals> in concatenate(*args, **kwargs)
ValueError: all the input arrays must have same number of dimensions, but the array at index 0 has 2 dimension(s) and the array at index 1 has 1 dimension(s)
SciPy/NumPy/Python version information
1.7.1 1.21.3 sys.version_info(major=3, minor=8, micro=10, releaselevel='final', serial=0)

How to use cross_validation with curve_fit?

I use scipy.optimize.curve_fit to check models on my data.
Normally, I would have a function of several variables, here y_data is a function of 2 variables in x_data:
import numpy as np
from scipy.optimize import curve_fit
x_data = np.array( [np.arange(10), np.arange(10)*-1+5] )
print( x_data )
[[ 0 1 2 3 4 5 6 7 8 9]
[ 5 4 3 2 1 0 -1 -2 -3 -4]]
y_data = np.arange(10) - 5
print( y_data )
[-5 -4 -3 -2 -1 0 1 2 3 4]
To fit the function, I just define it, and use curve_fit:
def lincomb( X, a, b ):
x1 = X[0]
x2 = X[1]
return a*x1*x2 + b
popt, pcov = curve_fit( lincomb, x_data, y_data )
print( popt )
[-0.17857143 -1.57142857]
The latter are the optimized values of the coefficients a and b in the function.
Now, I would like to use cross validation from sklearn to do the same fit. For this, I packed my function into a class to be used as an estimator, like this:
from sklearn.model_selection import cross_validate
class LinComb:
def __init__( self, a=None, b=None ):
self.a = a
self.b = b
def _lincomb_background(self, X, a, b):
x1 = X[0]
x2 = X[1]
return a*x1*x2 + b
def predict( self, X ):
return self._lincomb_background( X, self.a, self.b )
def fit( self, X, y ):
from scipy.optimize import curve_fit
popt, pcov = curve_fit( self._lincomb_background, X, y )
self.a = popt[0]
self.b = popt[1]
return self
def get_params( self, deep=False ):
return { 'a':self.a, 'b':self.b }
def set_params( self, **parameters ):
for parameter, value in parameters.intems():
setattr( self, parameter, value )
return self
When I then call the cross validation, I get an error in dimensions:
cross_validate( LinComb(), x_data, y_data, cv=5, scoring='neg_mean_squared_error' )
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-60-e0ff8bb83213> in <module>
----> 1 cross_validate( LinComb(), x_data, y_data, cv=5, scoring='neg_mean_squared_error' )
/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_validation.py in cross_validate(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)
215
216 """
--> 217 X, y, groups = indexable(X, y, groups)
218
219 cv = check_cv(cv, y, classifier=is_classifier(estimator))
/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in indexable(*iterables)
228 else:
229 result.append(np.array(X))
--> 230 check_consistent_length(*result)
231 return result
232
/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in check_consistent_length(*arrays)
203 if len(uniques) > 1:
204 raise ValueError("Found input variables with inconsistent numbers of"
--> 205 " samples: %r" % [int(l) for l in lengths])
206
207
ValueError: Found input variables with inconsistent numbers of samples: [2, 10]
Indeed, the dimensions of x_data and y_data are:
print( x_data.shape, y_data.shape )
(2, 10) (10,)
Still I do not understand why the dimensions work in the first, simple case, and how to achieve this cleanly.
Am I missing something?

I found out that I had 2 errors in the dimensions. Since they were simultaneous, I wasn't able to trace them back easily. I will post the answer here, maybe it will be useful sometime.
1. From the documentation
Modifying the example in the documentation helped to trace back the dimension errors.
from sklearn import datasets, linear_model
from sklearn.model_selection import cross_validate, cross_val_score
diabetes = datasets.load_diabetes()
X = diabetes.data[:150]
y = diabetes.target[:150]
lasso = linear_model.Lasso()
cv_results = cross_validate(lasso, X, y, cv=3)
sorted(cv_results.keys())
print( cv_results['test_score'] )
[0.33150734 0.08022311 0.03531764]
Note that cross_validation needs the first dimensions to be the same:
print( X.shape, y.shape )
(150, 10) (150,)
Note that with these new dimensions, the simple way to call curve_fit throws an error:
def lincomb( X, a, b ):
x1 = X[0]
x2 = X[1]
return a*x1*x2 + b
popt, pcov = curve_fit( lincomb, x_data, y_data )
print( popt )
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-6-dedaa241e377> in <module>
4 return a*x1*x2 + b
5
----> 6 popt, pcov = curve_fit( lincomb, x_data, y_data )
7 print( popt )
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in curve_fit(f, xdata, ydata, p0, sigma, absolute_sigma, check_finite, bounds, method, jac, **kwargs)
754 # Remove full_output from kwargs, otherwise we're passing it in twice.
755 return_full = kwargs.pop('full_output', False)
--> 756 res = leastsq(func, p0, Dfun=jac, full_output=1, **kwargs)
757 popt, pcov, infodict, errmsg, ier = res
758 cost = np.sum(infodict['fvec'] ** 2)
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in leastsq(func, x0, args, Dfun, full_output, col_deriv, ftol, xtol, gtol, maxfev, epsfcn, factor, diag)
381 if not isinstance(args, tuple):
382 args = (args,)
--> 383 shape, dtype = _check_func('leastsq', 'func', func, x0, args, n)
384 m = shape[0]
385
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in _check_func(checker, argname, thefunc, x0, args, numinputs, output_shape)
24 def _check_func(checker, argname, thefunc, x0, args, numinputs,
25 output_shape=None):
---> 26 res = atleast_1d(thefunc(*((x0[:numinputs],) + args)))
27 if (output_shape is not None) and (shape(res) != output_shape):
28 if (output_shape[0] != 1):
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in func_wrapped(params)
456 if transform is None:
457 def func_wrapped(params):
--> 458 return func(xdata, *params) - ydata
459 elif transform.ndim == 1:
460 def func_wrapped(params):
ValueError: operands could not be broadcast together with shapes (2,) (10,)
This can be solved by transposing again inside the call to curve_fit:
popt, pcov = curve_fit( lincomb, x_data.T, y_data )
print( popt )
[-0.17857143 -1.57142857]
2. Class
Using the new dimensions in x_data for cross_validation (using the class defined in the question) throws a different error:
from sklearn.model_selection import cross_validate
class LinComb:
def __init__( self, a=None, b=None ):
self.a = a
self.b = b
def _lincomb_background(self, X, a, b):
x1 = X[0]
x2 = X[1]
return a*x1*x2 + b
def predict( self, X ):
return self._lincomb_background( X, self.a, self.b )
def fit( self, X, y ):
from scipy.optimize import curve_fit
popt, pcov = curve_fit( self._lincomb_background, X, y )
self.a = popt[0]
self.b = popt[1]
return self
def get_params( self, deep=False ):
return { 'a':self.a, 'b':self.b }
def set_params( self, **parameters ):
for parameter, value in parameters.intems():
setattr( self, parameter, value )
return self
cross_validate( LinComb(), x_data, y_data, cv=5, scoring='neg_mean_squared_error' )
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-10-e0ff8bb83213> in <module>
----> 1 cross_validate( LinComb(), x_data, y_data, cv=5, scoring='neg_mean_squared_error' )
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py in cross_validate(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score, return_estimator, error_score)
229 return_times=True, return_estimator=return_estimator,
230 error_score=error_score)
--> 231 for train, test in cv.split(X, y, groups))
232
233 zipped_scores = list(zip(*scores))
/usr/local/lib/python3.6/dist-packages/joblib/parallel.py in __call__(self, iterable)
919 # remaining jobs.
920 self._iterating = False
--> 921 if self.dispatch_one_batch(iterator):
922 self._iterating = self._original_iterator is not None
923
/usr/local/lib/python3.6/dist-packages/joblib/parallel.py in dispatch_one_batch(self, iterator)
757 return False
758 else:
--> 759 self._dispatch(tasks)
760 return True
761
/usr/local/lib/python3.6/dist-packages/joblib/parallel.py in _dispatch(self, batch)
714 with self._lock:
715 job_idx = len(self._jobs)
--> 716 job = self._backend.apply_async(batch, callback=cb)
717 # A job can complete so quickly than its callback is
718 # called before we get here, causing self._jobs to
/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py in apply_async(self, func, callback)
180 def apply_async(self, func, callback=None):
181 """Schedule a func to be run"""
--> 182 result = ImmediateResult(func)
183 if callback:
184 callback(result)
/usr/local/lib/python3.6/dist-packages/joblib/_parallel_backends.py in __init__(self, batch)
547 # Don't delay the application, to avoid keeping the input
548 # arguments in memory
--> 549 self.results = batch()
550
551 def get(self):
/usr/local/lib/python3.6/dist-packages/joblib/parallel.py in __call__(self)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def __len__(self):
/usr/local/lib/python3.6/dist-packages/joblib/parallel.py in <listcomp>(.0)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def __len__(self):
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py in _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)
512 estimator.fit(X_train, **fit_params)
513 else:
--> 514 estimator.fit(X_train, y_train, **fit_params)
515
516 except Exception as e:
<ipython-input-9-ff88060f1729> in fit(self, X, y)
15 def fit( self, X, y ):
16 from scipy.optimize import curve_fit
---> 17 popt, pcov = curve_fit( self._lincomb_background, X, y )
18 self.a = popt[0]
19 self.b = popt[1]
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in curve_fit(f, xdata, ydata, p0, sigma, absolute_sigma, check_finite, bounds, method, jac, **kwargs)
754 # Remove full_output from kwargs, otherwise we're passing it in twice.
755 return_full = kwargs.pop('full_output', False)
--> 756 res = leastsq(func, p0, Dfun=jac, full_output=1, **kwargs)
757 popt, pcov, infodict, errmsg, ier = res
758 cost = np.sum(infodict['fvec'] ** 2)
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in leastsq(func, x0, args, Dfun, full_output, col_deriv, ftol, xtol, gtol, maxfev, epsfcn, factor, diag)
381 if not isinstance(args, tuple):
382 args = (args,)
--> 383 shape, dtype = _check_func('leastsq', 'func', func, x0, args, n)
384 m = shape[0]
385
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in _check_func(checker, argname, thefunc, x0, args, numinputs, output_shape)
24 def _check_func(checker, argname, thefunc, x0, args, numinputs,
25 output_shape=None):
---> 26 res = atleast_1d(thefunc(*((x0[:numinputs],) + args)))
27 if (output_shape is not None) and (shape(res) != output_shape):
28 if (output_shape[0] != 1):
/usr/local/lib/python3.6/dist-packages/scipy/optimize/minpack.py in func_wrapped(params)
456 if transform is None:
457 def func_wrapped(params):
--> 458 return func(xdata, *params) - ydata
459 elif transform.ndim == 1:
460 def func_wrapped(params):
ValueError: operands could not be broadcast together with shapes (2,) (8,)
3. Dimension error inside the class
This error is coming from curve_fit, not cross_validation, and must be corrected inside the class, in both functions calling the model _lincomb_background(), that is, both in fit() and predict(). The modified class is:
class LinComb:
def __init__( self, a=None, b=None ):
self.a = a
self.b = b
def _lincomb_background(self, X, a, b):
x1 = X[0]
x2 = X[1]
return a*x1*x2 + b
def predict( self, X ):
return self._lincomb_background( X.T, self.a, self.b ) # Call with transposed X!
def fit( self, X, y ):
from scipy.optimize import curve_fit
popt, pcov = curve_fit( self._lincomb_background, X.T, y ) # Call with transposed X!
self.a = popt[0]
self.b = popt[1]
return self
def get_params( self, deep=False ):
return { 'a':self.a, 'b':self.b }
def set_params( self, **parameters ):
for parameter, value in parameters.intems():
setattr( self, parameter, value )
return self
With these two modified calls, cross_validation works as expected:
cross_validate( LinComb(), x_data, y_data, cv=5, scoring='neg_mean_squared_error' )
{'fit_time': array([0.00105524, 0.00051618, 0.0004158 , 0.00040078, 0.00039887]),
'score_time': array([0.00158715, 0.0001812 , 0.00017715, 0.00017595, 0.00017548]),
'test_score': array([-12.89 , -0.29918379, -3.82378685, -2.72051908,
-7.25 ])}
4. Summary
a) First check that the dimensions for cross_validation() are correct
b) Then adjust dimensions inside the class, in the call to curve_fit()
c) Lastly adjust dimensions inside the class, in predict()

Scipy optimize function: matrixes not aligned

EDIT: The data set is the MNIST data set from the Homework of Week 4 of Andrew Ng's Machine Learning Course
I've checked the question on scipy optimize but I still couldn't figure out what is wrong with my code. I am trying to optimize theta for the oneVsAll question on the Andrew Ng coursera course.
Here is the relevant code
def sigmoid(x):
a = []
for item in x:
a.append(1/(1+math.exp(-item)))
return a
def hypothesis(x, theta):
return np.array(sigmoid(np.dot(x, theta)))
def costFunction(theta, x, y, lamba_):
m = X.shape[0]
part1 = np.dot(y.T, np.log(hypothesis(x, theta)).reshape(m,1))
part2 = np.dot((np.ones((m,1)) - y).T, np.log( 1 - hypothesis(x, theta)).reshape(m,1))
summ = (part1 + part2)
return -summ[0]/m
def gradientVect(theta, x, y, lambda_):
n = X.shape[1]
m = X.shape[0]
gradient = []
theta = theta.reshape(n,1)
beta = hypothesis(x, theta) - y
reg = theta[1:] * lambda_/m
grad = np.dot(X.T, beta) * 1./m
grad[1:] = grad[1:] * reg
return grad.flatten()
from scipy import optimize
def optimizeTheta(x, y, nLabels, lambda_):
for i in np.arange(0, nLabels):
theta = np.zeros((n,1))
res = optimize.minimize(costFunction, theta, args=(x, (y == i)*1, lambda_), method=None,
jac=gradientVect, options={'maxiter':50})
print(res)
return result
but running
optimizeTheta(X, y, 10, 0) # X shape = 401, 500
Gives me the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-247-e0e6e4c1eddd> in <module>()
3 n = X.shape[1]
4
----> 5 optimizeTheta(X, y, 10, 0)
<ipython-input-246-0a15e9f4769a> in optimizeTheta(x, y, nLabels, lambda_)
54 theta = np.zeros((n,1))
55 res = optimize.minimize(costFunction, x0 = theta, args=(x, (y == i)*1, lambda_), method=None,
---> 56 jac=gradientVect, options={'maxiter':50})
57 print(res)
58 return result
//anaconda/lib/python3.5/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
439 return _minimize_cg(fun, x0, args, jac, callback, **options)
440 elif meth == 'bfgs':
--> 441 return _minimize_bfgs(fun, x0, args, jac, callback, **options)
442 elif meth == 'newton-cg':
443 return _minimize_newtoncg(fun, x0, args, jac, hess, hessp, callback,
//anaconda/lib/python3.5/site-packages/scipy/optimize/optimize.py in _minimize_bfgs(fun, x0, args, jac, callback, gtol, norm, eps, maxiter, disp, return_all, **unknown_options)
859 gnorm = vecnorm(gfk, ord=norm)
860 while (gnorm > gtol) and (k < maxiter):
--> 861 pk = -numpy.dot(Hk, gfk)
862 try:
863 alpha_k, fc, gc, old_fval, old_old_fval, gfkp1 = \
ValueError: shapes (401,401) and (2005000,) not aligned: 401 (dim 1) != 2005000 (dim 0)
And I can't figure out why the shapes are not aligned.
Thanks!

So I realized what was wrong with my question.
The problem was the sigmoid function returning a list and not an integer and therefore it messed up the matrixes multiplications afterwards. The new sigmoid function is
def sigmoid(z):
return(1 / (1 + np.exp(-z)))

IndexError: too many indices working with Pandas Dataframe

OK so here's my code for a multi-classification task using one-vs-all logistic regression with some regularization. I've been struggling with this for the past 2 days, I don't know why it doesn't work.
import pandas as pd
import numpy as np
import scipy.optimize as sp
Data = pd.read_csv(Location,
sep=';',
dtype = np.float64,
header = None)
X = Data.ix[:,0:1]
y = Data.ix[:,2:]
y.columns = [0]
def sigmoid(z) :
g = 1.0/(1.0+np.exp(-z))
return g
def lrCostFunction(theta, X, y, lambd):
m , n = X.shape
J=-(y.T.dot(np.log(sigmoid(X.dot(theta))))+(1-y).T.dot(np.log(1-sigmoid(X.dot(theta)))))/m
J = J + (theta.T.dot(theta)- np.power(theta[0,0],2))*(lambd)/(2*m);
return J.ix[0,0]
def Gradient(theta, X, y, lambd):
m , n = X.shape
grad = X.T.dot(sigmoid(X.dot(theta))-y)/m
grad.ix[1:(n-1),:] = grad.ix[1:(n-1),:] + lambd*theta.ix[1:(n-1),:]/m;
return grad.values.flatten().tolist()
def oneVsAll(X, y, num_labels, lambd):
m , n = X.shape
all_theta = pd.DataFrame(data = [[0 for col in range(n+1)] for row in range(num_labels)])
ones = pd.DataFrame(data = [1 for i in range(X.shape[0])])
X = pd.concat([ones,X], axis = 1)
for c in range(0,num_labels-1) :
initial_theta = pd.DataFrame(data = [0 for i in range(n+1)])
theta = sp.minimize(fun = lrCostFunction,
x0 = initial_theta,
args = (X,y,lambd),
method = 'TNC',
jac = Gradient)
all_theta.ix[c,:] = theta
return all_theta
oneVsAll(X, y, 4, 0.1)
And it says :
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-27-b18648b06674> in <module>()
1 theta = pd.DataFrame(data = [0 for i in range(X.shape[1])])
----> 2 oneVsAll(X, y, 4, 0.1)
<ipython-input-26-ba0f7093d1f6> in oneVsAll(X, y, num_labels, lambd)
10 args = (X,y,lambd),
11 method = 'TNC',
---> 12 jac = Gradient)
13 all_theta.ix[c,:] = theta
14 return all_theta
/Users/jean-marcmarty/anaconda/lib/python2.7/site-packages/scipy/optimize/_minimize.pyc in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
381 elif meth == 'tnc':
382 return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,
--> 383 **options)
384 elif meth == 'cobyla':
385 return _minimize_cobyla(fun, x0, args, constraints, **options)
/Users/jean-marcmarty/anaconda/lib/python2.7/site-packages/scipy/optimize/tnc.pyc in _minimize_tnc(fun, x0, args, jac, bounds, eps, scale, offset, mesg_num, maxCGit, maxiter, eta, stepmx, accuracy, minfev, ftol, xtol, gtol, rescale, disp, callback, **unknown_options)
396 offset, messages, maxCGit, maxfun,
397 eta, stepmx, accuracy, fmin, ftol,
--> 398 xtol, pgtol, rescale, callback)
399
400 funv, jacv = func_and_grad(x)
/Users/jean-marcmarty/anaconda/lib/python2.7/site-packages/scipy/optimize/tnc.pyc in func_and_grad(x)
358 else:
359 def func_and_grad(x):
--> 360 f = fun(x, *args)
361 g = jac(x, *args)
362 return f, g
<ipython-input-24-5f31e87e00da> in lrCostFunction(theta, X, y, lambd)
2 m , n = X.shape
3 J=-(y.T.dot(np.log(sigmoid(X.dot(theta))))+(1-y).T.dot(np.log(1-sigmoid(X.dot(theta)))))/m
----> 4 J = J + (theta.T.dot(theta)- np.power(theta[0,0],2))*(lambd)/(2*m);
5 return J.ix[0,0]
IndexError: too many indices

I don't know anything about the math, but the error is coming from this code:
theta[0,0]
Theta is a 1d array, so you'd need to index at as theta[0], unless there was some reason you were expecting it to be 2d?

Categories

Resources