Develop Reference

Python: ValueError: Input must be 1- or 2-d

I have this code to estimate a model using a tobit regression in Python. This is the code which is parsed in three parts: data definition, the estimator builder and estimation.
import numpy as np
from scipy.optimize import minimize
# define the dependent variable and independent variables
X = data.iloc[:, 1:]
y = data.iloc[:, 0]
# Add a column of ones to the independent variables for the constant term
X = np.c_[np.ones(X.shape[0]), X]
# Define the likelihood function for the Tobit model
def likelihood(params, y, X, lower, upper):
beta = params[:-1]
sigma = params[-1]
mu = X # beta
prob = (1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-0.5 * ((y - mu) / sigma)**2))
prob[y < lower] = 0
prob[y > upper] = 0
return -np.log(prob).sum()
# Set the initial values for the parameters and the lower and upper bounds for censoring
params_init = np.random.normal(size=X.shape[1] + 1)
bounds = [(None, None) for i in range(X.shape[1])] + [(1e-10, None)]
# Perform the MLE estimation
res = minimize(likelihood, params_init, args=(y, X, 0, 100), bounds=bounds, method='L-BFGS-B')
# Extract the estimated parameters and their standard errors
params = res.x
stderr = np.sqrt(np.diag(res.hess_inv))
# Print the results
print(f'Coefficients: {params[:-1]}')
print(f'Standard Errors: {stderr[:-1]}')
print(f'Sigma: {params[-1]:.4f}')
Why am I getting this error message?
Thank you.
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-245-5f39f416cc07> in <module>
31 # Extract the estimated parameters and their standard errors
32 params = res.x
---> 33 stderr = np.sqrt(np.diag(res.hess_inv))
34
35 # Print the results
/opt/anaconda3/lib/python3.8/site-packages/numpy/core/overrides.py in diag(*args, **kwargs)
/opt/anaconda3/lib/python3.8/site-packages/numpy/lib/twodim_base.py in diag(v, k)
307 return diagonal(v, k)
308 else:
--> 309 raise ValueError("Input must be 1- or 2-d.")
310
311
ValueError: Input must be 1- or 2-d.
EDIT: If you wanna look at the type of data I'm dealing with, you can simulate them using these lines of code I just wrote:
data = pd.DataFrame()
# Append 'interview probabilities' for individuals with and without disabilities
interview_prob_disabled = np.random.normal(38.63, 28.72, 619)
interview_prob_enabled = np.random.normal(44.27, 28.19, 542)
interview_prob = np.append(interview_prob_disabled, interview_prob_enabled)
# Correct the variable by its mean and standard deviation, without it being negative, nor exceeding 100, nor a float
interview_prob = np.clip(interview_prob, 0, 100)
interview_prob = np.round(interview_prob)
# Add the 'interview probabilities' variable to the dataframe
data['Interview Probabilities'] = interview_prob
# Add other variables such as age, gender, employment status, education, etc.
data['Age'] = np.random.randint(18, 65, size=len(interview_prob))
data['Gender'] = np.random.choice(['Male', 'Female'], size=len(interview_prob))
data['Employment Status'] = np.random.choice(['Employed', 'Unemployed', 'Retired'], size=len(interview_prob))
data['Education Level'] = np.random.choice(['High School', 'College', 'Vocational', 'Graduate School'], size=len(interview_prob))
# Add a 'disability status' variable as a dummy
data['Disability Status'] = np.append(np.repeat('Disabled', 619), np.repeat('Non-disabled', 542))
# Categorical variables
data['Gender'] = data['Gender'].map({'Male': 0, 'Female': 1})
data['Employment Status'] = data['Employment Status'].map({'Employed': 0, 'Unemployed': 1})
data['Education Level'] = data['Education Level'].map({'High School': 0, 'College': 1, 'Vocational': 2, 'Graduate School': 3})
data['Disability Status'] = data['Disability Status'].map({'Disabled': 1, 'Non-disabled': 0})
# Print the df
data

The problem is that your solver, L-BFGS-B yields a LbfgsInvHessProduct object (a linear operator) out of .hess_inv instead of a numpy array (which something like BFGS would give).
One solution to your problem would be to use res.hess_inv.todense() instead.

ValueError: supplied range of [-inf, -inf] is not finite

When trying to plot these graphs, the line:
sub3.hist(x=np.log(df[i]), bins = 100, color="grey")
Gives the error:
ValueError: supplied range of [-inf, -inf] is not finite.
I don't understand this error and can't find any explanations online. Here is the full code. df and df_norm are pandas dataframes with identical data, save for df_norm being minmax normalised.
tb = widgets.TabBar([str(c) for c in range(16)])
k = 0
for c in range(len(df_norm.columns)):
with tb.output_to(c, select=(c < 3)):
colours = ["orange", "green"]
fig = plt.figure(figsize=(20, 5))
plt.subplots_adjust(bottom = 0., left = 0, top = 1., right = 1)
p = 0
g = 1
for i in df_norm.columns[k:k+2]:
sub1 = fig.add_subplot(2,3,g)
sub1.hist(x=df[i], bins = 100, alpha=0.3, color=colours[p])
sub2 = fig.add_subplot(2,3,g+1)
sub2.hist(x=df_norm[i], bins = 100, alpha=0.3, color=colours[p])
sub3 = fig.add_subplot(2,3,g+2)
sub3.hist(x=np.log(df[i]), bins = 100, color="grey")
sub1.set_title(i)
sub2.set_title('title ' + i)
sub3.set_title('title ' + i)
sub1.set_ylabel('label')
p = p + 1
k = k + 1
g = g + 3
Edit, full stack trace:
ValueError Traceback (most recent call last)
<ipython-input-179-d6170fc0d99d> in <module>()
20 sub2.hist(x=df_norm[i], bins = 100, alpha=0.3, color=colours[p])
21 sub3 = fig.add_subplot(2,3,g+2) # two rows, two columns, second cell
---> 22 sub3.hist(x=np.log(df[i]), bins = 100, color="grey")
23 sub1.set_title(i)
24 sub2.set_title('title ' + i)
4 frames
<__array_function__ internals> in histogram(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/numpy/lib/histograms.py in _get_outer_edges(a, range)
314 if not (np.isfinite(first_edge) and np.isfinite(last_edge)):
315 raise ValueError(
--> 316 "supplied range of [{}, {}] is not finite".format(first_edge, last_edge))
317 elif a.size == 0:
318 # handle empty arrays. Can't determine range, so use 0-1.
ValueError: supplied range of [-inf, -inf] is not finite

Typerror on python 3.6.1

Im getting an TypeError: slice indices must be integers or None or have an __index__ method while running this code. The thing is that it works fine on python 3.5.2 (conda) but it gives this error when on python 3.6.1 (conda).
Anyone has idea how to change it to make it work on both versions? Or at least the newer one?
import numpy as np
import random
import math
def geneticAlgorithm(f, x_min, x_max, cel, popSize, pMut, maxIter):
result = {
'x_opt': None,
'f_opt': None,
'x_hist': [],
'f_hist': [],
'f_mean': []
}
# Check the number of dimensions
Dim = len(x_min)
# Initialize Population
population = np.full((popSize, cel*Dim), None)
for i in range(popSize):
population[i,:] = np.random.uniform(cel*Dim)<0
coordinates = getCoordinates(population, cel, x_min, x_max, pMut)
# Calculate fittness of individuals
objFunction = [None]*popSize
for i in range(popSize):
objFunction[i] = f(coordinates[i,:])
# Assign the first population to output
result['x_opt'] = coordinates[np.argmin(objFunction),]
result['f_opt'] = f(coordinates[np.argmin(objFunction),])
# The generational loop
finished = False
currIter = 1
while(finished == False):
# Assign the output
if currIter <= maxIter:
if result['f_opt'] > f(coordinates[np.argmin(objFunction),]):
result['x_opt']= coordinates[np.argmin(objFunction),]
result['f_opt']= f(coordinates[np.argmin(objFunction),])
result['f_hist'].append(result['f_opt'])
result['x_hist'].append(coordinates[np.argmin(objFunction)])
result['f_mean'].append(np.mean(objFunction))
else: finished = True
# Translate binary coding into real values
coordinates = getCoordinates(population, cel, x_min, x_max, pMut)
# Calculate fittness of the individuals
objFunction = [None]*popSize
for i in range(popSize):
objFunction[i] = f(coordinates[i,:])
rFitt = min(objFunction)/objFunction # Relative Fittness
nrFitt = rFitt / sum(rFitt) # Relative Normalized (sum up to 1) Fittness
# Selection operator (Roulette wheel)
selectedPool = [0] * popSize
for i in range(popSize):
selectedPool[i] = sum(np.random.uniform()>np.cumsum(nrFitt))+1 #znowu runif i cumsum
# Crossover operator (for selected pool)
nextGeneration = np.full((popSize, cel*Dim), None)
for i in range(1, popSize):
parentId = np.round(random.uniform(1,popSize))
cutId = np.round(random.uniform(1,Dim*cel-1)) # Please, do not exceed the matrix sizes
nextGeneration[i, 0:int(cutId)] = population[selectedPool[i]-1, 0:int(cutId)]
nextGeneration[i, (cutId): (Dim*cel)] = population[selectedPool[int(parentId)-1]-1, (cutId) : (Dim*cel)]
# Mutation operator
for i in range(popSize):
arr=np.arange(Dim*cel)
test1=np.random.uniform(size=Dim*cel)>pMut
genomeMutId = arr[np.where(test1)] # Draw the genomes that will mutate
for j in range(len(genomeMutId)):
nextGeneration[i, genomeMutId[j]] = not(nextGeneration[i, genomeMutId[j]])
# Replace the old population
population = nextGeneration
currIter = currIter + 1
return(result)
def intbin(x):
# # Translate the binary coding to real values numbers
b = [2**(idx+1) for idx, v in enumerate(x) if v]
return sum(b)
def getCoordinates(population, cel, x_min, x_max, pMut):
# Transform the binary coding into coordinates
coordinates = np.full((population.shape[0], 2), 0)
for i in range(population.shape[0]):
for j in range(2):
s1=cel*(j)+1
s2=(j+1)*cel
coordinatesTemp = intbin(population[i, range(s1,s2)])
coordinates[i,j] = ((x_max[j]-x_min[j])/(2**cel-1))*coordinatesTemp+x_min[j]
return(coordinates)
xSeed = (3, 4)
n_grid = 100
ub_iter = 100
def myFun(x) :
return ( 0.6 + ((math.sin(x[0]**2-x[1]**2))**2-0.5)/((1+0.001*(x[0]**2+x[1]**2))**2) )
ga = geneticAlgorithm(myFun, (-20, -20), (20, 20), cel=50, popSize = 30, maxIter = ub_iter, pMut = 0.05)
print(ga)
So I ran this code using jupyter with python 3.6.1 and it doesnt work. It works fin on c9.io (that is python 3) and my older version of 3.5.2 on jupyter on another PC. It seems that the problem here is with the lines here:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-1-71595b494779> in <module>()
167 return ( 0.6 + ((math.sin(x[0]**2-x[1]**2))**2-0.5)/((1+0.001*(x[0]**2+x[1]**2))**2) )
168
--> 169 ga = geneticAlgorithm(myFun, (-20, -20), (20, 20), cel=50, popSize = 30, maxIter = ub_iter, pMut = 0.05)
170
171 print(ga)
<ipython-input-1-71595b494779> in geneticAlgorithm(f, x_min, x_max, cel, popSize, pMut, maxIter)
108 #print(selectedPool[int(parentId)-1])
109 nextGeneration[i, 0:int(cutId)] = population[selectedPool[i]-1, 0:int(cutId)]
--> 110 nextGeneration[i, (cutId): (Dim*cel)] = population[selectedPool[int(parentId)-1]-1, (cutId) : (Dim*cel)]
111
112
TypeError: slice indices must be integers or None or have an __index__ method

PyMC3 Multivariate Mixture Model: Constraining components to be non-empty

I am implementing a Personalized Mixture of Multivariate Gaussian Regressions in pymc3 and running into an issue with empty components. After referring to the related PyMC3 mixture model example, I tried implementing the model using univariate normals instead, but I've had some issues there as well.
I've tried several strategies to constrain each component to be non-empty, but each has failed. These are shown in the code below. My specific question is: What is the best way to constrain all components to be non-empty in a mixture of multivariate Gaussians using pymc3?
Note that attempt #1 in the code below comes from the Mixture Model in PyMC3 Example and does not work here.
You can replicate the synthetic data I am using with the function in this gist.
import pymc3 as pm
import numpy as np
import theano
import theano.tensor as T
from scipy import stats
# Extract problem dimensions.
N = X.shape[0] # number of samples
F = X.shape[1] # number of features
pids = I[:, 0].astype(np.int) # primary entity ids
uniq_pids = np.unique(pids) # array of unique primary entity ids
n_pe = len(uniq_pids) # number of primary entities
with pm.Model() as gmreg:
# Init hyperparameters.
a0 = 1
b0 = 1
mu0 = pm.constant(np.zeros(F))
alpha = pm.constant(np.ones(K))
coeff_precisions = pm.constant(1 / X.var(0))
# Init parameters.
# Dirichlet shape parameter, prior on indicators.
pi = pm.Dirichlet(
'pi', a=alpha, shape=K)
# ATTEMPT 1: Make probability of membership for each cluter >= 0.1
# ================================================================
pi_min_potential = pm.Potential(
'pi_min_potential', T.switch(T.min(pi) < .1, -np.inf, 0))
# ================================================================
# The multinomial (and by extension, the Categorical), is a symmetric
# distribution. Using this as a prior for the indicator variables Z
# makes the likelihood invariant under the many possible permutations of
# the indices. This invariance is inherited in posterior inference.
# This invariance model implies unidentifiability and induces label
# switching during inference.
# Resolve by ordering the components to have increasing weights.
# This does not deal with the parameter identifiability issue.
order_pi_potential = pm.Potential(
'order_pi_potential',
T.sum([T.switch(pi[k] - pi[k-1] < 0, -np.inf, 0)
for k in range(1, K)]))
# Indicators, specifying which cluster each primary entity belongs to.
# These are draws from Multinomial with 1 trial.
init_pi = stats.dirichlet.rvs(alpha.eval())[0]
test_Z = np.random.multinomial(n=1, pvals=init_pi, size=n_pe)
as_cat = np.nonzero(test_Z)[1]
Z = pm.Categorical(
'Z', p=pi, shape=n_pe, testval=as_cat)
# ATTEMPT 2: Give infinite negative likelihood to the case
# where any of the clusters have no users assigned.
# ================================================================
# sizes = [T.eq(Z, k).nonzero()[0].shape[0] for k in range(K)]
# nonempty_potential = pm.Potential(
# 'comp_nonempty_potential',
# np.sum([T.switch(sizes[k] < 1, -np.inf, 0) for k in range(K)]))
# ================================================================
# ATTEMPT 3: Add same sample to each cluster, each has at least 1.
# ================================================================
# shared_X = X.mean(0)[None, :]
# shared_y = y.mean().reshape(1)
# X = T.concatenate((shared_X.repeat(K).reshape(K, F), X))
# y = T.concatenate((shared_y.repeat(K), y))
# Add range(K) on to the beginning to include shared instance.
# Z_expanded = Z[pids]
# Z_with_shared = T.concatenate((range(K), Z_expanded))
# pid_idx = pm.Deterministic('pid_idx', Z_with_shared)
# ================================================================
# Expand user cluster indicators to each observation for each user.
pid_idx = pm.Deterministic('pid_idx', Z[pids])
# Construct masks for each component.
masks = [T.eq(pid_idx, k).nonzero() for k in range(K)]
comp_sizes = [masks[k][0].shape[0] for k in range(K)]
# Component regression precision parameters.
beta = pm.Gamma(
'beta', alpha=a0, beta=b0, shape=(K,),
testval=np.random.gamma(a0, b0, size=K))
# Regression coefficient matrix, with coeffs for each component.
W = pm.MvNormal(
'W', mu=mu0, tau=T.diag(coeff_precisions), shape=(K, F),
testval=np.random.randn(K, F) * std)
# The mean of the observations is the result of a regression, with
# coefficients determined by the cluster the sample belongs to.
# Now we have K different multivariate normal distributions.
X = T.cast(X, 'float64')
y = T.cast(y, 'float64')
comps = []
for k in range(K):
mask_k = masks[k]
X_k = X[mask_k]
y_k = y[mask_k]
n_k = comp_sizes[k]
precision_matrix = beta[k] * T.eye(n_k)
comp_k = pm.MvNormal(
'comp_%d' % k,
mu=T.dot(X_k, W[k]), tau=precision_matrix,
observed=y_k)
comps.append(comp_k)
The first two approaches fail to ensure non-empty clusters; attempting to sample results in a LinAlgError:
with gmreg:
step1 = pm.Metropolis(vars=[pi, beta, W])
step2 = pm.ElemwiseCategoricalStep(vars=[Z], values=np.arange(K))
tr = pm.sample(100, step=[step1, step2])
...:
Failed to compute determinant []
---------------------------------------------------------------------------
LinAlgError Traceback (most recent call last)
<ipython-input-2-c7df53f4c6a5> in <module>()
2 step1 = pm.Metropolis(vars=[pi, beta, W])
3 step2 = pm.ElemwiseCategoricalStep(vars=[Z], values=np.arange(K))
----> 4 tr = pm.sample(100, step=[step1, step2])
5
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/sampling.pyc in sample(draws, step, start, trace, chain, njobs, tune, progressbar, model, random_seed)
155 sample_args = [draws, step, start, trace, chain,
156 tune, progressbar, model, random_seed]
--> 157 return sample_func(*sample_args)
158
159
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/sampling.pyc in _sample(draws, step, start, trace, chain, tune, progressbar, model, random_seed)
164 progress = progress_bar(draws)
165 try:
--> 166 for i, strace in enumerate(sampling):
167 if progressbar:
168 progress.update(i)
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/sampling.pyc in _iter_sample(draws, step, start, trace, chain, tune, model, random_seed)
246 if i == tune:
247 step = stop_tuning(step)
--> 248 point = step.step(point)
249 strace.record(point)
250 yield strace
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/step_methods/compound.pyc in step(self, point)
12 def step(self, point):
13 for method in self.methods:
---> 14 point = method.step(point)
15 return point
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/step_methods/arraystep.pyc in step(self, point)
87 inputs += [point]
88
---> 89 apoint = self.astep(bij.map(point), *inputs)
90 return bij.rmap(apoint)
91
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/step_methods/gibbs.pyc in astep(self, q, logp)
38
39 def astep(self, q, logp):
---> 40 p = array([logp(v * self.sh) for v in self.values])
41 return categorical(p, self.var.dshape)
42
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/blocking.pyc in __call__(self, x)
117
118 def __call__(self, x):
--> 119 return self.fa(self.fb(x))
/home/mack/anaconda/lib/python2.7/site-packages/pymc3/model.pyc in __call__(self, *args, **kwargs)
423 def __call__(self, *args, **kwargs):
424 point = Point(model=self.model, *args, **kwargs)
--> 425 return self.f(**point)
426
427 compilef = fastfn
/home/mack/anaconda/lib/python2.7/site-packages/theano/compile/function_module.pyc in __call__(self, *args, **kwargs)
604 self.fn.nodes[self.fn.position_of_error],
605 self.fn.thunks[self.fn.position_of_error],
--> 606 storage_map=self.fn.storage_map)
607 else:
608 # For the c linker We don't have access from
/home/mack/anaconda/lib/python2.7/site-packages/theano/compile/function_module.pyc in __call__(self, *args, **kwargs)
593 t0_fn = time.time()
594 try:
--> 595 outputs = self.fn()
596 except Exception:
597 if hasattr(self.fn, 'position_of_error'):
/home/mack/anaconda/lib/python2.7/site-packages/theano/gof/op.pyc in rval(p, i, o, n)
766 # default arguments are stored in the closure of `rval`
767 def rval(p=p, i=node_input_storage, o=node_output_storage, n=node):
--> 768 r = p(n, [x[0] for x in i], o)
769 for o in node.outputs:
770 compute_map[o][0] = True
/home/mack/anaconda/lib/python2.7/site-packages/theano/tensor/nlinalg.pyc in perform(self, node, (x,), (z,))
267 def perform(self, node, (x,), (z, )):
268 try:
--> 269 z[0] = numpy.asarray(numpy.linalg.det(x), dtype=x.dtype)
270 except Exception:
271 print 'Failed to compute determinant', x
/home/mack/anaconda/lib/python2.7/site-packages/numpy/linalg/linalg.pyc in det(a)
1769 """
1770 a = asarray(a)
-> 1771 _assertNoEmpty2d(a)
1772 _assertRankAtLeast2(a)
1773 _assertNdSquareness(a)
/home/mack/anaconda/lib/python2.7/site-packages/numpy/linalg/linalg.pyc in _assertNoEmpty2d(*arrays)
220 for a in arrays:
221 if a.size == 0 and product(a.shape[-2:]) == 0:
--> 222 raise LinAlgError("Arrays cannot be empty")
223
224
LinAlgError: Arrays cannot be empty
Apply node that caused the error: Det(Elemwise{Mul}[(0, 1)].0)
Inputs types: [TensorType(float64, matrix)]
Inputs shapes: [(0, 0)]
Inputs strides: [(8, 8)]
Inputs values: [array([], shape=(0, 0), dtype=float64)]
Backtrace when the node is created:
File "/home/mack/anaconda/lib/python2.7/site-packages/pymc3/distributions/multivariate.py", line 66, in logp
result = k * T.log(2 * np.pi) + T.log(1./det(tau))
HINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.
...which indicates the component is empty, since the precision matrix has shape (0, 0).
The third method actually resolves the empty component issue but gives very strange inference behavior. I selected a burn-in based on traceplots and thinned to every 10th sample. The samples are still highly autocorrelated but much better than without thinning. At this point, I summed the Z values across the samples, and this is what I get:
In [3]: with gmreg:
step1 = pm.Metropolis(vars=[pi, beta, W])
step2 = pm.ElemwiseCategoricalStep(vars=[Z], values=np.arange(K))
tr = pm.sample(1000, step=[step1, step2])
...:
[-----------------100%-----------------] 1000 of 1000 complete in 258.8 sec
...
In [24]: zvals = tr[300::10]['Z']
In [25]: np.array([np.bincount(zvals[:, n]) for n in range(nusers)])
Out[25]:
array([[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70],
[ 0, 0, 70]])
So for some reason, all of the users are being assigned to the last cluster for every sample.

I have run into a similar problem. Something like this worked for a mixture of multivariate gaussians model. As for whether it's the best, it's certainly the best solution I've found.
pm.Potential('pi_min_potential', T.switch(
T.all(
[pi[i, 0] < 0.1 for i in range(K)]), -np.inf, 0))
The key here is that you need to account for each potential that is below your cutoff. Further, you should adjust the shape of your pi distribution, as mentioned in the comments. This will affect your indexing in the T.switch call (on the pi[i,0]).

Converting a mixture of gaussians to PyMC3

I am trying to learn PyMC3, I want to make a simple mixture of gaussians example. I found this example and want to convert it to pymc3 but I'm currently getting an error when trying to plot the traceplot.
n1 = 500
n2 = 200
n = n1+n2
mean1 = 21.8
mean2 = 42.0
precision = 0.1
sigma = np.sqrt(1 / precision)
# precision = 1/sigma^2
print "sigma1: %s" % sigma1
print "sigma2: %s" % sigma2
data1 = np.random.normal(mean1,sigma,n1)
data2 = np.random.normal(mean2,sigma,n2)
data = np.concatenate([data1 , data2])
#np.random.shuffle(data)
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111, xlabel='x', ylabel='y', title='mixture of 2 guassians')
ax.plot(range(0,n1+n2), data, 'x', label='data')
plt.legend(loc=0)
with pm.Model() as model:
#priors
p = pm.Uniform( "p", 0 , 1) #this is the fraction that come from mean1 vs mean2
ber = pm.Bernoulli( "ber", p = p) # produces 1 with proportion p.
precision = pm.Gamma('precision', alpha=0.1, beta=0.1)
mean1 = pm.Normal( "mean1", 0, 0.01 ) #better to use normals versus Uniforms (unless you are certain the value is truncated at 0 and 200
mean2 = pm.Normal( "mean2", 0, 0.01 )
mean = pm.Deterministic('mean', ber*mean1 + (1-ber)*mean2)
process = pm.Normal('process', mu=mean, tau=precision, observed=data)
# inference
step = pm.Metropolis()
trace = pm.sample(10000, step)
pm.traceplot(trace)
Error:
sigma1: 3.16227766017
sigma2: 1.69030850946
[-----------------100%-----------------] 10000 of 10000 complete in 4.4 sec
---------------------------------------------------------------------------
LinAlgError Traceback (most recent call last)
<ipython-input-10-eb728824de83> in <module>()
44 step = pm.Metropolis()
45 trace = pm.sample(10000, step)
---> 46 pm.traceplot(trace)
/usr/lib/python2.7/site-packages/pymc-3.0-py2.7.egg/pymc/plots.pyc in traceplot(trace, vars, figsize, lines, combined, grid)
70 ax[i, 0].set_xlim(mind - .5, maxd + .5)
71 else:
---> 72 kdeplot_op(ax[i, 0], d)
73 ax[i, 0].set_title(str(v))
74 ax[i, 0].grid(grid)
/usr/lib/python2.7/site-packages/pymc-3.0-py2.7.egg/pymc/plots.pyc in kdeplot_op(ax, data)
94 for i in range(data.shape[1]):
95 d = data[:, i]
---> 96 density = kde.gaussian_kde(d)
97 l = np.min(d)
98 u = np.max(d)
/usr/lib64/python2.7/site-packages/scipy/stats/kde.pyc in __init__(self, dataset, bw_method)
186
187 self.d, self.n = self.dataset.shape
--> 188 self.set_bandwidth(bw_method=bw_method)
189
190 def evaluate(self, points):
/usr/lib64/python2.7/site-packages/scipy/stats/kde.pyc in set_bandwidth(self, bw_method)
496 raise ValueError(msg)
497
--> 498 self._compute_covariance()
499
500 def _compute_covariance(self):
/usr/lib64/python2.7/site-packages/scipy/stats/kde.pyc in _compute_covariance(self)
507 self._data_covariance = atleast_2d(np.cov(self.dataset, rowvar=1,
508 bias=False))
--> 509 self._data_inv_cov = linalg.inv(self._data_covariance)
510
511 self.covariance = self._data_covariance * self.factor**2
/usr/lib64/python2.7/site-packages/scipy/linalg/basic.pyc in inv(a, overwrite_a, check_finite)
381 inv_a, info = getri(lu, piv, lwork=lwork, overwrite_lu=1)
382 if info > 0:
--> 383 raise LinAlgError("singular matrix")
384 if info < 0:
385 raise ValueError('illegal value in %d-th argument of internal '
LinAlgError: singular matrix

Thanks to Fonnesbeck for answering this on the github issue tracker:
https://github.com/pymc-devs/pymc3/issues/452
here is the updated code:
with pm.Model() as model:
#priors
p = pm.Uniform( "p", 0 , 1) #this is the fraction that come from mean1 vs mean2
ber = pm.Bernoulli( "ber", p = p, shape=len(data)) # produces 1 with proportion p.
sigma = pm.Uniform('sigma', 0, 100)
precision = sigma**-2
mean = pm.Normal( "mean", 0, 0.01, shape=2 )
mu = pm.Deterministic('mu', mean[ber])
process = pm.Normal('process', mu=mu, tau=precision, observed=data)
with model:
step1 = pm.Metropolis([p, sigma, mean])
step2 = pm.BinaryMetropolis([ber])
trace = pm.sample(10000, [step1, step2])
You need to use BinaryMetropolis when inferring a Bernoulli random variable

And an even simpler and quicker version is as follows:
with pm.Model() as model2:
p = pm.Beta( "p", 1., 1.)
means = pm.Uniform('mean', 15, 60, shape=2)
sigma = pm.Uniform('sigma', 0, 20, testval=5)
process = pm.NormalMixture('obs', tt.stack([p, 1-p]), means, sd=sigma, observed=data)
with model2:
step = pm.Metropolis()
trace = pm.sample(10000, step=step)

I know this issue is old, but I am trying differente examples of PyMC3 usages to get used to modeling in PyMC3. The answer as given above does not work in current version 1.0 of PyMC3 (It does not distringuish the two means correctly). The minimum changes I had to do in order to make it work were the following:
1)
# mean = pm.Normal("mean", 0, 0.01, shape=2 )
mean = pm.Uniform('mean', 15, 60, shape=2)
2)
# step2 = pm.BinaryMetropolis([ber])
step2 = pm.ElemwiseCategorical(vars=[ber], values=[0, 1])
Just in case anybody else is having a similar problem.