Develop Reference

Multivariate KDE Scipy Stats - what if it's not Gaussian?

I have some 2D data that I am smoothing using:
from scipy.stats import gaussian_kde
kde = gaussian_kde(data)
but what if my data isn't Gaussian/tophat/the other options? Mine looks more elliptical before smoothing, so should I really have a different bandwidth in x and then y? The variance in one direction is a lot higher, and also the values of the x axis are higher, so it feels like a simple Gaussian might miss something?

This is what I get with your defined X and Y. Seems good. Were you expecting something different?
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
def generate(n):
# generate data
np.random.seed(42)
x = np.random.normal(size=n, loc=1, scale=0.01)
np.random.seed(1)
y = np.random.normal(size=n, loc=200, scale=100)
return x, y
x, y = generate(100)
xmin = x.min()
xmax = x.max()
ymin = y.min()
ymax = y.max()
X, Y = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
positions = np.vstack([X.ravel(), Y.ravel()])
values = np.vstack([x, y])
kernel = stats.gaussian_kde(values)
Z = np.reshape(kernel(positions).T, X.shape)
fig, ax = plt.subplots(figsize=(7, 7))
ax.imshow(np.rot90(Z), cmap=plt.cm.gist_earth_r,
extent=[xmin, xmax, ymin, ymax],
aspect='auto', alpha=.75
)
ax.plot(x, y, 'ko', ms=5)
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
plt.show()
The distributions of x and y are Gaussian.
You can verify with seaborn too
import pandas as pd
import seaborn as sns
# I pass a DataFrame because passing
# (x,y) alone will be soon deprecated
g = sns.jointplot(data=pd.DataFrame({'x':x, 'y':y}), x='x', y='y')
g.plot_joint(sns.kdeplot, color="r", zorder=0, levels=6)
update
Kernel Density Estimate of 2-dimensional data is done separately along each axis and then join together.
Let's make an example with the dataset we already used.
As we can see in the seaborn jointplot, you have not only the estimated 2d-kde but also marginal distributions of x and y (the histograms).
So, step by step, let's estimate the density of x and y and then evaluate the density over a linearspace
kde_x = sps.gaussian_kde(x)
kde_x_space = np.linspace(x.min(), x.max(), 100)
kde_x_eval = kde_x.evaluate(kde_x_space)
kde_x_eval /= kde_x_eval.sum()
kde_y = sps.gaussian_kde(y)
kde_y_space = np.linspace(y.min(), y.max(), 100)
kde_y_eval = kde_y.evaluate(kde_y_space)
kde_y_eval /= kde_y_eval.sum()
fig, ax = plt.subplots(1, 2, figsize=(12, 4))
ax[0].plot(kde_x_space, kde_x_eval, 'k.')
ax[0].set(title='KDE of x')
ax[1].plot(kde_y_space, kde_y_eval, 'k.')
ax[1].set(title='KDE of y')
plt.show()
So we now have the marginal distributions of x and y. These are probability density functions so, the joint-probability of x and y can be seen as the intersection of independent events x and y, thus we can multiply the estimated probability density of x and y in a 2d-matrix and plot on 3d projection
# Grid of x and y
X, Y = np.meshgrid(kde_x_space, kde_y_space)
# Grid of probability density
kX, kY = np.meshgrid(kde_x_eval, kde_y_eval)
# Intersection
Z = kX * kY
fig, ax = plt.subplots(
2, 2,
subplot_kw={"projection": "3d"},
figsize=(10, 10))
for i, (elev, anim, title) in enumerate(zip([10, 10, 25, 25],
[0, -90, 25, -25],
['y axis', 'x axis', 'view 1', 'view 2']
)):
# Plot the surface.
surf = ax.flat[i].plot_surface(X, Y, Z, cmap=plt.cm.gist_earth_r,
linewidth=0, antialiased=False, alpha=.75)
ax.flat[i].scatter(x, y, zs=0, zdir='z', c='k')
ax.flat[i].set(
xlabel='x', ylabel='y',
title=title
)
ax.flat[i].view_init(elev=elev, azim=anim)
plt.show()
This is a very simple and naif method but only to have an idea on how it works and why x and y scales don't matter for a 2d-KDE.

How to take into account the data's uncertainty (standard deviation) when fitting with scipy.linalg.lstsq?

I am trying to surface fit 3d data (z is a function of x and y). I have assymetrical error bars for each point. I would like the fit to take this uncertainty into account.
I am using scipy.linalg.lstsq(). It does not have any option for uncertainties in its arguments.
I am trying to adapt some code found on this page.
import numpy as np
import scipy.linalg
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
# Create data with x and y random over [-2, 2], and z a Gaussian function of x and y.
np.random.seed(12345)
x = 2 * (np.random.random(500) - 0.5)
y = 2 * (np.random.random(500) - 0.5)
def f(x, y):
return np.exp(-(x + y ** 2))
z = f(x, y)
data = np.c_[x,y,z]
# regular grid covering the domain of the data
mn = np.min(data, axis=0)
mx = np.max(data, axis=0)
X,Y = np.meshgrid(np.linspace(mn[0], mx[0], 20), np.linspace(mn[1], mx[1], 20))
XX = X.flatten()
YY = Y.flatten()
# best-fit quadratic curve (2nd-order)
A = np.c_[np.ones(data.shape[0]), data[:,:2], np.prod(data[:,:2], axis=1), data[:,:2]**2]
C,_,_,_ = scipy.linalg.lstsq(A, data[:,2])
# evaluate it on a grid
Z = np.dot(np.c_[np.ones(XX.shape), XX, YY, XX*YY, XX**2, YY**2], C).reshape(X.shape)
# plot points and fitted surface using Matplotlib
fig = plt.figure(figsize=(10, 10))
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z, rstride=1, cstride=1, alpha=0.2)
ax.scatter(data[:,0], data[:,1], data[:,2], c='r', s=50)
plt.xlabel('X')
plt.ylabel('Y')
ax.set_zlabel('Z')
ax.axis('equal')
ax.axis('tight')

Changing the linewidth and the color simultaneously in matplotlib

The figure above is a great artwork showing the wind speed, wind direction and temperature simultaneously. detailedly:
The X axes represent the date
The Y axes shows the wind direction(Southern, western, etc)
The variant widths of the line were stand for the wind speed through timeseries
The variant colors of the line were stand for the atmospheric temperature
This simple figure visualized 3 different attribute without redundancy.
So, I really want to reproduce similar plot in matplotlib.
My attempt now
## Reference 1 http://stackoverflow.com/questions/19390895/matplotlib-plot-with-variable-line-width
## Reference 2 http://stackoverflow.com/questions/17240694/python-how-to-plot-one-line-in-different-colors
def plot_colourline(x,y,c):
c = plt.cm.jet((c-np.min(c))/(np.max(c)-np.min(c)))
lwidths=1+x[:-1]
ax = plt.gca()
for i in np.arange(len(x)-1):
ax.plot([x[i],x[i+1]], [y[i],y[i+1]], c=c[i],linewidth = lwidths[i])# = lwidths[i])
return
x=np.linspace(0,4*math.pi,100)
y=np.cos(x)
lwidths=1+x[:-1]
fig = plt.figure(1, figsize=(5,5))
ax = fig.add_subplot(111)
plot_colourline(x,y,prop)
ax.set_xlim(0,4*math.pi)
ax.set_ylim(-1.1,1.1)
Does someone has a more interested way to achieve this? Any advice would be appreciate!

Using as inspiration another question.
One option would be to use fill_between. But perhaps not in the way it was intended. Instead of using it to create your line, use it to mask everything that is not the line. Under it you can have a pcolormesh or contourf (for example) to map color any way you want.
Look, for instance, at this example:
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import interp1d
def windline(x,y,deviation,color):
y1 = y-deviation/2
y2 = y+deviation/2
tol = (y2.max()-y1.min())*0.05
X, Y = np.meshgrid(np.linspace(x.min(), x.max(), 100), np.linspace(y1.min()-tol, y2.max()+tol, 100))
Z = X.copy()
for i in range(Z.shape[0]):
Z[i,:] = c
#plt.pcolormesh(X, Y, Z)
plt.contourf(X, Y, Z, cmap='seismic')
plt.fill_between(x, y2, y2=np.ones(x.shape)*(y2.max()+tol), color='w')
plt.fill_between(x, np.ones(x.shape) * (y1.min() - tol), y2=y1, color='w')
plt.xlim(x.min(), x.max())
plt.ylim(y1.min()-tol, y2.max()+tol)
plt.show()
x = np.arange(100)
yo = np.random.randint(20, 60, 21)
y = interp1d(np.arange(0, 101, 5), yo, kind='cubic')(x)
dv = np.random.randint(2, 10, 21)
d = interp1d(np.arange(0, 101, 5), dv, kind='cubic')(x)
co = np.random.randint(20, 60, 21)
c = interp1d(np.arange(0, 101, 5), co, kind='cubic')(x)
windline(x, y, d, c)
, which results in this:
The function windline accepts as arguments numpy arrays with x, y , a deviation (like a thickness value per x value), and color array for color mapping. I think it can be greatly improved by messing around with other details but the principle, although not perfect, should be solid.

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.collections import LineCollection
x = np.linspace(0,4*np.pi,10000) # x data
y = np.cos(x) # y data
r = np.piecewise(x, [x < 2*np.pi, x >= 2*np.pi], [lambda x: 1-x/(2*np.pi), 0]) # red
g = np.piecewise(x, [x < 2*np.pi, x >= 2*np.pi], [lambda x: x/(2*np.pi), lambda x: -x/(2*np.pi)+2]) # green
b = np.piecewise(x, [x < 2*np.pi, x >= 2*np.pi], [0, lambda x: x/(2*np.pi)-1]) # blue
a = np.ones(10000) # alpha
w = x # width
fig, ax = plt.subplots(2)
ax[0].plot(x, r, color='r')
ax[0].plot(x, g, color='g')
ax[0].plot(x, b, color='b')
# mysterious parts
points = np.array([x, y]).T.reshape(-1, 1, 2)
segments = np.concatenate([points[:-1], points[1:]], axis=1)
# mysterious parts
rgba = list(zip(r,g,b,a))
lc = LineCollection(segments, linewidths=w, colors=rgba)
ax[1].add_collection(lc)
ax[1].set_xlim(0,4*np.pi)
ax[1].set_ylim(-1.1,1.1)
fig.show()
I notice this is what I suffered.

Contourf on the faces of a Matplotlib cube

I am trying to 'paint' the faces of a cube with a contourf function using Python Matplotlib. Is this possible?
This is similar idea to what was done here but obviously I cannot use patches. Similarly, I don't think I can use add_collection3d like this as it only supports PolyCollection, LineColleciton and PatchCollection.
I have been trying to use contourf on a fig.gca(projection='3d'). Toy example below.
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
plt.close('all')
fig = plt.figure()
ax = fig.gca(projection='3d')
############################################
# plotting the 'top' layer works okay... #
############################################
X = np.linspace(-5, 5, 43)
Y = np.linspace(-5, 5, 28)
X, Y = np.meshgrid(X, Y)
varone=np.random.rand(75,28,43)
Z=varone[0,:,:]
cset = ax.contourf(X, Y, Z, zdir='z', offset=1,
levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
#see [1]
plt.show()
#################################################
# but now trying to plot a vertical slice.... #
#################################################
plt.close('all')
fig = plt.figure()
ax = fig.gca(projection='3d')
Z=varone[::-1,:,-1]
X = np.linspace(-5, 5, 28)
Y = np.linspace(-5, 5, 75)
X, Y = np.meshgrid(X, Y)
#this 'projection' doesn't result in what I want, I really just want to rotate it
cset = ax.contourf(X, Y, Z, offset=5,zdir='x',
levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
#here's what it should look like....
ax=fig.add_subplot(1, 2,1)
cs1=ax.contourf(X,Y,Z,levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
#see [2]
plt.show()
1 From the example, the top surface comes easily:
2 But I'm not sure how to do the sides. Left side of this plot is what the section should look like (but rotated)...
Open to other python approaches. The data I'm actually plotting are geophysical netcdf files.

You have to assign the data to the right axis. The zig-zag results from the fact that now you are at x = const and have your oscillation in the z-direction (from the random data, which is generated between 0 and 1).
If you you assign the matrixes differently in your example, you end up with the desired result:
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
plt.close('all')
fig = plt.figure()
ax = fig.gca(projection='3d')
X = np.linspace(-5, 5, 43)
Y = np.linspace(-5, 5, 28)
X, Y = np.meshgrid(X, Y)
varone=np.random.rand(75,28,43) * 5.0 - 10.0
Z=varone[0,:,:]
cset = [[],[],[]]
# this is the example that worked for you:
cset[0] = ax.contourf(X, Y, Z, zdir='z', offset=5,
levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
# now, for the x-constant face, assign the contour to the x-plot-variable:
cset[1] = ax.contourf(Z, Y, X, zdir='x', offset=5,
levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
# likewise, for the y-constant face, assign the contour to the y-plot-variable:
cset[2] = ax.contourf(X, Z, Y, zdir='y', offset=-5,
levels=np.linspace(np.min(Z),np.max(Z),30),cmap='jet')
# setting 3D-axis-limits:
ax.set_xlim3d(-5,5)
ax.set_ylim3d(-5,5)
ax.set_zlim3d(-5,5)
plt.show()
The result looks like this:

The answer given below is not fully satisfying. Indeed, planes in x, y and z direction reproduce the same field.
Hereafter, a function that allows to represent the correct field in each of the planes.
import numpy as np
import matplotlib.pyplot as plt
def plot_cube_faces(arr, ax):
"""
External faces representation of a 3D array with matplotlib
Parameters
----------
arr: numpy.ndarray()
3D array to handle
ax: Axes3D object
Axis to work with
"""
x0 = np.arange(arr.shape[0])
y0 = np.arange(arr.shape[1])
z0 = np.arange(arr.shape[2])
x, y, z = np.meshgrid(x0, y0, z0)
xmax, ymax, zmax = max(x0), max(y0), max(z0)
vmin, vmax = np.min(arr), np.max(arr)
ax.contourf(x[:, :, 0], y[:, :, 0], arr[:, :, -1].T,
zdir='z', offset=zmax, vmin=vmin, vmax=vmax)
ax.contourf(x[0, :, :].T, arr[:, 0, :].T, z[0, :, :].T,
zdir='y', offset=0, vmin=vmin, vmax=vmax)
ax.contourf(arr[-1, :, :].T, y[:, 0, :].T, z[:, 0, :].T,
zdir='x', offset=xmax, vmin=vmin, vmax=vmax)
x0 = np.arange(30)
y0 = np.arange(20)
z0 = np.arange(10)
x, y, z = np.meshgrid(x0, y0, z0)
arr = (x + y + z) // 10
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
plot_cube_faces(arr, ax)
plt.show()

How do you create a 3D surface plot with missing values matplotlib?

I am trying to create a 3D surface energy diagram where an x,y position on a grid contains an associated z level. The issue is that the grid is not uniform (ie, there is not a z component for every x,y position). Is there a way to refrain from plotting those values by calling them NaN in the corresponding position in the array?
Here is what I have tried so far:
import numpy as np
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pylab
from matplotlib import cm
#Z levels
energ = np.array([0,3.5,1,-0.3,-1.5,-2,-3.4,-4.8])
#function for getting x,y associated z values?
def fun(x,y,array):
return array[x]
#arrays for grid
x = np.arange(0,7,0.5)
y = np.arange(0,7,0.5)
#create grid
X, Y = np.meshgrid(x,y)
zs = np.array([fun(x,y,energ) for x in zip(np.ravel(X))])
Z = zs.reshape(X.shape)
plt3d = plt.figure().gca(projection='3d')
#gradients now with respect to x and y, but ideally with respect to z only
Gx, Gz = np.gradient(X * Y)
G = (Gx ** 2 + Gz ** 2) ** .5 # gradient magnitude
N = G / G.max() # normalize 0..1
plt3d.plot_surface(X, Y, Z, rstride=1, cstride=1,
facecolors=cm.jet(N), edgecolor='k', linewidth=0, antialiased=False, shade=False)
plt.show()
I cannot post image here of this plot but if you run the code you will see it
But I would like to not plot certain x,y pairs, so the figure should triangle downward to the minimum. Can this be accomplished by using nan values? Also would like spacing between each level, to be connected by lines.
n = np.NAN
#energ represents the z levels, so the overall figure should look like a triangle.
energ = np.array([[0,0,0,0,0,0,0,0,0,0,0,0,0],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,2.6,n,2.97,n,2.6,n,2.97,n,2.6,n,3.58,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,1.09,n,1.23,n,1.09,n,1.23,n,1.7,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,-0.65,n,-0.28,n,-0.65,n,0.33,n,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,-2.16,n,-2.02,n,-1.55,n,n,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,n,-3.9,n,-2.92,n,n,n,n,n,],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,n,n,-4.8,n,n,n,n,n,n,]])
plt3d = plt.figure().gca(projection='3d')
Gx, Gz = np.gradient(X * energ) # gradients with respect to x and z
G = (Gx ** 2 + Gz ** 2) ** .5 # gradient magnitude
N = G / G.max() # normalize 0..1
x = np.arange(0,13,1)
y = np.arange(0,13,1)
X, Y = np.meshgrid(x,y)
#but the shapes don't seem to match up
plt3d.plot_surface(X, Y, energ, rstride=1, cstride=1,
facecolors=cm.jet(N), edgecolor='k',
linewidth=0, antialiased=False, shade=False
)
Using masked arrays generates the following error: local Python[7155] : void CGPathCloseSubpath(CGMutablePathRef): no current point.
n = np.NAN
energ = np.array([[0,0,0,0,0,0,0,0,0,0,0,0,0],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,2.6,n,2.97,n,2.6,n,2.97,n,2.6,n,3.58,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,1.09,n,1.23,n,1.09,n,1.23,n,1.7,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,-0.65,n,-0.28,n,-0.65,n,0.33,n,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,-2.16,n,-2.02,n,-1.55,n,n,n,n],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,n,-3.9,n,-2.92,n,n,n,n,n,],[n,n,n,n,n,n,n,n,n,n,n,n,n],[n,n,n,n,n,n,-4.8,n,n,n,n,n,n,]])
x = np.arange(0,13,1)
y = np.arange(0,13,1)
X, Y = np.meshgrid(x,y)
#create masked arrays
mX = ma.masked_array(X, mask=[[0,0,0,0,0,0,0,0,0,0,0,0,0],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,0,1,0,1,0,1,0,1,0,1,0,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,0,1,0,1,0,1,0,1,0,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,0,1,0,1,0,1,0,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,0,1,0,1,0,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,0,1,0,1,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,1,0,1,1,1,1,1,1]])
mY = ma.masked_array(Y, mask=[[0,0,0,0,0,0,0,0,0,0,0,0,0],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,0,1,0,1,0,1,0,1,0,1,0,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,0,1,0,1,0,1,0,1,0,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,0,1,0,1,0,1,0,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,0,1,0,1,0,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,0,1,0,1,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,1,0,1,1,1,1,1,1]])
m_energ = ma.masked_array(energ, mask=[[0,0,0,0,0,0,0,0,0,0,0,0,0],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,0,1,0,1,0,1,0,1,0,1,0,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,0,1,0,1,0,1,0,1,0,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,0,1,0,1,0,1,0,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,0,1,0,1,0,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,0,1,0,1,1,1,1,1],[1,1,1,1,1,1,1,1,1,1,1,1,1],[1,1,1,1,1,1,0,1,1,1,1,1,1]])
plt3d = plt.figure().gca(projection='3d')
plt3d.plot_surface(mX, mY, m_energ, rstride=1, cstride=1, edgecolor='k', linewidth=0, antialiased=False, shade=False)
plt.show()

I was playing around with the code from this forum post, and I was able to make the graph have missing values. You can try the code yourself! I got it to work using float("nan") for the missing values.
import plotly.graph_objects as go
import numpy as np
x = np.arange(0.1,1.1,0.1)
y = np.linspace(-np.pi,np.pi,10)
#print(x)
#print(y)
X,Y = np.meshgrid(x,y)
#print(X)
#print(Y)
result = []
for i,j in zip(X,Y):
result.append(np.log(i)+np.sin(j))
result[0][0] = float("nan")
upper_bound = np.array(result)+1
lower_bound = np.array(result)-1
fig = go.Figure(data=[
go.Surface(z=result),
go.Surface(z=upper_bound, showscale=False, opacity=0.3,colorscale='purp'),
go.Surface(z=lower_bound, showscale=False, opacity=0.3,colorscale='purp')])
fig.show()