Develop Reference

Multivariate KDE Scipy Stats - what if it's not Gaussian?

I have some 2D data that I am smoothing using:
from scipy.stats import gaussian_kde
kde = gaussian_kde(data)
but what if my data isn't Gaussian/tophat/the other options? Mine looks more elliptical before smoothing, so should I really have a different bandwidth in x and then y? The variance in one direction is a lot higher, and also the values of the x axis are higher, so it feels like a simple Gaussian might miss something?

This is what I get with your defined X and Y. Seems good. Were you expecting something different?
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
def generate(n):
# generate data
np.random.seed(42)
x = np.random.normal(size=n, loc=1, scale=0.01)
np.random.seed(1)
y = np.random.normal(size=n, loc=200, scale=100)
return x, y
x, y = generate(100)
xmin = x.min()
xmax = x.max()
ymin = y.min()
ymax = y.max()
X, Y = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
positions = np.vstack([X.ravel(), Y.ravel()])
values = np.vstack([x, y])
kernel = stats.gaussian_kde(values)
Z = np.reshape(kernel(positions).T, X.shape)
fig, ax = plt.subplots(figsize=(7, 7))
ax.imshow(np.rot90(Z), cmap=plt.cm.gist_earth_r,
extent=[xmin, xmax, ymin, ymax],
aspect='auto', alpha=.75
)
ax.plot(x, y, 'ko', ms=5)
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
plt.show()
The distributions of x and y are Gaussian.
You can verify with seaborn too
import pandas as pd
import seaborn as sns
# I pass a DataFrame because passing
# (x,y) alone will be soon deprecated
g = sns.jointplot(data=pd.DataFrame({'x':x, 'y':y}), x='x', y='y')
g.plot_joint(sns.kdeplot, color="r", zorder=0, levels=6)
update
Kernel Density Estimate of 2-dimensional data is done separately along each axis and then join together.
Let's make an example with the dataset we already used.
As we can see in the seaborn jointplot, you have not only the estimated 2d-kde but also marginal distributions of x and y (the histograms).
So, step by step, let's estimate the density of x and y and then evaluate the density over a linearspace
kde_x = sps.gaussian_kde(x)
kde_x_space = np.linspace(x.min(), x.max(), 100)
kde_x_eval = kde_x.evaluate(kde_x_space)
kde_x_eval /= kde_x_eval.sum()
kde_y = sps.gaussian_kde(y)
kde_y_space = np.linspace(y.min(), y.max(), 100)
kde_y_eval = kde_y.evaluate(kde_y_space)
kde_y_eval /= kde_y_eval.sum()
fig, ax = plt.subplots(1, 2, figsize=(12, 4))
ax[0].plot(kde_x_space, kde_x_eval, 'k.')
ax[0].set(title='KDE of x')
ax[1].plot(kde_y_space, kde_y_eval, 'k.')
ax[1].set(title='KDE of y')
plt.show()
So we now have the marginal distributions of x and y. These are probability density functions so, the joint-probability of x and y can be seen as the intersection of independent events x and y, thus we can multiply the estimated probability density of x and y in a 2d-matrix and plot on 3d projection
# Grid of x and y
X, Y = np.meshgrid(kde_x_space, kde_y_space)
# Grid of probability density
kX, kY = np.meshgrid(kde_x_eval, kde_y_eval)
# Intersection
Z = kX * kY
fig, ax = plt.subplots(
2, 2,
subplot_kw={"projection": "3d"},
figsize=(10, 10))
for i, (elev, anim, title) in enumerate(zip([10, 10, 25, 25],
[0, -90, 25, -25],
['y axis', 'x axis', 'view 1', 'view 2']
)):
# Plot the surface.
surf = ax.flat[i].plot_surface(X, Y, Z, cmap=plt.cm.gist_earth_r,
linewidth=0, antialiased=False, alpha=.75)
ax.flat[i].scatter(x, y, zs=0, zdir='z', c='k')
ax.flat[i].set(
xlabel='x', ylabel='y',
title=title
)
ax.flat[i].view_init(elev=elev, azim=anim)
plt.show()
This is a very simple and naif method but only to have an idea on how it works and why x and y scales don't matter for a 2d-KDE.

How to plot 3D plots on top of each other and highlight their intersection in python

I have 2 dataframes that look like this:
import random
import numpy as np
import pandas as pd
A = pd.DataFrame({'x':[random.uniform(0, 1) for i in range(0,100)], 'y':[random.uniform(0, 1) for i in range(0,100)],
'z':[random.uniform(0, 1) for i in range(0,100)], 'w':[random.uniform(0, 1) for i in range(0,100)]})
B = pd.DataFrame({'x':[random.uniform(0, 1) for i in range(0,100)], 'y':[random.uniform(0, 1) for i in range(0,100)],
'z':[random.uniform(0, 1) for i in range(0,100)], 'w':[random.uniform(0, 1) for i in range(0,100)]})
From these two dataframes using this function:
import matplotlib.pyplot as plt
import scipy.stats as st
def plot_2d_kde(df):
# Extract x and y
x = df['x']
y = df['y']
# Define the borders
deltaX = (max(x) - min(x))/10
deltaY = (max(y) - min(y))/10
xmin = min(x) - deltaX
xmax = max(x) + deltaX
ymin = min(y) - deltaY
ymax = max(y) + deltaY
# Create meshgrid
xx, yy = np.mgrid[xmin:xmax:100j, ymin:ymax:100j]
# We will fit a gaussian kernel using the scipy’s gaussian_kde method
positions = np.vstack([xx.ravel(), yy.ravel()])
values = np.vstack([x, y])
kernel = st.gaussian_kde(values)
f = np.reshape(kernel(positions).T, xx.shape)
fig = plt.figure(figsize=(13, 7))
ax = plt.axes(projection='3d')
surf = ax.plot_surface(xx, yy, f, rstride=1, cstride=1, cmap='coolwarm', edgecolor='none')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('PDF')
ax.set_title('Surface plot of Gaussian 2D KDE')
fig.colorbar(surf, shrink=0.5, aspect=5) # add color bar indicating the PDF
ax.view_init(60, 35)
I can produce these two plots
plot_2d_kde(A)
plot_2d_kde(B)
Is there a way to plot these two plots together and highlight their intersection ? By intersection I mean the surface which is common for both plots

Does the fill_between function in matplotlib have a gradient feature? [duplicate]

I happened to see a beautiful graph on this page which is shown below:
Is it possible to get such color gradients in matplotlib?

There have been a handful of previous answers to similar questions (e.g. https://stackoverflow.com/a/22081678/325565), but they recommend a sub-optimal approach.
Most of the previous answers recommend plotting a white polygon over a pcolormesh fill. This is less than ideal for two reasons:
The background of the axes can't be transparent, as there's a filled polygon overlying it
pcolormesh is fairly slow to draw and isn't smoothly interpolated.
It's a touch more work, but there's a method that draws much faster and gives a better visual result: Set the clip path of an image plotted with imshow.
As an example:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
from matplotlib.patches import Polygon
np.random.seed(1977)
def main():
for _ in range(5):
gradient_fill(*generate_data(100))
plt.show()
def generate_data(num):
x = np.linspace(0, 100, num)
y = np.random.normal(0, 1, num).cumsum()
return x, y
def gradient_fill(x, y, fill_color=None, ax=None, **kwargs):
"""
Plot a line with a linear alpha gradient filled beneath it.
Parameters
----------
x, y : array-like
The data values of the line.
fill_color : a matplotlib color specifier (string, tuple) or None
The color for the fill. If None, the color of the line will be used.
ax : a matplotlib Axes instance
The axes to plot on. If None, the current pyplot axes will be used.
Additional arguments are passed on to matplotlib's ``plot`` function.
Returns
-------
line : a Line2D instance
The line plotted.
im : an AxesImage instance
The transparent gradient clipped to just the area beneath the curve.
"""
if ax is None:
ax = plt.gca()
line, = ax.plot(x, y, **kwargs)
if fill_color is None:
fill_color = line.get_color()
zorder = line.get_zorder()
alpha = line.get_alpha()
alpha = 1.0 if alpha is None else alpha
z = np.empty((100, 1, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
z[:,:,-1] = np.linspace(0, alpha, 100)[:,None]
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
im = ax.imshow(z, aspect='auto', extent=[xmin, xmax, ymin, ymax],
origin='lower', zorder=zorder)
xy = np.column_stack([x, y])
xy = np.vstack([[xmin, ymin], xy, [xmax, ymin], [xmin, ymin]])
clip_path = Polygon(xy, facecolor='none', edgecolor='none', closed=True)
ax.add_patch(clip_path)
im.set_clip_path(clip_path)
ax.autoscale(True)
return line, im
main()

Please note Joe Kington deserves the lion's share of the credit here; my sole contribution is zfunc.
His method opens to door to many gradient/blur/drop-shadow
effects. For example, to make the lines have an evenly blurred underside, you
could use PIL to build an alpha layer which is 1 near the line and 0 near the bottom edge.
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import matplotlib.patches as patches
from PIL import Image
from PIL import ImageDraw
from PIL import ImageFilter
np.random.seed(1977)
def demo_blur_underside():
for _ in range(5):
# gradient_fill(*generate_data(100), zfunc=None) # original
gradient_fill(*generate_data(100), zfunc=zfunc)
plt.show()
def generate_data(num):
x = np.linspace(0, 100, num)
y = np.random.normal(0, 1, num).cumsum()
return x, y
def zfunc(x, y, fill_color='k', alpha=1.0):
scale = 10
x = (x*scale).astype(int)
y = (y*scale).astype(int)
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
w, h = xmax-xmin, ymax-ymin
z = np.empty((h, w, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
# Build a z-alpha array which is 1 near the line and 0 at the bottom.
img = Image.new('L', (w, h), 0)
draw = ImageDraw.Draw(img)
xy = np.column_stack([x, y])
xy -= xmin, ymin
# Draw a blurred line using PIL
draw.line(list(map(tuple, xy)), fill=255, width=15)
img = img.filter(ImageFilter.GaussianBlur(radius=100))
# Convert the PIL image to an array
zalpha = np.asarray(img).astype(float)
zalpha *= alpha/zalpha.max()
# make the alphas melt to zero at the bottom
n = zalpha.shape[0] // 4
zalpha[:n] *= np.linspace(0, 1, n)[:, None]
z[:,:,-1] = zalpha
return z
def gradient_fill(x, y, fill_color=None, ax=None, zfunc=None, **kwargs):
if ax is None:
ax = plt.gca()
line, = ax.plot(x, y, **kwargs)
if fill_color is None:
fill_color = line.get_color()
zorder = line.get_zorder()
alpha = line.get_alpha()
alpha = 1.0 if alpha is None else alpha
if zfunc is None:
h, w = 100, 1
z = np.empty((h, w, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
z[:,:,-1] = np.linspace(0, alpha, h)[:,None]
else:
z = zfunc(x, y, fill_color=fill_color, alpha=alpha)
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
im = ax.imshow(z, aspect='auto', extent=[xmin, xmax, ymin, ymax],
origin='lower', zorder=zorder)
xy = np.column_stack([x, y])
xy = np.vstack([[xmin, ymin], xy, [xmax, ymin], [xmin, ymin]])
clip_path = patches.Polygon(xy, facecolor='none', edgecolor='none', closed=True)
ax.add_patch(clip_path)
im.set_clip_path(clip_path)
ax.autoscale(True)
return line, im
demo_blur_underside()
yields

I've tried something :
import matplotlib.pyplot as plt
import numpy as np
fig = plt.figure()
xData = range(100)
yData = range(100)
plt.plot(xData, yData)
NbData = len(xData)
MaxBL = [[MaxBL] * NbData for MaxBL in range(100)]
Max = [np.asarray(MaxBL[x]) for x in range(100)]
for x in range (50, 100):
plt.fill_between(xData, Max[x], yData, where=yData >Max[x], facecolor='red', alpha=0.02)
for x in range (0, 50):
plt.fill_between(xData, yData, Max[x], where=yData <Max[x], facecolor='green', alpha=0.02)
plt.fill_between([], [], [], facecolor='red', label="x > 50")
plt.fill_between([], [], [], facecolor='green', label="x < 50")
plt.legend(loc=4, fontsize=12)
plt.show()
fig.savefig('graph.png')
.. and the result:
Of course the gradient could go down to 0 by changing the range of feel_between function.

Summing overlapping bubbles with gradient in python

I want to plot a map of specific sites to interpret their effects on the surrounding city environment. To do this, I would like to plot the sites as bubbles, with a decreasing gradient towards the edge of the circle, and where the gradient of the overlapping circles is the sum.
As an example I've used this:
# libraries
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
# create data
x = np.random.rand(15)
y = x+np.random.rand(15)
z = x+np.random.rand(15)
z=z*z
# Change color with c and alpha. I map the color to the X axis value.
plt.scatter(x, y, s=1500, c=z, cmap="Blues", alpha=0.4, edgecolors="grey", linewidth=1)
# Add titles (main and on axis)
plt.xlabel("the X axis")
plt.ylabel("the Y axis")
plt.title("A colored bubble plot")
plt.show();
which produces:
However, the color of the circles does not decay, nor do they seem to sum the intended way.
Is there any smart way to do this, or could it possibly be easier with some kind of heatmap solution, or using grids and a decaying effect on adjacent tiles?

Here is an approach with densities placed at each x and y, enlarged by the z value.
Depending on the distance to each x,y position a quantity is added.
import matplotlib.pyplot as plt
import numpy as np
from numpy.linalg import norm # calculate the length of a vector
# import seaborn as sns
# create data
x = np.random.rand(15)
y = x+np.random.rand(15)
z = x+np.random.rand(15)
z=z*z
fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(12,5))
# Change color with c and alpha. I map the color to the X axis value.
ax1.scatter(x, y, s=1500, c=z, cmap="Blues", alpha=0.4, edgecolors="grey", linewidth=1)
ax1.set_xlabel("the X axis")
ax1.set_ylabel("the Y axis")
ax1.set_title("A colored bubble plot")
centers = np.dstack((x, y))[0]
xmin = min(x)-0.2
xmax = max(x)+0.2
ymin = min(y)-0.2
ymax = max(y)+0.2
zmin = min(z)
zmax = max(z)
xx, yy = np.meshgrid(np.linspace(xmin, xmax, 100),
np.linspace(ymin, ymax, 100))
xy = np.dstack((xx, yy))
zz = np.zeros_like(xx)
for ci, zi in zip(centers, z):
sigma = zi / zmax * 0.3
sigma2 = sigma ** 2
zz += np.exp(- norm(xy - ci, axis=-1) ** 2 / sigma2 / 2)
img = ax2.imshow(zz, extent=[xmin, xmax, ymin, ymax], origin='lower', aspect='auto', cmap='Blues')
#plt.colorbar(img, ax=ax2)
ax2.set_xlabel("the X axis")
ax2.set_ylabel("the Y axis")
ax2.set_title("Density depending on z")
plt.show()
The plot compares the two approaches using the same random data.

Is it possible to get color gradients under curve in matplotlib?

I happened to see a beautiful graph on this page which is shown below:
Is it possible to get such color gradients in matplotlib?

There have been a handful of previous answers to similar questions (e.g. https://stackoverflow.com/a/22081678/325565), but they recommend a sub-optimal approach.
Most of the previous answers recommend plotting a white polygon over a pcolormesh fill. This is less than ideal for two reasons:
The background of the axes can't be transparent, as there's a filled polygon overlying it
pcolormesh is fairly slow to draw and isn't smoothly interpolated.
It's a touch more work, but there's a method that draws much faster and gives a better visual result: Set the clip path of an image plotted with imshow.
As an example:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
from matplotlib.patches import Polygon
np.random.seed(1977)
def main():
for _ in range(5):
gradient_fill(*generate_data(100))
plt.show()
def generate_data(num):
x = np.linspace(0, 100, num)
y = np.random.normal(0, 1, num).cumsum()
return x, y
def gradient_fill(x, y, fill_color=None, ax=None, **kwargs):
"""
Plot a line with a linear alpha gradient filled beneath it.
Parameters
----------
x, y : array-like
The data values of the line.
fill_color : a matplotlib color specifier (string, tuple) or None
The color for the fill. If None, the color of the line will be used.
ax : a matplotlib Axes instance
The axes to plot on. If None, the current pyplot axes will be used.
Additional arguments are passed on to matplotlib's ``plot`` function.
Returns
-------
line : a Line2D instance
The line plotted.
im : an AxesImage instance
The transparent gradient clipped to just the area beneath the curve.
"""
if ax is None:
ax = plt.gca()
line, = ax.plot(x, y, **kwargs)
if fill_color is None:
fill_color = line.get_color()
zorder = line.get_zorder()
alpha = line.get_alpha()
alpha = 1.0 if alpha is None else alpha
z = np.empty((100, 1, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
z[:,:,-1] = np.linspace(0, alpha, 100)[:,None]
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
im = ax.imshow(z, aspect='auto', extent=[xmin, xmax, ymin, ymax],
origin='lower', zorder=zorder)
xy = np.column_stack([x, y])
xy = np.vstack([[xmin, ymin], xy, [xmax, ymin], [xmin, ymin]])
clip_path = Polygon(xy, facecolor='none', edgecolor='none', closed=True)
ax.add_patch(clip_path)
im.set_clip_path(clip_path)
ax.autoscale(True)
return line, im
main()

Please note Joe Kington deserves the lion's share of the credit here; my sole contribution is zfunc.
His method opens to door to many gradient/blur/drop-shadow
effects. For example, to make the lines have an evenly blurred underside, you
could use PIL to build an alpha layer which is 1 near the line and 0 near the bottom edge.
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import matplotlib.patches as patches
from PIL import Image
from PIL import ImageDraw
from PIL import ImageFilter
np.random.seed(1977)
def demo_blur_underside():
for _ in range(5):
# gradient_fill(*generate_data(100), zfunc=None) # original
gradient_fill(*generate_data(100), zfunc=zfunc)
plt.show()
def generate_data(num):
x = np.linspace(0, 100, num)
y = np.random.normal(0, 1, num).cumsum()
return x, y
def zfunc(x, y, fill_color='k', alpha=1.0):
scale = 10
x = (x*scale).astype(int)
y = (y*scale).astype(int)
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
w, h = xmax-xmin, ymax-ymin
z = np.empty((h, w, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
# Build a z-alpha array which is 1 near the line and 0 at the bottom.
img = Image.new('L', (w, h), 0)
draw = ImageDraw.Draw(img)
xy = np.column_stack([x, y])
xy -= xmin, ymin
# Draw a blurred line using PIL
draw.line(list(map(tuple, xy)), fill=255, width=15)
img = img.filter(ImageFilter.GaussianBlur(radius=100))
# Convert the PIL image to an array
zalpha = np.asarray(img).astype(float)
zalpha *= alpha/zalpha.max()
# make the alphas melt to zero at the bottom
n = zalpha.shape[0] // 4
zalpha[:n] *= np.linspace(0, 1, n)[:, None]
z[:,:,-1] = zalpha
return z
def gradient_fill(x, y, fill_color=None, ax=None, zfunc=None, **kwargs):
if ax is None:
ax = plt.gca()
line, = ax.plot(x, y, **kwargs)
if fill_color is None:
fill_color = line.get_color()
zorder = line.get_zorder()
alpha = line.get_alpha()
alpha = 1.0 if alpha is None else alpha
if zfunc is None:
h, w = 100, 1
z = np.empty((h, w, 4), dtype=float)
rgb = mcolors.colorConverter.to_rgb(fill_color)
z[:,:,:3] = rgb
z[:,:,-1] = np.linspace(0, alpha, h)[:,None]
else:
z = zfunc(x, y, fill_color=fill_color, alpha=alpha)
xmin, xmax, ymin, ymax = x.min(), x.max(), y.min(), y.max()
im = ax.imshow(z, aspect='auto', extent=[xmin, xmax, ymin, ymax],
origin='lower', zorder=zorder)
xy = np.column_stack([x, y])
xy = np.vstack([[xmin, ymin], xy, [xmax, ymin], [xmin, ymin]])
clip_path = patches.Polygon(xy, facecolor='none', edgecolor='none', closed=True)
ax.add_patch(clip_path)
im.set_clip_path(clip_path)
ax.autoscale(True)
return line, im
demo_blur_underside()
yields

I've tried something :
import matplotlib.pyplot as plt
import numpy as np
fig = plt.figure()
xData = range(100)
yData = range(100)
plt.plot(xData, yData)
NbData = len(xData)
MaxBL = [[MaxBL] * NbData for MaxBL in range(100)]
Max = [np.asarray(MaxBL[x]) for x in range(100)]
for x in range (50, 100):
plt.fill_between(xData, Max[x], yData, where=yData >Max[x], facecolor='red', alpha=0.02)
for x in range (0, 50):
plt.fill_between(xData, yData, Max[x], where=yData <Max[x], facecolor='green', alpha=0.02)
plt.fill_between([], [], [], facecolor='red', label="x > 50")
plt.fill_between([], [], [], facecolor='green', label="x < 50")
plt.legend(loc=4, fontsize=12)
plt.show()
fig.savefig('graph.png')
.. and the result:
Of course the gradient could go down to 0 by changing the range of feel_between function.