Develop Reference

Griddata and Contourf produce artifacts with increasing steps/levels

I am using SciPy Griddata to interpolate data in its Cartesian form and then plot these data using contourf with a polar projection. When the Cartesian interpolated data is plotted with contourf there are no artifacts. However, when the projection is polar, artifacts develop with increasing "levels".
The artifacts are polygons or rays that form near regions of steep gradients. The code below plots the brightness of the sky with the moon. With graphlevels of "12" there isn't an issue. Artifacts develop with graphlevel of "25." My desired level is 80 or more - which shows terrible artifacts. The below is example real data from one night. These artifacts always occur. See images with Levels = 12 and Levels = 80
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import griddata
gridsize =150
graphlevels =12
plt.figure(figsize=(12,10))
ax = plt.subplot(111,projection='polar')
x = [72.90,68.00,59.14,44.38,29.63,63.94,59.68,51.92,38.98,26.03,47.34,44.20,38.46,28.89,19.31,23.40,20.40,15.34,10.28,-0.18,-0.14,-0.09,-0.04,0.02,-25.39,-23.66,-20.57,-15.40,-10.23,-47.56,-44.34,-38.54,-28.89,-19.22,-64.01,-59.68,-51.89,-38.90,-25.90,-72.77,-67.84,-58.98,-44.21,-29.44,-72.75,-67.83,-58.96,-44.18,-29.41,-59.63,-51.82,-38.83,-25.84,-47.42,-44.20,-38.40,-28.76,-19.12,-23.40,-20.32,-15.19,-10.08,0.27,0.25,0.23,0.20,23.92,20.80,15.63,10.46,47.93,44.67,38.86,29.17,19.48,64.40,60.03,52.20,39.18,26.15,73.08,68.12,59.26,44.47,29.68,-4.81]
y = [12.93,12.01,10.38,7.67,4.99,37.03,34.49,29.93,22.33,14.77,56.60,52.75,45.82,34.26,22.72,64.60,56.14,42.02,27.90,73.66,68.67,59.68,44.68,29.68,69.12,64.45,56.00,41.92,27.84,56.26,52.45,45.56,34.08,22.61,36.59,34.11,29.61,22.11,14.62,12.48,11.62,10.04,7.43,4.83,-13.33,-12.31,-10.78,-8.21,-5.58,-34.84,-30.36,-22.87,-15.36,-57.04,-53.20,-46.31,-34.83,-23.34,-65.20,-56.72,-42.62,-28.53,-69.33,-60.31,-45.31,-30.31,-65.09,-56.63,-42.55,-28.47,-56.81,-52.99,-46.13,-34.69,-23.23,-36.99,-34.53,-30.08,-22.66,-15.22,-12.73,-11.93,-10.44,-7.94,-5.40,-1.22,]
skybrightness = [19.26,19.31,19.21,19.65,19.40,19.26,19.23,19.43,19.57,19.52,19.19,19.31,19.33,19.68,19.50,19.29,19.45,19.50,19.23,18.98,19.28,19.46,19.54,19.22,19.03,19.18,19.35,19.37,19.08,18.99,18.98,19.26,19.36,19.08,18.79,18.85,19.13,19.17,19.05,18.51,18.64,18.88,18.92,18.93,18.12,18.34,18.72,18.82,18.74,18.22,18.46,18.76,18.26,18.13,18.24,18.46,18.58,17.30,18.38,18.08,18.24,17.68,18.34,18.46,18.65,18.23,18.70,18.52,18.79,18.83,18.18,18.51,19.01,19.08,19.08,18.99,19.02,19.07,19.20,19.27,19.06,19.01,19.28,19.46,19.30,18.94]
xgrid = np.linspace(min(x), max(x),gridsize)
ygrid = np.linspace(min(y), max(y),gridsize)
xgrid, ygrid = np.meshgrid(xgrid, ygrid, indexing='ij')
nsb_grid = griddata((x,y),skybrightness,(xgrid, ygrid), method='linear')
r = np.sqrt(xgrid**2 + ygrid**2)
theta = np.arctan2(ygrid, xgrid)
plt.rc('ytick', labelsize=16)
ax.set_facecolor('#eeddcc')
colors = plt.cm.get_cmap('RdYlBu')
levels,steps = np.linspace(min(skybrightness), max(skybrightness)+0.3,graphlevels, retstep=True)
ticks = np.linspace(min(skybrightness), max(skybrightness)+0.3,12)
cax = ax.contourf(theta, r, nsb_grid, levels=levels, cmap=colors)
cbar = plt.colorbar(cax, fraction=0.046, pad=0.04, ticks=ticks)
cbar.set_label(r'mag/arcsec$^2$')
ax.set_theta_zero_location('N')
ax.set_theta_direction(-1)
ax.set_rmax(75)
ax.set_yticks(range(10, 80, 20))
ax.set_xticklabels([r'N', r'NE', r'E', r'SE', r'S', r'SW', r'W', r'NW'])
ax.grid(alpha=0.3)
plt.savefig('StackOverflowHELP.png')

I am going to leave my question and this answer on StackOverflow... because I did get an answer from the developers of Matploblib. The problem is Contourf . In its attempt to project data in polar dimensions there are overlaps and extensions of polygons at the cyclic boundaries that cause problems. The only way to avoid this is to add points at the boundary. To quote the developer:
The workaround is a lot of effort and has to be tuned to each particular problem, so is a very long way from being ideal. We (Matplotlib) should do better in these situations. Inserting extra points into the triangulation isn't the right approach, we should instead correct the lines/polygons that traverse the discontinuity to provide a general solution.
See https://github.com/matplotlib/matplotlib/issues/20060 for the full discussion
The answer I settled on is to interpolate and render the result in Cartesian space. Then I format an empty polar plot with axes and labels to overlay on the top... and get on with my life!
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import griddata
gridsize =150
graphlevels = 200
fig = plt.figure(figsize=(12,10))
ax = fig.add_subplot(111, aspect='equal')
pax = fig.add_subplot(111,projection='polar')
pax.set_facecolor('none')
ax.set_axis_off()
ax.set_xlim([-75,75])
ax.set_ylim([-75,75])
x = [72.90,68.00,59.14,44.38,29.63,63.94,59.68,51.92,38.98,26.03,47.34,44.20,38.46,28.89,19.31,23.40,20.40,15.34,10.28,-0.18,-0.14,-0.09,-0.04,0.02,-25.39,-23.66,-20.57,-15.40,-10.23,-47.56,-44.34,-38.54,-28.89,-19.22,-64.01,-59.68,-51.89,-38.90,-25.90,-72.77,-67.84,-58.98,-44.21,-29.44,-72.75,-67.83,-58.96,-44.18,-29.41,-59.63,-51.82,-38.83,-25.84,-47.42,-44.20,-38.40,-28.76,-19.12,-23.40,-20.32,-15.19,-10.08,0.27,0.25,0.23,0.20,23.92,20.80,15.63,10.46,47.93,44.67,38.86,29.17,19.48,64.40,60.03,52.20,39.18,26.15,73.08,68.12,59.26,44.47,29.68,-4.81]
y = [12.93,12.01,10.38,7.67,4.99,37.03,34.49,29.93,22.33,14.77,56.60,52.75,45.82,34.26,22.72,64.60,56.14,42.02,27.90,73.66,68.67,59.68,44.68,29.68,69.12,64.45,56.00,41.92,27.84,56.26,52.45,45.56,34.08,22.61,36.59,34.11,29.61,22.11,14.62,12.48,11.62,10.04,7.43,4.83,-13.33,-12.31,-10.78,-8.21,-5.58,-34.84,-30.36,-22.87,-15.36,-57.04,-53.20,-46.31,-34.83,-23.34,-65.20,-56.72,-42.62,-28.53,-69.33,-60.31,-45.31,-30.31,-65.09,-56.63,-42.55,-28.47,-56.81,-52.99,-46.13,-34.69,-23.23,-36.99,-34.53,-30.08,-22.66,-15.22,-12.73,-11.93,-10.44,-7.94,-5.40,-1.22,]
skybrightness = [19.26,19.31,19.21,19.65,19.40,19.26,19.23,19.43,19.57,19.52,19.19,19.31,19.33,19.68,19.50,19.29,19.45,19.50,19.23,18.98,19.28,19.46,19.54,19.22,19.03,19.18,19.35,19.37,19.08,18.99,18.98,19.26,19.36,19.08,18.79,18.85,19.13,19.17,19.05,18.51,18.64,18.88,18.92,18.93,18.12,18.34,18.72,18.82,18.74,18.22,18.46,18.76,18.26,18.13,18.24,18.46,18.58,17.30,18.38,18.08,18.24,17.68,18.34,18.46,18.65,18.23,18.70,18.52,18.79,18.83,18.18,18.51,19.01,19.08,19.08,18.99,19.02,19.07,19.20,19.27,19.06,19.01,19.28,19.46,19.30,18.94]
xgrid = np.linspace(min(x), max(x),gridsize)
ygrid = np.linspace(min(y), max(y),gridsize)
xgrid, ygrid = np.meshgrid(xgrid, ygrid, indexing='ij')
nsb_grid = griddata((x,y),skybrightness,(xgrid, ygrid), method='linear')
plt.rc('ytick', labelsize=16) #colorbar font
colors = plt.cm.get_cmap('RdYlBu')
levels,steps = np.linspace(min(skybrightness), max(skybrightness)+0.3,graphlevels, retstep=True)
ticks = np.linspace(min(skybrightness), max(skybrightness)+0.3,12)
cax = ax.contourf(xgrid, ygrid, nsb_grid, levels=levels, cmap=colors)
cbar = plt.colorbar(cax, fraction=0.046, pad=0.04, ticks=ticks)
cbar.set_label(r'mag/arcsec$^2$')
pax.set_theta_zero_location('N')
pax.set_theta_direction(-1)
pax.set_rmax(75)
pax.set_yticks(range(10, 80, 20))
pax.set_xticklabels([r'N', r'NE', r'E', r'SE', r'S', r'SW', r'W', r'NW'])
pax.grid(alpha=0.3)

Make a heatmap of x,y,z data in Python

I am trying to draw a heatmap in Python.
I have studied several tutorials but still can't achieve what I need.
My data has 3 columns: X, Y (coordinates in the scatterplot) and cluster (a group/cluster each row is placed). The desired output should look like that (6 clusters and X,Y points distributed in the coloured areas):
My current code:
# libraries
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import kde
from scipy.interpolate import griddata
# Get the data (csv file is hosted on the web)
url = 'https://raw.githubusercontent.com/ampil/sandbox/master/latest-sales-sample.csv'
df = pd.read_csv(url, sep = ';')
df = df.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)
# create data
x = df['X']
y = np.log(df['Y'])
z = df['cluster']
# target grid to interpolate to
xi = yi = np.arange(0, 1.01, 0.01)
xi, yi = np.meshgrid(xi,yi)
# interpolate
zi = griddata((x,y),z,(xi,yi),method='cubic')
# plot
fig = plt.figure()
ax = fig.add_subplot(111)
ax.axis((x.min(), x.max(), y.min(), y.max()))
plt.contourf(xi, yi, zi, np.arange(0, 1.01, 0.01), cmap='coolwarm')
plt.plot(x,y,'k.')
plt.xlabel('x',fontsize=16)
plt.ylabel('y',fontsize=16)
plt.show()
plt.close(fig)
gives me
Later on, I plan to publish the graph via dash.
Any help is appreciated!

My answer is a small edit to answer provided by warped.
The difference is the inclusion of the parameter 'extend' in the contourf method call.
https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.contourf.html
For some more information about the minimum/maximum color-map behaviour, see:
https://matplotlib.org/3.1.1/gallery/images_contours_and_fields/contourf_demo.html#sphx-glr-gallery-images-contours-and-fields-contourf-demo-py
# libraries
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import kde
from scipy.interpolate import griddata
# Get the data (csv file is hosted on the web)
url = 'https://raw.githubusercontent.com/ampil/sandbox/master/latest-sales-sample.csv'
df = pd.read_csv(url, sep = ';')
df = df.dropna(axis=0, how='any', thresh=None, subset=None, inplace=False)
# create data
x = df['X']
y = np.log(df['Y'])
z = df['cluster']
#following 2 lines provided by user-warped
xi = np.arange(0, np.max(x), 0.1)
yi = np.arange(0, np.max(y), 0.1)
xi, yi = np.meshgrid(xi,yi)
# interpolate
zi = griddata((x,y),z,(xi,yi),method='cubic')
#define color map...which you can choose to modify with 'set_under' and 'set_over'
#as per: https://matplotlib.org/3.1.1/gallery/images_contours_and_fields/contourf_demo.html#sphx-glr-gallery-images-contours-and-fields-contourf-demo-py
cmap = plt.cm.get_cmap("coolwarm")
fig = plt.figure()
ax = fig.add_subplot(111)
ax.axis((x.min(), x.max(), y.min(), y.max()))
#added the 'extend' parameter to user:warped edit as per documentation of plt.contourf
#https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.contourf.html
plt.contourf(xi, yi, zi, levels=[1,2,3,4,5,6], cmap=cmap, extend='both')
plt.plot(x, y,'k.')
plt.xlabel('x',fontsize=16)
plt.ylabel('y',fontsize=16)
plt.show()
As far as extending the colours beyond what they are now...you'd get a result that would look very different from the main area and would likely have little meaning. If this were a GIS application, I would have those exterior pixels to be "NODATA".
EDIT:
Providing evidence that filling the exterior would look strange...
Using a canned gdal method to fill nodatas, this is what it would look like:
This was quick and dirty and other methods likely exist but would probably look equally odd. Perhaps numpy.nan_to_num is another solution if you don't have gdal.
In case you're curious...here's the code (continues from previous code block):
import gdal
ds = gdal.GetDriverByName('MEM').Create('', zi.shape[1], zi.shape[0], 1, gdal.GDT_Float32)
in_band = ds.GetRasterBand(1)
in_band.SetNoDataValue(-9999)
in_band.FlushCache()
raster_data = np.copy(zi)
raster_data[np.isnan(zi)] = -9999
in_band.WriteArray(raster_data)
#this line takes a while to run...grab a coffee
result = gdal.FillNodata(in_band, None, maxSearchDist=20000, smoothingIterations=0)
in_band.FlushCache()
newz = in_band.ReadAsArray()
fig = plt.figure()
ax = fig.add_subplot(111)
ax.axis((x.min(), x.max(), y.min(), y.max()))
#added the 'extend' parameter as per documentation of plt.contourf
#https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.contourf.html
plt.contourf(xi, yi, newz, levels=[1,2,3,4,5,6], cmap=cmap, extend='both')
plt.plot(x, y,'k.')
plt.xlabel('x',fontsize=16)
plt.ylabel('y',fontsize=16)
plt.show()

import matplotlib
colors = ['red','green','blue','purple','black', 'coral']
fig = plt.figure(figsize=(8,8))
plt.scatter(x, y, c = z, cmap=matplotlib.colors.ListedColormap(colors))
plt.show()

looking at df.describe():
id Y X cluster
count 706.000000 706.000000 706.000000 706.000000
mean 357.035411 18401.784703 3217.385269 3.002833
std 205.912934 46147.403750 950.665697 0.532616
min 1.000000 278.000000 328.000000 1.000000
25% 178.500000 3546.000000 2498.500000 3.000000
50% 358.500000 6869.500000 3574.000000 3.000000
75% 534.750000 17169.000000 3997.500000 3.000000
max 712.000000 877392.000000 4321.000000 6.000000
X is between 328 and 4321, Y is between 278 and 887392.
your lines
xi = yi = np.arange(0, 1.01, 0.01)
xi, yi = np.meshgrid(xi,yi)
create a grid with x,y values between zero and one.
So, you are trying to interpolate using data that is very far away from your grid.
Setting
xi = np.arange(0, np.max(x), 0.1)
yi = np.arange(0, np.max(y), 0.1)
xi, yi = np.meshgrid(xi,yi)
and leaving out the np.arange...in plt.contour:
fig = plt.figure()
ax = fig.add_subplot(111)
ax.axis((x.min(), x.max(), y.min(), y.max()))
plt.contourf(xi, yi, zi, cmap='coolwarm') # <-- removed np.arange()
plt.plot(x,y,'k.')
plt.xlabel('x',fontsize=16)
plt.ylabel('y',fontsize=16)
plt.show()
using levels to draw contours:
plt.contourf(xi, yi, zi, levels=[1,2,3,4,5,6], cmap='coolwarm')

Get coordinates from points density with KDE

I have a set of points (> 1k) in this form:
y,x
173.549,308.176
173.549,313.328
213.26,419.588
Using KDE, i can plot points density with pcolormesh and contourf. This is an example result, plotting points too:
This is the code i used to have the plot:
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from scipy.stats.kde import gaussian_kde
x, y = np.genfromtxt('terzinoSX.csv', delimiter=',', unpack=True)
y = y[np.logical_not(np.isnan(y))]
x = x[np.logical_not(np.isnan(x))]
k = gaussian_kde(np.vstack([x, y]))
xi, yi = np.mgrid[x.min():x.max():x.size**0.5*1j,y.min():y.max():y.size**0.5*1j]
zi = k(np.vstack([xi.flatten(), yi.flatten()]))
fig = plt.figure(figsize=(7,4))
ax2 = fig.add_subplot(111)
#alpha=0.5 will make the plots semitransparent
#ax1.pcolormesh(yi, xi, zi.reshape(xi.shape), alpha=0.5)
ax2.contourf(yi, xi, zi.reshape(xi.shape), alpha=0.5)
plt.axis('off')
ax2.plot(y,x, "o")
ax2.set_xlim(0, 740)
ax2.set_ylim(515, 0)
#overlay soccer field
im = plt.imread('statszone_football_pitch.png')
ax2.imshow(im, extent=[0, 740, 0, 515], aspect='auto')
fig.savefig('test.png', bbox_inches='tight')
I would like to have one point representing coordinates of most populated zone (middle point for example), like a middle point over the "red" zone. Is it possible in some way?

I solved this by adding these lines that calculate the point in the most populated area:
xy = np.vstack([x,y])
kde = stats.gaussian_kde(xy)
density = kde(xy)
pts = xy.T[np.argmax(density)]

You can use np.argmax to get the coordinates of the maximum. For example:
kde = compute_my_kde() # Returns a two-dimensional array
y, x = np.argmax(kde) # x and y are swapped because matplotlib displays images as a matrix (first index is rows, second index is colums)
plt.imshow(kde) # Show the kde
plt.scatter(x, y) # Show the maximum point

Pcolormesh not getting correct position matplotlib

I'm plotting data on top of a png image with this code:
x, y = np.genfromtxt('pogba_t1314.csv', delimiter=',', unpack=True)
print(x[1], y[1])
y = y[np.logical_not(np.isnan(y))]
x = x[np.logical_not(np.isnan(x))]
k = gaussian_kde(np.vstack([x, y]))
xi, yi = np.mgrid[x.min():x.max():x.size**0.5*1j,y.min():y.max():y.size**0.5*1j]
zi = k(np.vstack([xi.flatten(), yi.flatten()]))
fig = plt.figure(figsize=(9,10))
ax1 = fig.add_subplot(211)
ax1.pcolormesh(xi, yi, zi.reshape(xi.shape), alpha=0.5)
ax1.plot(y,x, "o")
ax1.set_xlim(0, 740)
ax1.set_ylim(515, 0)
im = plt.imread('statszone_football_pitch.png')
ax1.imshow(im, extent=[0, 740, 0, 515], aspect='auto')
Here is the result:
Pcolormesh should be over the whole image, but it is not. How can i set the correct position?
Why i get smaller pcolormesh respect of whole pitch? I should have something like this (even if this refers to contourf):

Your code seems to work fine. The problem is that what represents your x-axis in the image is not what you giving as x-axis in the plot. That said if you change x to y (and vice-versa) you'll obtain the correct plot:
from scipy.stats.kde import gaussian_kde
import matplotlib.pyplot as plt
import numpy as np
x, y = np.genfromtxt('pogba_t1314.csv', delimiter=',', unpack=True)
print(x[1], y[1])
y = y[np.logical_not(np.isnan(y))]
x = x[np.logical_not(np.isnan(x))]
k = gaussian_kde(np.vstack([x, y]))
#xi, yi = np.mgrid[x.min():x.max():x.size**0.5*1j,y.min():y.max():y.size**0.5*1j]
xi, yi = np.mgrid[y.min():y.max():y.size**0.5*1j,x.min():x.max():x.size**0.5*1j] # Changed this.
#zi = k(np.vstack([xi.flatten(), yi.flatten()]))
zi = k(np.vstack([yi.flatten(),xi.flatten()])) # And this.
fig = plt.figure(figsize=(9,10))
ax1 = fig.add_subplot(111)
ax1.pcolormesh(xi, yi, zi.reshape(xi.shape), alpha=0.5)
ax1.plot(y,x, "o")
ax1.set_xlim(0, 740)
ax1.set_ylim(515, 0)
im = plt.imread('statszone_football_pitch.jpg')
ax1.imshow(im, extent=[0, 740, 0, 515], aspect='auto')
plt.show()
The result is this:
I guess it's a kind of density plot for your scattered points.

putting limits to x,y,z interpolated heatmap in matplotlib

I'm looking to plot a heatmap for which I have the value (=heatmap color) z at each couple of spatial x,y coordinates but I want to mark out the z values between [z0,z1] with z0=0.0 and z1=0.4 while some of interpolated z values are under and above those boundaries.
from numpy.random import uniform, seed
from matplotlib.mlab import griddata
import matplotlib.pyplot as plt
import numpy as np
# make up data.
#npts = int(raw_input('enter # of random points to plot:'))
seed(0)
npts = 200
x = uniform(-2, 2, npts)
y = uniform(-2, 2, npts)
z = x*np.exp(-x**2 - y**2)
# define grid.
xi = np.linspace(0, 1, 1000)
yi = np.linspace(0, 1, 1000)
# grid the data.
zi = griddata(x, y, z, xi, yi, interp='linear')
# contour the gridded data, plotting dots at the nonuniform data points.
CS = plt.contourf(xi, yi, zi, 15, cmap=plt.cm.rainbow,
vmax=abs(zi).max(), vmin=-abs(zi).max())
plt.colorbar() # draw colorbar
# plot data points.
plt.show()
I would like to restrict the colorbar and heatmap color from 0.0 to 0.4 (so avoid in the heatmap and in the colorbar valies under 0.0 and above 0.4).
How to do that? Thanks

You can set the values in a numpy array to None to leave them unplotted. For example,
zmin = 0.0
zmax = 0.4
zi[(zi<zmin) | (zi>zmax)] = None
CS = plt.contourf(xi, yi, zi, 15, cmap=plt.cm.rainbow,
vmax=zmax, vmin=zmin)