Develop Reference

How to combine two outputs of KernelDensity from sklearn (Python)

I want to multiply two Kernel Density Estimates to get a composite likelihood ... The multiplication operator doesn't exist for sklearn.
I have KDR for data from 3 independent sources and I want multiply KDE from each sources. The below code is from https://scikit-learn.org/stable/auto_examples/neighbors/plot_species_kde.html#sphx-glr-auto-examples-neighbors-plot-species-kde-py and should run easily.
Although I made some small modification to get get two KDEs and then multiply them which fails.
Someone can help. Below code should run without error except the multiplication part.
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import fetch_species_distributions
from sklearn.neighbors import KernelDensity
# if basemap is available, we'll use it.
# otherwise, we'll improvise later...
try:
from mpl_toolkits.basemap import Basemap
basemap = True
except ImportError:
basemap = False
def construct_grids(batch):
"""Construct the map grid from the batch object
Parameters
----------
batch : Batch object
The object returned by :func:`fetch_species_distributions`
Returns
-------
(xgrid, ygrid) : 1-D arrays
The grid corresponding to the values in batch.coverages
"""
# x,y coordinates for corner cells
xmin = batch.x_left_lower_corner + batch.grid_size
xmax = xmin + (batch.Nx * batch.grid_size)
ymin = batch.y_left_lower_corner + batch.grid_size
ymax = ymin + (batch.Ny * batch.grid_size)
# x coordinates of the grid cells
xgrid = np.arange(xmin, xmax, batch.grid_size)
# y coordinates of the grid cells
ygrid = np.arange(ymin, ymax, batch.grid_size)
return (xgrid, ygrid)
# Get matrices/arrays of species IDs and locations
data = fetch_species_distributions()
species_names = ['Bradypus Variegatus', 'Microryzomys Minutus']
Xtrain = np.vstack([data['train']['dd lat'], data['train']['dd long']]).T
ytrain = np.array([d.decode('ascii').startswith('micro')
for d in data['train']['species']], dtype='int')
Xtrain *= np.pi / 180. # Convert lat/long to radians
# Set up the data grid for the contour plot
xgrid, ygrid = construct_grids(data)
X, Y = np.meshgrid(xgrid[::5], ygrid[::5][::-1])
land_reference = data.coverages[6][::5, ::5]
land_mask = (land_reference > -9999).ravel()
xy = np.vstack([Y.ravel(), X.ravel()]).T
xy = xy[land_mask]
xy *= np.pi / 180.
# Plot map of South America with distributions of each species
fig = plt.figure()
fig.subplots_adjust(left=0.05, right=0.95, wspace=0.05)
kde0 = KernelDensity(bandwidth=0.04, metric='haversine',
kernel='gaussian', algorithm='ball_tree')
kde0.fit(Xtrain[ytrain == 0])
kde1 = KernelDensity(bandwidth=0.04, metric='haversine',
kernel='gaussian', algorithm='ball_tree')
kde1.fit(Xtrain[ytrain == 1])
kde01=kde0*kde1
plt.subplot(1, 1, 1)
# evaluate only on the land: -9999 indicates ocean
Z = np.full(land_mask.shape[0], -9999, dtype='int')
Z[land_mask] = np.exp(kde01.score_samples(xy))
Z = Z.reshape(X.shape)
# plot contours of the density
levels = np.linspace(0, Z.max(), 25)
plt.contourf(X, Y, Z, levels=levels, cmap=plt.cm.Reds)
plt.show()

How to extract a 2D plane from a 3D numpy meshgrid

[TLDR]:
Essentially my question boils down to how one can extract the 2d data of a plane from a 3D numpy meshgrid
[Detailed Description]:
I am calculating the electric field of two (or more) point charges. I did this in 2D and can plot the results via matplotlib using quiver or streamplot
import numpy as np
from matplotlib import pyplot as plt
eps_0 = 8e-12
fac = (1./(4*np.pi*eps_0))
charges = [1.0,-1.0]
qx = [-2.0,2.0]
qy = [0.0,0.0]
# GRID
gridsize = 4.0
N = 11
X,Y = np.meshgrid( np.linspace(-gridsize,gridsize,N),
np.linspace(-gridsize,gridsize,N))
# CALC E-FIELD
sumEx = np.zeros_like(X)
sumEy = np.zeros_like(Y)
for q, qxi, qyi in zip(charges,qx,qy):
dist_vec_x = X - qxi
dist_vec_y = Y - qyi
dist = np.sqrt(dist_vec_x**2 + dist_vec_y**2)
Ex = fac * q * (dist_vec_x/dist**3)
Ey = fac * q * (dist_vec_y/dist**3)
sumEx += Ex
sumEy += Ey
# PLOT
fig = plt.figure()
ax = fig.add_subplot(111)
ax.streamplot(X,Y,sumEx,sumEy)
plt.show()
This produces the correct results
I can easily extend this to 3D
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
eps_0 = 8e-12
fac = (1./(4*np.pi*eps_0))
charges = [1.0,-1.0]
qx = [-2.0,2.0]
qy = [0.0,0.0]
qz = [0.0,0.0]
# GRID
gridsize = 4.0
N = 11
X,Y,Z = np.meshgrid( np.linspace(-gridsize,gridsize,N),
np.linspace(-gridsize,gridsize,N),
np.linspace(-gridsize,gridsize,N))
# CALC E-FIELD
sumEx = np.zeros_like(X)
sumEy = np.zeros_like(Y)
sumEz = np.zeros_like(Z)
for q, qxi, qyi, qzi in zip(charges,qx,qy,qz):
dist_vec_x = X - qxi
dist_vec_y = Y - qyi
dist_vec_z = Z - qzi
dist = np.sqrt(dist_vec_x**2 + dist_vec_y**2 + dist_vec_z**2)
Ex = fac * q * (dist_vec_x/dist**3)
Ey = fac * q * (dist_vec_y/dist**3)
Ez = fac * q * (dist_vec_z/dist**3)
sumEx += Ex
sumEy += Ey
sumEz += Ez
# PLOT
fig = plt.figure()
ax = fig.gca(projection='3d')
ax.quiver(X,Y,Z,sumEx,sumEy,sumEz, pivot='middle', normalize=True)
plt.show()
This also yields the correct result when plotted in 3D (as far as I can tell)
But for some reason I can not figure out how to extract the data from one x-y plane from the generated 3D numpy mesh. I thought I could just do something like
zplane = round(N/2)
ax.quiver(X,Y,sumEx[:,:,zplane],sumEy[:,:,zplane])
but this does not do the trick. Does anyone know the proper way here?

Remove projection='3d' and index X and Y:
fig = plt.figure()
ax = fig.gca()
zplane = round(N / 2)
ax.quiver(X[:, :, zplane], Y[:, :, zplane], sumEx[:, :, zplane], sumEy[:, :, zplane])
plt.show()
If you select a specific zplane your plot is no longer a 3D-plot.

Creating a filled numpy array of ones from a polygon points Python

I would be very gratefull if anyone could help me find a faster solution to my problem.
Heres the scenario:- I have a polygon of float points which I want to map to a grid. The grid cells can be different width and height not uniformed like my image shows. ie rectangular.
I have tried using Image draw but it only uses ints. Converting floats to ints means I have to scale the floats up and remove decimal to keep some precision but image draw will not work with the larger polygon of points.
Is there a more eloquent and fast way of achieving a numpy array of ones (blue) for the filled area of the polygon and zeros (red) for the rest. I have read a little on mesh grid but cant see how it can be of use for this scenario.
Many thanks
The results from the code are
cols = 4
rows = 4
points = [[1535116L, 1725047L], [1535116L, 2125046L], [-464884L, 2125046L], [-464884L, 125046L]]
bbCut = getPythonBoundBox(points)
cutWidth = bbCut[1][0]-bbCut[0][0]
scale = float(cutWidth) / float(rows)
###Center data to origin
for p in range(len(points)):
points[p][0] -= (bbCut[1][0] - bbCut[0][0])/2
points[p][1] -= (bbCut[1][1] - bbCut[0][1])/2
points[p][0] /= scale
points[p][1] /= scale
##move points to Zero
bbCut = getPythonBoundBox(points)
for p in range(len(points)):
points[p][0] -=bbCut[0][0]
points[p][1] -=bbCut[0][1]
pointToTuple= []
for p in range(len(points)):
pointToTuple.append((points[p][0], points[p][1]))
imgWidth = float(rows)
imgHeight = float(cols)
img = Image.new('L', (int(imgWidth), int(imgHeight)), 0)
draw = ImageDraw.Draw(img)
draw.polygon(pointToTuple, fill=1)
array = np.reshape(list(img.getdata()), (cols, rows))
############This is the result from the array############
##If you compare this array to the coloured scaled image ive have drawn
##its missing a 1 on the second value in the first row
##and another 1 on the second row 3rd value
##I'm assuming there is some parsing happening here with float to int?
array([1, 0, 0, 0])
array([1, 1, 0, 0])
array([1, 1, 1, 1])
array([1, 1, 1, 1])
#########################################################
def getPythonBoundBox(points):
bigNumber = 10e10
xmin = bigNumber
xmax = -bigNumber
ymin = bigNumber
ymax = -bigNumber
g = []
a = len(points)
for i in xrange(a):
if points[i][0] < xmin: xmin = points[i][0]
if points[i][0] > xmax: xmax = points[i][0]
if points[i][1] < ymin: ymin = points[i][1]
if points[i][1] > ymax: ymax = points[i][1]
p1 = [xmin,ymin]
g.append(p1)
p2 = [xmax,ymax]
g.append(p2)
return (g)

matplotlib.path.Path has a method contains_points. Hence simply instantiate a path with your polygon points and then check your grid coordinates if they fall within that path. Your grid can have any resolution you want. This is controlled by nx and ny (or alternatively dx and dy) in the code below.
Code:
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import PathPatch
from matplotlib.path import Path
# create a matplotlib path
points = [[1535116L, 1725047L],
[1535116L, 2125046L],
[-464884L, 2125046L],
[-464884L, 125046L],
[1535116L, 1725047L]]
codes = [Path.MOVETO,
Path.LINETO,
Path.LINETO,
Path.LINETO,
Path.CLOSEPOLY,
]
path = Path(points, codes)
# check the path
fig, (ax1, ax2, ax3) = plt.subplots(1,3)
patch = PathPatch(path, facecolor='k')
ax1.add_patch(patch)
xmin, ymin = np.min(points, axis=0)
xmax, ymax = np.max(points, axis=0)
ax1.set_ylim(ymin,ymax)
ax1.set_xlim(xmin,xmax)
ax1.set_aspect('equal')
# create a grid
nx, ny = 1000, 1000
x = np.linspace(xmin, xmax, nx)
y = np.linspace(ymin, ymax, ny)
xgrid, ygrid = np.meshgrid(x, y)
pixel_coordinates = np.c_[xgrid.ravel(), ygrid.ravel()]
# find points within path
img = path.contains_points(pixel_coordinates).reshape(nx,ny)
# plot
ax2.imshow(img, cmap='gray_r', interpolation='none', origin='lower')
# repeat, but this time specify pixel widths explicitly
dx, dy = 2000, 2000
x = np.arange(xmin, xmax, dx)
y = np.arange(ymin, ymax, dy)
xgrid, ygrid = np.meshgrid(x, y)
pixel_coordinates = np.c_[xgrid.ravel(), ygrid.ravel()]
img = path.contains_points(pixel_coordinates).reshape(len(x), len(y))
ax3.imshow(img, cmap='gray_r', interpolation='none', origin='lower')
UPDATE:
Ok, so this now tests the if any of the corners of each tile are within the path. For some reason, I still get another answer than the picture suggests. How sure are you, that the points that you provide are exact?
Code + image:
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import PathPatch
from matplotlib.path import Path
# create a matplotlib path
points = [[1535116L, 1725047L],
[1535116L, 2125046L],
[-464884L, 2125046L],
[-464884L, 125046L],
[1535116L, 1725047L]]
codes = [Path.MOVETO,
Path.LINETO,
Path.LINETO,
Path.LINETO,
Path.CLOSEPOLY,
]
path = Path(points, codes)
fig, (ax1, ax2) = plt.subplots(1,2)
patch = PathPatch(path, facecolor='k')
ax1.add_patch(patch)
xmin, ymin = np.min(points, axis=0)
xmax, ymax = np.max(points, axis=0)
ax1.set_ylim(ymin,ymax)
ax1.set_xlim(xmin,xmax)
ax1.set_aspect('equal')
nx, ny = 4, 4
x = np.linspace(xmin, xmax, nx)
y = np.linspace(ymin, ymax, ny)
xgrid, ygrid = np.meshgrid(x, y)
pixel_centers = np.c_[xgrid.ravel(), ygrid.ravel()]
def pixel_center_to_corners(x, y, dx, dy, precision=0.):
"""
Returns array indexed by (pixel, corner, (x,y))
"""
# make dx and dy ever so slightly smaller,
# such that the points fall **inside** the path (not **on** the path)
dx -= precision
dy -= precision
return np.array([(x - dx/2., y - dy/2.), # lower left
(x + dx/2., y - dy/2.), # lower right
(x + dx/2., y + dy/2.), # upper right
(x - dx/2., y + dy/2.), # upper left
]).transpose([2,0,1])
# get pixel corners
dx = (xmax - xmin) / float(nx)
dy = (ymax - ymin) / float(ny)
pixel_corners = pixel_center_to_corners(pixel_centers[:,0], pixel_centers[:,1], dx, dy)
# test corners of each pixel;
# set img to True, iff any corners within path;
img = np.zeros((len(pixel_corners)))
for ii, pixel in enumerate(pixel_corners):
is_inside_path = path.contains_points(pixel)
img[ii] = np.any(is_inside_path)
img = img.reshape(len(x), len(y))
ax2.imshow(img, cmap='gray_r', interpolation='none', origin='lower')

Plotting the temperature distribution on a sphere with python

I have the following problem:
a have N points on a sphere specified by a array x, with x.shape=(N,3). This array contains their cartesian coordinates. Furthermore, at each point, I have a specified temperature. This quantity is saved in an array T, with T.shape=(N,).
Is there any straight forward way to map this temperature distribution into the plane using different colors?
If it simplifies the task, the position can also be given in polar coordinates (\theta,\phi).

To plot your data, you can use Basemap. The only problem is, that both contour and contourf routines needs gridded data. Here is example with naive (and slow) IDW-like interpolation on sphere. Any comments are welcome.
import numpy as np
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
def cart2sph(x, y, z):
dxy = np.sqrt(x**2 + y**2)
r = np.sqrt(dxy**2 + z**2)
theta = np.arctan2(y, x)
phi = np.arctan2(z, dxy)
theta, phi = np.rad2deg([theta, phi])
return theta % 360, phi, r
def sph2cart(theta, phi, r=1):
theta, phi = np.deg2rad([theta, phi])
z = r * np.sin(phi)
rcosphi = r * np.cos(phi)
x = rcosphi * np.cos(theta)
y = rcosphi * np.sin(theta)
return x, y, z
# random data
pts = 1 - 2 * np.random.rand(500, 3)
l = np.sqrt(np.sum(pts**2, axis=1))
pts = pts / l[:, np.newaxis]
T = 150 * np.random.rand(500)
# naive IDW-like interpolation on regular grid
theta, phi, r = cart2sph(*pts.T)
nrows, ncols = (90,180)
lon, lat = np.meshgrid(np.linspace(0,360,ncols), np.linspace(-90,90,nrows))
xg,yg,zg = sph2cart(lon,lat)
Ti = np.zeros_like(lon)
for r in range(nrows):
for c in range(ncols):
v = np.array([xg[r,c], yg[r,c], zg[r,c]])
angs = np.arccos(np.dot(pts, v))
idx = np.where(angs == 0)[0]
if idx.any():
Ti[r,c] = T[idx[0]]
else:
idw = 1 / angs**2 / sum(1 / angs**2)
Ti[r,c] = np.sum(T * idw)
# set up map projection
map = Basemap(projection='ortho', lat_0=45, lon_0=15)
# draw lat/lon grid lines every 30 degrees.
map.drawmeridians(np.arange(0, 360, 30))
map.drawparallels(np.arange(-90, 90, 30))
# compute native map projection coordinates of lat/lon grid.
x, y = map(lon, lat)
# contour data over the map.
cs = map.contourf(x, y, Ti, 15)
plt.title('Contours of T')
plt.show()

One way to do this is to set facecolors by mapping your heat data through the colormap.
Here's an example:
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
u = np.linspace(0, 2 * np.pi, 80)
v = np.linspace(0, np.pi, 80)
# create the sphere surface
x=10 * np.outer(np.cos(u), np.sin(v))
y=10 * np.outer(np.sin(u), np.sin(v))
z=10 * np.outer(np.ones(np.size(u)), np.cos(v))
# simulate heat pattern (striped)
myheatmap = np.abs(np.sin(y))
ax.plot_surface(x, y, z, cstride=1, rstride=1, facecolors=cm.hot(myheatmap))
plt.show()
Here, my "heatmap" is just stripes along the y-axis, which I made using the function np.abs(np.sin(y)), but anything that goes form 0 to 1 will work (and, of course, it needs to match the shapes on x, etc.

Plot GDAL raster using matplotlib Basemap

I would like to plot a raster tiff (download-723Kb) using matplotlib Basemap. My raster's projection coordinates is in meter:
In [2]:
path = r'albers_5km.tif'
raster = gdal.Open(path, gdal.GA_ReadOnly)
array = raster.GetRasterBand(20).ReadAsArray()
print ('Raster Projection:\n', raster.GetProjection())
print ('Raster GeoTransform:\n', raster.GetGeoTransform())
Out [2]:
Raster Projection:
PROJCS["unnamed",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]],PROJECTION["Albers_Conic_Equal_Area"],PARAMETER["standard_parallel_1",15],PARAMETER["standard_parallel_2",65],PARAMETER["latitude_of_center",30],PARAMETER["longitude_of_center",95],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]]]
Raster GeoTransform:
(190425.8243, 5000.0, 0.0, 1500257.0112, 0.0, -5000.0)
If I try to plot this using a Robin projection using contourf with latlon=False than x and y are assumed to be map projection coordinates (see docs, I think that's what I have).
But if I look to the plot I notice it's placed bottom left very small:
Using this code:
In [3]:
xy = raster.GetGeoTransform()
x = raster.RasterXSize
y = raster.RasterYSize
lon_start = xy[0]
lon_stop = x*xy[1]+xy[0]
lon_step = xy[1]
lat_start = xy[3]
lat_stop = y*xy[5]+xy[3]
lat_step = xy[5]
fig = plt.figure(figsize=(16,10))
map = Basemap(projection='robin',resolution='c',lat_0=0,lon_0=0)
lons = np.arange(lon_start, lon_stop, lon_step)
lats = np.arange(lat_start, lat_stop, lat_step)
xx, yy = np.meshgrid(lons,lats)
levels = [array.min(),-0.128305,array.max()]
map.contourf(xx, yy,array, levels, cmap=cm.RdBu_r, latlon=False)
map.colorbar(cntr,location='right',pad='10%')
map.drawcoastlines(linewidth=.5)
map.drawcountries(color='red')
Eventually I don't want to have a world view but a detailed view. But this gives me a zoom level where the coastlines and countries are drawn, but data is again placed in bottom left corner, but not as small as previous time:
Using the following code:
In [4]:
extent = [ xy[0],xy[0]+x*xy[1], xy[3],xy[3]+y*xy[5]]
width_x = (extent[1]-extent[0])*10
height_y = (extent[2]-extent[3])*10
fig = plt.figure(figsize=(16,10))
map = Basemap(projection='stere', resolution='c', width = width_x , height = height_y, lat_0=40.2,lon_0=99.6,)
xx, yy = np.meshgrid(lons,lats)
levels = [array.min(),-0.128305,array.max()]
map.contourf(xx, yy, array, levels, cmap=cm.RdBu_r, latlon=False)
map.drawcoastlines(linewidth=.5)
map.drawcountries(color='red')

You can use the following code to convert the coordinates, it automatically takes the projection from your raster as the source and the projection from your Basemap object as the target coordinate system.
Imports
from mpl_toolkits.basemap import Basemap
import osr, gdal
import matplotlib.pyplot as plt
import numpy as np
Coordinate conversion
def convertXY(xy_source, inproj, outproj):
# function to convert coordinates
shape = xy_source[0,:,:].shape
size = xy_source[0,:,:].size
# the ct object takes and returns pairs of x,y, not 2d grids
# so the the grid needs to be reshaped (flattened) and back.
ct = osr.CoordinateTransformation(inproj, outproj)
xy_target = np.array(ct.TransformPoints(xy_source.reshape(2, size).T))
xx = xy_target[:,0].reshape(shape)
yy = xy_target[:,1].reshape(shape)
return xx, yy
Reading and processing the data
# Read the data and metadata
ds = gdal.Open(r'albers_5km.tif')
data = ds.ReadAsArray()
gt = ds.GetGeoTransform()
proj = ds.GetProjection()
xres = gt[1]
yres = gt[5]
# get the edge coordinates and add half the resolution
# to go to center coordinates
xmin = gt[0] + xres * 0.5
xmax = gt[0] + (xres * ds.RasterXSize) - xres * 0.5
ymin = gt[3] + (yres * ds.RasterYSize) + yres * 0.5
ymax = gt[3] - yres * 0.5
ds = None
# create a grid of xy coordinates in the original projection
xy_source = np.mgrid[ymax+yres:ymin:yres, xmin:xmax+xres:xres]
Plotting
# Create the figure and basemap object
fig = plt.figure(figsize=(12, 6))
m = Basemap(projection='robin', lon_0=0, resolution='c')
# Create the projection objects for the convertion
# original (Albers)
inproj = osr.SpatialReference()
inproj.ImportFromWkt(proj)
# Get the target projection from the basemap object
outproj = osr.SpatialReference()
outproj.ImportFromProj4(m.proj4string)
# Convert from source projection to basemap projection
xx, yy = convertXY(xy_source, inproj, outproj)
# plot the data (first layer)
im1 = m.pcolormesh(xx, yy, data[0,:,:], cmap=plt.cm.jet, shading='auto')
# annotate
m.drawcountries()
m.drawcoastlines(linewidth=.5)
plt.savefig('world.png',dpi=75)
If you need the pixels location to be 100% correct you might want to check the creation of the coordinate arrays really careful yourself (because i didn't at all). This example should hopefully set you on the right track.