Develop Reference

Python: Surface Plot not Representing Data Set Properly

I am currently trying to create a 3D surface plot based on some topographic survey data I took. The data consists of a GPS position based in UTM. I then created a mesh, interpolated using the nearest neighbor method, and plotted the surface using plotly. My main problem comes from the fact that my surface plot is very spiky, whereas the scatter plot of just the raw points looks more like what I want. For reference this is topographic data of 4 dunes on a beach.
I think I have narrowed the problem down to the interpolation method, but I just don't know how to fix it so that the graph will be more smooth.
Thanks in advance for any advice/ suggestions.
import numpy as np
import plotly.graph_objects as go
import pandas as pd
from scipy.interpolate import griddata
# Read Data
eBAD = pd.read_csv('EastBeachAllDunes20190202.txt', header = None, delimiter = ',')
eBAD.columns = ["Point #","Northing","Easting","Zed","NaN"]
eBAD = pd.DataFrame(data = eBAD)
# Define Variables
x = np.array(eBAD.Easting)
y = np.array(eBAD.Northing)
z = np.array(eBAD.Zed)
# Creating Mesh
xi = np.linspace(min(x), max(x))
yi = np.linspace(min(y), max(y))
X, Y = np.meshgrid(xi, yi)
Z = griddata((x, y), z, (xi[None,:], yi[:,None]), method='nearest')
# Plot Scatter of Raw GPS Data
fig = go.Figure(data=[go.Scatter3d(x=x, y=y, z=z,mode='markers')])
#fig.update_layout(scene_aspectmode='data')
fig.update_layout(scene_aspectmode='manual',
scene_aspectratio=dict(x = 5, y = 5, z = 0.5))
fig.show()
# Plot Surface Using Mesh
fig2 = go.Figure(data=[go.Surface(x = xi, y = yi, z=Z)])
#fig2.update_layout(scene_aspectmode='data')
fig2.update_layout(scene_aspectmode='manual',
scene_aspectratio=dict(x = 5, y = 5, z = 0.5))
fig2.show()
# Plot Surface Using Mesh
fig3 = go.Figure(data=[go.Surface(x = xi, y = yi, z=Z),go.Scatter3d(x=x, y=y, z=z,mode='markers')])
#fig2.update_layout(scene_aspectmode='data')
fig3.update_layout(scene_aspectmode='manual',
scene_aspectratio=dict(x = 5, y = 5, z = 0.5))
fig3.show()
Scatter Plot
Surface Plot
Scatter Plot Over Surface Plot. Here you can see that the higher points do not create a good surface visualization.

Aligning data (contourf) on Basemap

I've started working with Basemap, which seems potentially very useful.
If I plot some global data on a latitude/longitude grid as filled contours, it works great: Iff I leave the lat_0 and lon_0 as zero. Once I change the center location, the map moves but the data doesn't. I would be grateful for advice.
I've created a simple version of the code I'm using, with some simple sample data that illustrates the problem. The values should be (are) large at the equator but small at the poles. If you run the code with lat_0 and lon_0 = 0, it works fine. But if you change the center location to a different coordinate, the same pattern/data is presented even though the map has moved.
from mpl_toolkits.basemap import Basemap, cm
import matplotlib.pyplot as plt
import numpy as np
# create data
lat = np.linspace(-90,90,num=180)
lon = np.linspace(-180,180,num=361)
h2o_north = np.linspace(1,65,num=90)
h2o_south = np.flipud(h2o_north)
h2o = np.append(h2o_north,h2o_south)
data = np.transpose(np.tile(h2o,(len(lon),1)))
# create figure and axes instances
fig = plt.figure(figsize=(10,10))
ax = fig.add_axes([0.1,0.1,0.8,0.8])
# create map
m = Basemap(projection='ortho',lon_0=-50,lat_0=50,resolution='l')
# draw coastlines and country boundaries
m.drawcoastlines()
m.drawcountries()
# draw parallels
parallels = np.arange(-90.,90,10.)
m.drawparallels(parallels)
# draw meridians
meridians = np.arange(180.,360.,10.)
m.drawmeridians(meridians)
ny = data.shape[0]
nx = data.shape[1]
lons, lats = m.makegrid(nx, ny) # get lat/lons of ny by nx evenly space grid
x, y = m(lons, lats) # compute map projection coordinates
# draw filled contours.
clevs = np.linspace(0,70,num=281)
cs = m.contourf(x,y,data,clevs,cmap=plt.cm.jet)
# colorbar
cbar = m.colorbar(cs,location='bottom',pad="5%",ticks=np.linspace(0,70,15))
cbar.set_label('Scale of the data')
plt.title('Some global data', fontsize=14)

Use np.meshgrid() to create the meshgrid of lon-lat, then, convert it to projection coordinates, and the data are ready to generate contours and plot.
Here is the working code:
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
import numpy as np
# data for z (2D array)
h2o_north = np.linspace(1, 65, num=90)
h2o_south = np.flipud(h2o_north)
h2o = np.append(h2o_north, h2o_south)
data = np.transpose(np.tile(h2o, (len(h2o_north), 1)))
# create figure and axes instances
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot()
# create basemap instance
m = Basemap(projection='ortho', lon_0=-50, lat_0=50, resolution='c', ax=ax)
# create meshgrid covering the whole globe with ...
# conforming dimensions of the `data`
lat = np.linspace(-90, 90, data.shape[0])
lon = np.linspace(-180, 180, data.shape[1])
xs, ys = np.meshgrid(lon, lat) # basic mesh in lon, lat (degrees)
x, y = m(xs, ys) # convert (lon,lat) to map (x,y)
# draw filled contours
clevs = np.linspace(0, np.max(data), 60)
cs = m.contourf(x, y, data, clevs, cmap=plt.cm.jet)
m.drawcoastlines()
m.drawcountries()
m.drawmeridians(range(-180, 180, 30))
m.drawparallels(range(-90, 90, 30))
# draw colorbar
cbar = m.colorbar(cs, location='bottom', pad="5%", ticks=np.linspace(0, np.max(data), 5))
cbar.set_label('Scale of the data')
plt.show()
The resulting plot:

Get coordinates from points density with KDE

I have a set of points (> 1k) in this form:
y,x
173.549,308.176
173.549,313.328
213.26,419.588
Using KDE, i can plot points density with pcolormesh and contourf. This is an example result, plotting points too:
This is the code i used to have the plot:
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from scipy.stats.kde import gaussian_kde
x, y = np.genfromtxt('terzinoSX.csv', delimiter=',', unpack=True)
y = y[np.logical_not(np.isnan(y))]
x = x[np.logical_not(np.isnan(x))]
k = gaussian_kde(np.vstack([x, y]))
xi, yi = np.mgrid[x.min():x.max():x.size**0.5*1j,y.min():y.max():y.size**0.5*1j]
zi = k(np.vstack([xi.flatten(), yi.flatten()]))
fig = plt.figure(figsize=(7,4))
ax2 = fig.add_subplot(111)
#alpha=0.5 will make the plots semitransparent
#ax1.pcolormesh(yi, xi, zi.reshape(xi.shape), alpha=0.5)
ax2.contourf(yi, xi, zi.reshape(xi.shape), alpha=0.5)
plt.axis('off')
ax2.plot(y,x, "o")
ax2.set_xlim(0, 740)
ax2.set_ylim(515, 0)
#overlay soccer field
im = plt.imread('statszone_football_pitch.png')
ax2.imshow(im, extent=[0, 740, 0, 515], aspect='auto')
fig.savefig('test.png', bbox_inches='tight')
I would like to have one point representing coordinates of most populated zone (middle point for example), like a middle point over the "red" zone. Is it possible in some way?

I solved this by adding these lines that calculate the point in the most populated area:
xy = np.vstack([x,y])
kde = stats.gaussian_kde(xy)
density = kde(xy)
pts = xy.T[np.argmax(density)]

You can use np.argmax to get the coordinates of the maximum. For example:
kde = compute_my_kde() # Returns a two-dimensional array
y, x = np.argmax(kde) # x and y are swapped because matplotlib displays images as a matrix (first index is rows, second index is colums)
plt.imshow(kde) # Show the kde
plt.scatter(x, y) # Show the maximum point

Shifting grid with matplotlib

I'm plotting data which is formatted between 0 and 360 degrees. I'm trying to plot this on cyl or merc projection, but it is only showing data from 0 onwards (I want to plot the data with the GMT in the center, so need the data on a lon grid of -180 to 180). If I shift the grid (lon = lon -180) then all data shows, but the data is in the wrong place by -180 degrees.
Issue:
Works fine in ortho projection though. Relevant code below.
lat = np.linspace(90,-90,721)
lon = np.linspace(0,360,1440)
m = Basemap(projection='cyl',llcrnrlat=-90,urcrnrlat=90,llcrnrlon=0,urcrnrlon=360,resolution='c',)
X, Y = np.meshgrid(lon, lat)
X, Y = m(X, Y)
cs = m.contourf(X,Y,Plot,scale, cmap=cmap)

Please Try:
import numpy as np
from mpl_toolkits.basemap import shiftgrid
from mpl_toolkits.basemap import Basemap
lat = np.linspace(-90, 90, 721)
lon = np.linspace(0, 360, 1440)
Plot, lon = shiftgrid(180., Plot, lon, start=False) # shiftgrid
m = Basemap(projection='cyl', llcrnrlat=-90, urcrnrlat=90, llcrnrlon=-180, urcrnrlon=180, resolution='c',)
X, Y = np.meshgrid(lon, lat)
X, Y = m(X, Y)
cs = m.contourf(X, Y, Plot, scale, cmap=cmap)
shiftgrid: shifts global lat/lon grids east or west.

I have a solution (albeit an ugly one). By reordering the data.
temp = np.zeros((721,1440))
temp[:,0:720] = Plot[:,720:1440]
temp[:,720:1440] = Plot[:,0:720]
Plot[:]=temp[:]
Or by using np.roll (if you know how many gridpoints to shift)

Plot only on continent in matplotlib

I am drawing a map using basemap from matplotlib. The data are spreaded all over the world, but I just want to retain all the data on the continent and drop those on the ocean. Is there a way that I can filter the data, or is there a way to draw the ocean again to cover the data?

There's method in matplotlib.basemap: is_land(xpt, ypt)
It returns True if the given x,y point (in projection coordinates) is over land, False otherwise. The definition of land is based upon the GSHHS coastline polygons associated with the class instance. Points over lakes inside land regions are not counted as land points.
For more information, see here.

is_land() will loop all the polygons to check whether it's land or not. For large data size, it's very slow. You can use points_inside_poly() from matplotlib to check an array of points quickly. Here is the code. It doesn't check lakepolygons, if you want remove points in lakes, you can add your self.
It took 2.7 seconds to check 100000 points on my PC. If you want more speed, you can convert the polygons into a bitmap, but it's a little difficult to do this. Please tell me if the following code is not fast enought for your dataset.
from mpl_toolkits.basemap import Basemap
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.nxutils as nx
def points_in_polys(points, polys):
result = []
for poly in polys:
mask = nx.points_inside_poly(points, poly)
result.extend(points[mask])
points = points[~mask]
return np.array(result)
points = np.random.randint(0, 90, size=(100000, 2))
m = Basemap(projection='moll',lon_0=0,resolution='c')
m.drawcoastlines()
m.fillcontinents(color='coral',lake_color='aqua')
x, y = m(points[:,0], points[:,1])
loc = np.c_[x, y]
polys = [p.boundary for p in m.landpolygons]
land_loc = points_in_polys(loc, polys)
m.plot(land_loc[:, 0], land_loc[:, 1],'ro')
plt.show()

The HYRY's answer won't work on new versions of matplotlib (nxutils is deprecated). I've made a new version that works:
from mpl_toolkits.basemap import Basemap
import matplotlib.pyplot as plt
from matplotlib.path import Path
import numpy as np
map = Basemap(projection='cyl', resolution='c')
lons = [0., 0., 16., 76.]
lats = [0., 41., 19., 51.]
x, y = map(lons, lats)
locations = np.c_[x, y]
polygons = [Path(p.boundary) for p in map.landpolygons]
result = np.zeros(len(locations), dtype=bool)
for polygon in polygons:
result += np.array(polygon.contains_points(locations))
print result

The simplest way is to use basemap's maskoceans.
If for each lat, lon you have a data and you want to
use contours:
After meshgrid and interpolation:
from scipy.interpolate import griddata as gd
from mpl_toolkits.basemap import Basemap, cm, maskoceans
xi, yi = np.meshgrid(xi, yi)
zi = gd((mlon, mlat),
scores,
(xi, yi),
method=grid_interpolation_method)
#mask points on ocean
data = maskoceans(xi, yi, zi)
con = m.contourf(xi, yi, data, cmap=cm.GMT_red2green)
#note instead of zi we have data now.
Update (much faster than in_land or in_polygon solutions):
If for each lat, lon you don't have any data, and you just want to scatter the points only over land:
x, y = m(lons, lats)
samples = len(lons)
ocean = maskoceans(lons, lats, datain=np.arange(samples),
resolution='i')
ocean_samples = np.ma.count_masked(ocean)
print('{0} of {1} points in ocean'.format(ocean_samples, samples))
m.scatter(x[~ocean.mask], y[~ocean.mask], marker='.', color=colors[~ocean.mask], s=1)
m.drawcountries()
m.drawcoastlines(linewidth=0.7)
plt.savefig('a.png')

I was answering this question, when I was told that it would be better to post my answer over here. Basically, my solution extracts the polygons that are used to draw the coastlines of the Basemap instance and combines these polygons with the outline of the map to produce a matplotlib.PathPatch that overlays the ocean areas of the map.
This especially useful if the data is coarse and interpolation of the data is not wanted. In this case using maskoceans produces a very grainy outline of the coastlines, which does not look very good.
Here is the same example I posted as answer for the other question:
from matplotlib import pyplot as plt
from mpl_toolkits import basemap as bm
from matplotlib import colors
import numpy as np
import numpy.ma as ma
from matplotlib.patches import Path, PathPatch
fig, ax = plt.subplots()
lon_0 = 319
lat_0 = 72
##some fake data
lons = np.linspace(lon_0-60,lon_0+60,10)
lats = np.linspace(lat_0-15,lat_0+15,5)
lon, lat = np.meshgrid(lons,lats)
TOPO = np.sin(np.pi*lon/180)*np.exp(lat/90)
m = bm.Basemap(resolution='i',projection='laea', width=1500000, height=2900000, lat_ts=60, lat_0=lat_0, lon_0=lon_0, ax = ax)
m.drawcoastlines(linewidth=0.5)
x,y = m(lon,lat)
pcol = ax.pcolormesh(x,y,TOPO)
##getting the limits of the map:
x0,x1 = ax.get_xlim()
y0,y1 = ax.get_ylim()
map_edges = np.array([[x0,y0],[x1,y0],[x1,y1],[x0,y1]])
##getting all polygons used to draw the coastlines of the map
polys = [p.boundary for p in m.landpolygons]
##combining with map edges
polys = [map_edges]+polys[:]
##creating a PathPatch
codes = [
[Path.MOVETO] + [Path.LINETO for p in p[1:]]
for p in polys
]
polys_lin = [v for p in polys for v in p]
codes_lin = [c for cs in codes for c in cs]
path = Path(polys_lin, codes_lin)
patch = PathPatch(path,facecolor='white', lw=0)
##masking the data:
ax.add_patch(patch)
plt.show()
This produces the following plot:
Hope this is helpful to someone :)