Develop Reference

How to plot geoTiff file with shape file and lat long gridlines using python?

I am trying to plot a GeoTiff file using python as given in the example (https://www.neonscience.org/plot-neon-rgb-py#comment-668). This script convert GeoTIFF file into the true-color image which likes how human eyes would see. I could plot the GeoTIFF but I want to overlay a shapefile and lat and long grid lines.
import gdal
import numpy as np
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
def RGBraster2array(RGB_geotif):
"""RGBraster2array reads in a NEON AOP geotif file and returns
a numpy array, and header containing associated metadata with spatial information.
--------
Parameters
RGB_geotif -- full or relative path and name of reflectance hdf5 file
--------
Returns
--------
array:
numpy array of geotif values
metadata:
dictionary containing the following metadata (all strings):
array_rows
array_cols
bands
driver
projection
geotransform
pixelWidth
pixelHeight
extent
noDataValue
scaleFactor
--------
Example Execution:
--------
RGB_geotif = '2017_SERC_2_368000_4306000_image.tif'
RGBcam_array, RGBcam_metadata = RGBraster2array(RGB_geotif) """
metadata = {}
dataset = gdal.Open(RGB_geotif)
metadata['array_rows'] = dataset.RasterYSize
metadata['array_cols'] = dataset.RasterXSize
metadata['bands'] = dataset.RasterCount
metadata['driver'] = dataset.GetDriver().LongName
metadata['projection'] = dataset.GetProjection()
metadata['geotransform'] = dataset.GetGeoTransform()
mapinfo = dataset.GetGeoTransform()
metadata['pixelWidth'] = mapinfo[1]
metadata['pixelHeight'] = mapinfo[5]
metadata['ext_dict'] = {}
metadata['ext_dict']['xMin'] = mapinfo[0]
metadata['ext_dict']['xMax'] = mapinfo[0] + dataset.RasterXSize/mapinfo[1]
metadata['ext_dict']['yMin'] = mapinfo[3] + dataset.RasterYSize/mapinfo[5]
metadata['ext_dict']['yMax'] = mapinfo[3]
metadata['extent'] = (metadata['ext_dict']['xMin'],metadata['ext_dict']['xMax'],
metadata['ext_dict']['yMin'],metadata['ext_dict']['yMax'])
raster = dataset.GetRasterBand(1)
array_shape = raster.ReadAsArray(0,0,metadata['array_cols'],metadata['array_rows']).astype(np.float).shape
metadata['noDataValue'] = raster.GetNoDataValue()
metadata['scaleFactor'] = raster.GetScale()
array = np.zeros((array_shape[0],array_shape[1],dataset.RasterCount),'uint8') #pre-allocate stackedArray matrix
for i in range(1, dataset.RasterCount+1):
band = dataset.GetRasterBand(i).ReadAsArray(0,0,metadata['array_cols'],metadata['array_rows']).astype(np.float)
band[band==metadata['noDataValue']]=np.nan
band = band/metadata['scaleFactor']
array[...,i-1] = band
return array, metadata
RGB_geotif = '/home/krishna/Desktop/Py_works/geo/True_color.tiff'
SERC_RGBcam_array, SERC_RGBcam_metadata = RGBraster2array(RGB_geotif)
SERC_RGBcam_array.shape
#Display information stored in header
for key in sorted(SERC_RGBcam_metadata.keys()):
print(key)
def plot_band_array(band_array,
refl_extent,
colorlimit,
ax=plt.gca(),
title='',
cbar ='on',
cmap_title='',
colormap='spectral'):
'''plot_band_array reads in and plots a single band or an rgb band combination of a reflectance array
--------
Parameters
--------
band_array: flightline array of reflectance values, created from h5refl2array function
refl_extent: extent of reflectance data to be plotted (xMin, xMax, yMin, yMax) - use metadata['extent'] from h5refl2array function
colorlimit: range of values to plot (min,max). Best to look at the histogram of reflectance values before plotting to determine colorlimit.
ax: optional, default = current axis
title: string, optional; plot title
cmap_title: string, optional; colorbar title
colormap: string, optional; see https://matplotlib.org/examples/color/colormaps_reference.html for list of colormaps
--------
Returns
plots array of single band or RGB if given a 3-band
--------
Example:
--------
plot_band_array(SERC_RGBcam_array,
SERC_RGBcam_metadata['extent'],
(1,255),
title='SERC RGB Camera Tile',
cbar='off')'''
plot = plt.imshow(band_array,extent=refl_extent,clim=colorlimit);
if cbar == 'on':
cbar = plt.colorbar(plot,aspect=40); plt.set_cmap(colormap);
cbar.set_label(cmap_title,rotation=90,labelpad=20)
plot_band_array(SERC_RGBcam_array,
SERC_RGBcam_metadata['extent'],
(1,255),
title='SERC RGB Camera Tile',
cbar='off')
plt.savefig('/home/krishna/Desktop/Py_works/geo/PCorrected_reflectance', dpi=100)
plt.show()
How to do that? I tried with cartopy to overlay the grid lines with ax, but that doesn't plotted the GeoTIFF file well. Someone could help me on this?

Python's Basemap doesn't align with correct coordinates?

I have a .dat file containing a list of coordinates (~100k) and a temperature at each coordinate. It has a structure like this:
-59.083 -26.583 0.2
-58.417 -26.250 0.6
-58.412 -26.417 0.4
...
To visually display the temperature ranges, I created a numpy array and plotted the datasets using the Basemap module for Python. The code I wrote is the following:
from matplotlib import pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np
m = Basemap(projection='mill',llcrnrlat=-90,urcrnrlat=90,\
llcrnrlon=-180,urcrnrlon=180,resolution='c')
m.drawcoastlines(linewidth=0.15)
data = np.loadtxt('gridly.dat')
xcoordlist = []
ycoordlist = []
tempvallist = []
for i in data:
xcoord = i[0]
ycoord = i[1]
tempval = i[2]
xcoord2 = xcoord*111139 #<--- Multiplying converts each coordinate's degrees to meters)
ycoord2 = ycoord*111139
xcoordlist.append(xcoord2)
ycoordlist.append(ycoord2)
tempvallist.append(tempval)
xco = np.array(xcoordlist)
yco = np.array(ycoordlist)
tval = np.array(tempvallist)
gridsize = 100
m.hexbin(yco, xco, C=tval, gridsize=gridsize)
cb = m.colorbar()
plt.show()
When I plot the data, I'm getting almost exactly what I want, however, the hexagonal heatmap is offset for some reason, giving me the following chart:
I've been searching online for what might be wrong but unfortunately couldn't find answers or troubleshoot. Does anyone know how I can fix this issue?

After hours of digging around, I finally figured it out! What was wrong with my code was that I was trying to manually convert the geographic coordinates into point coordinates for the displaying chart (by multiplying by 111139).
While the logic for doing this makes sense, I believe this process broke down when I began to plot the data onto different kinds of charts (i.e. orthogonal, miller projection etc.) because the different projections/charts will have different point coordinates (kind of like how the pixel locations on your computer screen may not align with the pixel locations on a different computer screen).
Instead, the Basemap module has a built-in function that will convert real-world coordinates into coordinates that can be plotted on the chart, for you: m(x, y).
So, the improved and correct script would be:
from matplotlib import pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np
m = Basemap(projection='mill',llcrnrlat=-90,urcrnrlat=90,\
llcrnrlon=-180,urcrnrlon=180,resolution='c')
m.drawcoastlines(linewidth=0.15)
data = np.loadtxt('gridly.dat')
xcoordlist = []
ycoordlist = []
tempvallist = []
for i in data:
lat = i[0]
lon = i[1]
tempval = i[2]
xpt, ypt = m(lon, lat)
xcoordlist.append(xpt)
ycoordlist.append(ypt)
tempvallist.append(tempval)
xco = np.array(xcoordlist)
yco = np.array(ycoordlist)
tval = np.array(tempvallist)
gridsize = 100
m.hexbin(xco, yco, C=tval, gridsize=gridsize)
cb = m.colorbar()
plt.show()
As you can see where it says xpt, ypt = m(lon, lat), the function converts the real world longitudes (lon) and latitudes (lat) from the .dat file into pottable points. Hope this helps anyone else that may have this problem in the future!

Plot precipitation data onto a matplotlib basemap map

Inspired by the example Plot precip with filled contours from this website I want to make a plot of yesterday's precipitation data, projected onto a map. The example from that website can, however, no longer be used as the data format of the precipitation data has changed.
My approach is as follows:
download the netCDF4-file from the National Weather Service website
open the netCDF4-file and extract the relevant information
create a map with Basemap
project the precipitation data onto the map
I guess my problem is that I do not understand the netCDF4-file format, and in particular the metadata, since the information about the grid origin of the precipitation data must be hidden somewhere in it.
My code looks as follows:
from datetime import datetime, timedelta
import netCDF4
import numpy as np
import matplotlib.pyplot as plt
import os.path
import urllib
from mpl_toolkits.basemap import Basemap
# set date for precipitation (1 day ago)
precip_date = datetime.utcnow() - timedelta(days=1)
precip_fname = 'nws_precip_1day_{0:%Y%m%d}_conus.nc'.format( precip_date )
precip_url = 'http://water.weather.gov/precip/downloads/{0:%Y/%m/%d}/{1}'.format( precip_date, precip_fname )
# download netCDF4-file if it does not exist already
if not os.path.isfile( precip_fname ):
urllib.urlretrieve( precip_url, precip_fname )
# read netCDF4 dataset and extract relevant data
precip_dSet = netCDF4.Dataset( precip_fname )
# spatial coordinates
precip_x = precip_dSet['x'][:]
precip_y = precip_dSet['y'][:]
# precipitation data (is masked array in netCDF4-dataset)
precip_data = np.ma.getdata( precip_dSet['observation'][:] )
# grid information
precip_lat0 = precip_dSet[ precip_dSet['observation'].grid_mapping ].latitude_of_projection_origin
precip_lon0 = precip_dSet[ precip_dSet['observation'].grid_mapping ].straight_vertical_longitude_from_pole
precip_latts = precip_dSet[ precip_dSet['observation'].grid_mapping ].standard_parallel
# close netCDF4 dataset
precip_dSet.close()
fig1, ax1 = plt.subplots(1,1, figsize=(9,6) )
# create the map
my_map = Basemap( projection='stere', resolution='l',
width=(precip_x.max()-precip_x.min()),
height=(precip_y.max()-precip_y.min()),
lat_0=30, # what is the correct value here?
lon_0=precip_lon0,
lat_ts=precip_latts
)
# white background
my_map.drawmapboundary( fill_color='white' )
# grey coastlines, country borders, state borders
my_map.drawcoastlines( color='0.1' )
my_map.drawcountries( color='0.5' )
my_map.drawstates( color='0.8' )
# contour plot of precipitation data
# create the grid for the precipitation data
precip_lons, precip_lats = my_map.makegrid( precip_x.shape[0], precip_y.shape[0] )
precip_xx, precip_yy = my_map( precip_lons, precip_lats )
# make the contour plot
cont_precip = my_map.contourf( precip_xx, precip_yy, precip_data )
plt.show()
This is how the output looks like (yes, for the final plot the color-levels have to be adjusted):
I know that this is a very specific question, so any suggestions/hints are greatly appreciated.

If I understand correctly you are able to make the plot but want hints on adding extras?
xarray is a fantastic toolbox for working with netCDF files. It works like pandas but for netCDF files and is a big improvement on 'netCDF4':
http://xarray.pydata.org/en/stable/
To specify specific contours you can input the levels:
cont_precip = my_map.contourf( precip_xx, precip_yy, precip_data,levels=[10,20,30]) # Edit for exact contours needed
If you wanted you can add a colorbar:
fig1.colorbar(cont_precip,ax=ax1)

Plotting netCDF data with Python: How to change grid?

I'm an new one in python and plotting data with Matplotlib. I really need help and thank you in advance for the answers.
So, I have a netCDF file with v-component of wind data. Grid coordinates: points=9600 (240x40)
lon : 0 to 358.5 by 1.5 degrees_east circular
lat : 88.5 to 30 by -1.5 degrees_north
My code is:
import numpy as np
import matplotlib
matplotlib.use('Agg')
from netCDF4 import Dataset
from matplotlib.mlab import griddata
from matplotlib import pyplot as plt
from mpl_toolkits.basemap import Basemap
#read data from NETcdf file ".nc"
my_file = '/home/Era-Interim/NH-EraInt-1979.nc'
fh = Dataset(my_file, mode='r')
lons = fh.variables['lon'][:]
lats = fh.variables['lat'][:]
V = fh.variables['V'][:]
V_units = fh.variables['V'].units
fh.close()
# create figure
fig = plt.figure(figsize=(20,20))
# create a map
m = Basemap(projection='nplaea',boundinglat=30,lon_0=10,resolution='l',round=True)
#draw parallels, meridians, coastlines, countries, mapboundary
m.drawcoastlines(linewidth=0.5)
m.drawcountries(linewidth=0.5)
#m.drawmapboundary(linewidth=2)
m.drawparallels(np.arange(30,90,20), labels=[1,1,0,0]) #paral in 10 degree, right, left
m.drawmeridians(np.arange(0,360,30), labels=[1,1,1,1]) #merid in 10 degree, bottom
#Plot the data on top of the map
lon,lat = np.meshgrid(lons,lats)
x,y = m(lon,lat)
cs = m.pcolor(x,y,np.squeeze(V),cmap=plt.cm.RdBu_r)
plt.title("", fontsize=25, verticalalignment='baseline')
plt.savefig("/home/Era-Interim/1.png")
As a result, I received a map (you can find in my dropbox folder) https://www.dropbox.com/sh/nvy8wcodk9jtat0/AAC-omkPP8_7uINSSXbzImeja?dl=0
On the map, there are white pixels between 358.5 and 0 (360) lon, because I have no data between 358.5 and 0 (360) lon.
The question is: how can I change the size of the grid, regrid it, interpolate data, or something else in order to not have this white sector?

I have found a solution. At the beginning of the script, you must add
from mpl_toolkits.basemap import Basemap, addcyclic
and further
datain, lonsin = addcyclic(np.squeeze(Q), lons)
lons, Q = m.shiftdata(lonsin, datain = np.squeeze(Q), lon_0=180.)
print lons
lon, lat = np.meshgrid(lons, lats)
x,y = m(lon, lat)
cs = m.pcolor(x,y,datain,cmap=plt.cm.RdBu_r)
The difference can be seen in the figures (I still can not post images).
https://www.dropbox.com/sh/nvy8wcodk9jtat0/AAC-omkPP8_7uINSSXbzImeja?dl=0

I think in this case some kind of interpolation techniques can be applied.
Check this out. There was similar problem.
Hope it is useful.

The simple answer is 360 degrees is 0 degrees, so you can copy the 0 degrees data and it should look right. I may be interpreting this wrong though, as I believe that the data is representing the pressure levels at each of the points, not between the two points (i.e. at zero degrees, not between zero degrees and 1.5 degrees).
My interpretation means that, yes, you don't have data between 358.5 and 0, but you also don't have data between 357 and 358.5. This seems more likely than just skipping an area. This would mean that the data from 358.5 should be touching the data from 0 as it is just as far away as 0 is from 1.5 which is touching.
Copying the last bit would grant you the ability to change your m.pcolor call to an imshow call (as in Roman Dryndik's link) and use interpolation to smooth out the graph.

How to separate two regions of latlon data on python

I am currently working with BUFR files with wind data. When I read this file on python I get 4 large vectors, latitude vector, longitude vector, wind_direction vector, and wind_speed vector.
Both wind vectors are masked python arrays because there is non-valid data. This happens because the data comes from a non-geostationary satellite. In fact I successfully generated the following image from this BUFR file to show you the general shape that the data takes.
In this image I have plotted a color field to represent the wind speed, while the arrows obviously represent the wind direction.
Please notice the two bands of actual data. Unfortunately the way I am plotting the data, generates a third band (where the color field is smooth), in-between the actual data bands. This is an artefact of the function pcolormesh. If I could superimpose two `pcolormesh plots, each one representing one of the bands, this problem would disappear.
Unfortunately, I do not know how I could separate the data "regions". I have thought about clustering techniques but do not know how to cluster along latlon data using ANOTHER array (the wind data) as the clustering rule.
This is my current code:
#!/usr/bin/python
import bufr
import numpy as np
import sys
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from matplotlib import mlab
WIND_DIR_INDEX = 97
WIND_SPEED_INDEX = 96
bfrfile = sys.argv[1]
print bfrfile
bfr = bufr.BUFRFile(bfrfile)
lon = []
lat = []
wind_d = []
wind_s = []
for record in bfr:
for entry in record:
if entry.index == WIND_DIR_INDEX:
wind_d.append(entry.data)
if entry.index == WIND_SPEED_INDEX:
wind_s.append(entry.data)
if entry.name.find("LONGITUDE") == 0:
lon.append(entry.data)
if entry.name.find("LATITUDE") == 0:
lat.append(entry.data)
lons = np.concatenate(lon)
lats = np.concatenate(lat)
winds_d = np.concatenate(wind_d)
winds_s = np.concatenate(wind_s)
winds_d = np.ma.masked_greater(winds_d,1.0e+6)
winds_s = np.ma.masked_greater(winds_s,1.0e+6)
windu = np.cos((winds_d-180)*(np.pi/180))
windv = np.sin((winds_d-180)*(np.pi/180))
# Data interpolation for pcolormesh (needs gridded data)
xi = np.linspace(lons.min(),lons.max(),lons.size/10)
yi = np.linspace(lats.min(),lats.max(),lats.size/10)
Z = mlab.griddata(lons,lats,winds_s,xi,yi)
X,Y = np.meshgrid(xi,yi)
mydpi = 96
fig = plt.figure(frameon=True)
fig.set_size_inches(1600/mydpi,1200/mydpi)
ax = plt.Axes(fig,[0,0,1,1])
#ax.set_axis_off()
fig.add_axes(ax)
plt.hold(True);
plt.quiver(lons[::5],lats[::5],windu[::5],windv[::5],linewidths=0)
for method in (ax.set_xticks,ax.set_xticklabels,ax.set_yticks,ax.set_yticklabels):
method([])
fig.savefig('/home/cendas/bin/python/bufr_ascat.png',bbox_inches=0,dpi=5*mydpi)
mydpi = 96
fig = plt.figure(frameon=True)
fig.set_size_inches(1600/mydpi,1200/mydpi)
ax = plt.Axes(fig,[0,0,1,1])
#ax.set_axis_off()
fig.add_axes(ax)
plt.hold(True);
try:
plt.pcolormesh(X,Y,Z,alpha=None)
plt.clim(0,10)
except ValueError:
pass
print "Warning: Empty data array."
for method in (ax.set_xticks,ax.set_xticklabels,ax.set_yticks,ax.set_yticklabels):
method([])
fig.savefig('/home/cendas/bin/python/bufr_ascat_color.png',bbox_inches=0,dpi=5*mydpi)
I then usually follow this python code with the following terminal commands to combine the images:
convert bufr_ascat.png -transparent white bufr_ascat.png
convert bufr_ascat_color.png -transparent white bufr_ascat_color.png
composite bufr_ascat.png bufr_ascat_color.png bufrascat.png

Don't abuse clustering for this.
What you need is a simple selection / filtering; not a structure discovery process.
Choose the mean of the masked data. All non-masked data left of that mean is the left part, all non-masked data on the right is the other?
Clustering is the wrong tool for this task.