Develop Reference

My plot is nested in another plot, and negative values are saturated using a diverging colormap

When making a plot using plt.subplot, my plot is nested within another plot, leading to an extra set of axes with ranges 0-1 and cutting off my colorbar.
Additionally, when using a diverging colorbar, all negative values are falsely saturated.
I have used this code format before without having an additional set of axes.
Is there another way to make this kind of plot without using plt.subplot if that is causing the issue?
import numpy as np
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import matplotlib
import cmocean
import h5py
import pandas as pd
import matplotlib.colors as colors
import cartopy
import matplotlib.colormaps as colormaps
july = h5py.File('data/CCS_Colour_Kahru/chl/1km/daily/2016/Daily_Chlorophyll_July.mat','r')
daily_chl_july = july.get('daily_chl')
lat_july = daily_chl_july.get('Latitude')
lon_july = daily_chl_july.get('Longitude')
chl_july = daily_chl_july.get('chl')
date_july = daily_chl_july.get('date')
data_july = daily_chl_july.get('data')
lat_july = np.array(lat_july)
lat_july = lat_july[0]
lon_july = np.array(lon_july)
lon_july = lon_july[0]
chl_july = np.array(chl_july)
inside_lat_july = np.where(lat_july>=(32.9))
lat_july = lat_july[inside_lat_july]
lon_july = lon_july[inside_lat_july]
chl_july = chl_july[inside_lat_july]
inside_lat_july = np.where(lat_july<=(34.05))
lat_july = lat_july[inside_lat_july]
lon_july = lon_july[inside_lat_july]
chl_july = chl_july[inside_lat_july]
inside_lon_july = np.where(lon_july>=(-118.96))
lon_july = lon_july[inside_lon_july]
lat_july = lat_july[inside_lon_july]
chl_july = chl_july[inside_lon_july]
inside_lon_july = np.where(lon_july<=(-117.54))
lon_july = lon_july[inside_lon_july]
lat_july = lat_july[inside_lon_july]
chl_july = chl_july[inside_lon_july]
mean_chl = np.nanmean(chl_july, axis=1)
remove_nan = ~np.isnan(mean_chl)
mean_chl = mean_chl[remove_nan]
chl_july_mean = chl_july[remove_nan]
lat_july_mean = lat_july[remove_nan]
lon_july_mean = lon_july[remove_nan]
for i in range (0,31,1):
day = i+1
chl_now = chl_july_mean[:,i]
remove_nan = ~np.isnan(chl_now)
chl_now_clean1 = chl_now[remove_nan]
lon_now_clean1 = lon_july_mean[remove_nan]
lat_now_clean1 = lat_july_mean[remove_nan]
mean_chl_clean = mean_chl[remove_nan]
if np.size(chl_now_clean1)==0:
continue
anomaly = chl_now_clean1 - mean_chl_clean
print('min')
print(anomaly.min())
print('max')
print(anomaly.max())
fig, ax = plt.subplots(1,1,figsize=(15,10))
ax = plt.axes(projection=ccrs.PlateCarree())
ax.set_extent([-118.96,-117.54,32.9,34.05])
im = ax.scatter(lon_now_clean1, lat_now_clean1, norm=colors.TwoSlopeNorm(vmin=-20, vcenter=0, vmax=20), c=anomaly, cmap='PiYG')
ax.add_feature(cfeature.LAND,zorder=4,color='gray')
ax.set_title("Chlorophyll Anomaly 2016 July "+str(day),fontsize=25)
ax.set_xlabel("Longitude",fontsize=20)
ax.set_ylabel("Latitude",fontsize=20)
xticks = np.linspace(-118.96,-117.54,5)
yticks = np.linspace(32.9,34.05,5)
ax.set_xticks(ticks=xticks)
ax.set_yticks(ticks=yticks)
fig.colorbar(im, ax=ax)
im.set_clim(0.1,1.5)
ax.coastlines()
plt.savefig('frankieleelopez/0124anomaly72016'+str(day))
plt.close()
enter image description here

Plotting a large database in Basemap: Memory error

Hi I extract the data from an interpolation (The data are in the Basemap Grid) with the command and save them as CSV:
def inter_todf(interpolation, grid):
grid['x'], grid['y'] = basemap(grid['x'],grid['y'],inverse=True) # Wandelt Grid in Long und Lat wieder um
dfl = pd.DataFrame({
'Latitude': grid['y'].reshape(-1),
'Longitude': grid['x'].reshape(-1),
'Value': interpolation.reshape(-1)
});
return(dfl)
dfl= inter_todf(interpolation, grid)
dfl.to_csv(plot_folder+'Dezember/'+file.replace(".csv", "grid.csv"))
Afterwards I want to plot them again in another file in Basemap (so it is still planned that I modify the data but that is something else). I do the plotting with the code:
from traceback import print_tb
import numpy as np
from pykrige.ok import OrdinaryKriging
from pykrige.kriging_tools import write_asc_grid
import pykrige.kriging_tools as kt
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib.patches import Path, PathPatch
import pandas as pd
def load_data():
df = pd.read_csv(r"GridFile.csv")
return(df)
def get_data(df):
return {
"lons": df['Longitude'].values.reshape(52920,),
"lats": df['Latitude'].values.reshape(52920,),
"values": df['Value'].values.reshape(52920,)
}
def extend_data(data):
return {
"lons": np.concatenate([np.array([lon-360 for lon in data["lons"]]), data["lons"], np.array([lon+360 for lon in data["lons"]])]),
"lats": np.concatenate([data["lats"], data["lats"], data["lats"]]),
"values": np.concatenate([data["values"], data["values"], data["values"]]),
}
def generate_grid(data, basemap, delta=1):
grid = {
'lon': np.arange(-180, 180, delta),
'lat': np.arange(-89.9, 89.9, delta)
}
grid["x"], grid["y"] = np.meshgrid(grid["lon"], grid["lat"])
grid["x"], grid["y"] = basemap(grid["x"], grid["y"])
return grid
def interpolate(data, grid):
OK =OrdinaryKriging(
data["lons"],
data["lats"],
data["values"],
variogram_model='exponential',
)
return OK.execute("grid", grid["lon"], grid["lat"])
def prepare_map_plot():
figure, axes = plt.subplots(figsize=(10,10))
basemap = Basemap(projection='robin', lon_0=0, lat_0=0, resolution='l',area_thresh=1000000,ax=axes)
return figure, axes, basemap
def plot_mesh_data(interpolation, grid, basemap):
colormesh = basemap.contourf(grid["x"], grid["y"], interpolation,32, cmap='RdBu_r', ) #plot the data on the map. plt.cm.RdYlBu_r
color_bar = basemap.colorbar(colormesh,location='bottom',pad="10%")
df = load_data()
base_data = get_data(df)
# print(df['Latitude'].shape)
figure, axes, basemap = prepare_map_plot()
grid = generate_grid(base_data, basemap, 40)
extended_data = extend_data(base_data)
interpolation, interpolation_error = interpolate(extended_data, grid)
plot_mesh_data(interpolation, grid,basemap)
plt.show()
Unfortunately I get an error message:
numpy.core._exceptions.MemoryError: Unable to allocate 93.9 GiB for an array with shape (12602289420,) and data type float64
How do I have to change my data so that it no longer uses so much memory?

How to merge multiple MODIS swaths into one plot in python?

I want to mosaic/merge multiple swaths of the MODIS dataset (MOD06_L2) using python. I used the example (http://hdfeos.org/zoo/MORE/LAADS/MOD/MOD04_L2_merge.py) to read multiple files and merge. But I am getting an error while doing so, how to correct this error?
I would like to know is there any better way than this, to merge/mosaic MODIS HDF files into one?
import os
import glob
import matplotlib as mpl
import matplotlib.pyplot as plt
# import cartopy.crs as ccrs
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np
# The first file in 3 swath files.
FILE_NAME = 'MOD06_L2.A2017126.0655.061.2019226193408.hdf'
GEO_FILE_NAME ='MOD06_L2.A2017126.0655.061.2019226193408.hdf'
DATAFIELD_NAME = 'Brightness_Temperature'
from pyhdf.SD import SD, SDC
i = 0
for file in list(glob.glob('MOD06*.hdf')):
reader = open(file)
hdf = SD(file, SDC.READ)
# Read dataset.
data2D = hdf.select(DATAFIELD_NAME)
data = data2D[:,:].astype(np.double)
hdf_geo = SD(GEO_FILE_NAME, SDC.READ)
# Read geolocation dataset.
lat = hdf_geo.select('Latitude')
latitude = lat[:,:]
lon = hdf_geo.select('Longitude')
longitude = lon[:,:]
# Retrieve attributes.
attrs = data2D.attributes(full=1)
lna=attrs["long_name"]
long_name = lna[0]
aoa=attrs["add_offset"]
add_offset = aoa[0]
fva=attrs["_FillValue"]
_FillValue = fva[0]
sfa=attrs["scale_factor"]
scale_factor = sfa[0]
vra=attrs["valid_range"]
valid_min = vra[0][0]
valid_max = vra[0][1]
ua=attrs["units"]
units = ua[0]
invalid = np.logical_or(data > valid_max,data < valid_min)
invalid = np.logical_or(invalid, data == _FillValue)
data[invalid] = np.nan
data = (data - add_offset) * scale_factor
datam = np.ma.masked_array(data, np.isnan(data))
if i == 0 :
data_m = datam
latitude_m = latitude
longitude_m = longitude
else:
data_m = np.vstack([data_m, datam])
latitude_m = np.vstack([latitude_m, latitude])
longitude_m = np.vstack([longitude_m, longitude])
i = i + 1
m = Basemap(projection='cyl', resolution='l',
llcrnrlat=-90, urcrnrlat=90,
llcrnrlon=-180, urcrnrlon=180)
m.drawcoastlines(linewidth=0.5)
m.drawparallels(np.arange(-90, 91, 45))
m.drawmeridians(np.arange(-180, 180, 45), labels=[True,False,False,True])
sc = m.scatter(longitude_m, latitude_m, c=data_m, s=0.1, cmap=plt.cm.jet,
edgecolors=None, linewidth=0)
cb = m.colorbar()
cb.set_label(units)
# Put title using the first file.
basename = os.path.basename(FILE_NAME)
plt.title('{0}\n{1}'.format(basename, DATAFIELD_NAME))
fig = plt.gcf()
# Save image.
pngfile = "{0}.py.png".format(basename)
fig.savefig(pngfile)
It showing an error
ValueError: 'c' argument has 4604040 elements, which is inconsistent with 'x' and 'y' with size 657720.

Optimisation of plot animation for large data with dynamic grid

I am trying to generate an animation for a large data with a dynamic grid (ocean waves). I have managed to write a script that is functional but it is time and resource consuming. I was hoping if anyone could see what i can improve in my code to help speed it up.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
from matplotlib import animation as anim
import xarray as xr
import PySimpleGUI as sg
import sys
#file importing mechanism
jan = xr.open_dataset('Model/Bar_mig/xboutput_equi.nc')
########################## labeling the variables in the data-set ##############
nx = jan.variables['globalx']
globaltime = jan.variables['globaltime']
zb = jan.variables['zb'][:,0,:]
zs = jan.variables['zs'][:,0,:]
ccz = jan.variables['ccz'][:,:,0,:]
uz = jan.variables['uz'][:,:,0,:]
nz = np.array(range(0,100))
nx = (np.array(range(0,nx.size)))
globaltime_ar = np.array(globaltime)
conc = np.vstack(globaltime_ar)
newdf = pd.DataFrame(conc)
itr = len(newdf.index)
uz1=np.flip(uz,0)
uz2=np.flip(np.flip(uz1,1),axis=0)
depth1 = (zb)
a = np.array(depth1)
b = pd.DataFrame(a)
depth = b.dropna(axis=1, how='all')
zba1 = (np.array(zb))
zsa1 = (np.array(-zs))
zba = pd.DataFrame(zba1)
zsa = pd.DataFrame(zsa1)
This is how i am setting up the dynamic grid. ( example of the the output)
#dynamic grid
for w in range(0,itr):
AA=[]
sizer = depth.iloc[w,]
sizer1 = sizer.dropna(axis=0, how='all')
for j in range(0,sizer1.size):
maxi = -zsa.iloc[w,j]
mini = depth.iloc[w,j]
step = mini/nz.shape[-1]
globals()['col_{}'.format(j)] = pd.DataFrame(np.linspace(maxi,mini,nz.shape[-1],endpoint=True))
globals()['col_{}'.format(j)] = globals()['col_{}'.format(j)].reset_index(drop=True)
AA.append(globals()['col_{}'.format(j)])
globals()['df_{}'.format(w)] = pd.concat(AA, axis=1).iloc[:nz.size]
globals()['df_{}'.format(w)].columns = range(globals()['df_{}'.format(w)].shape[1])
AA.clear()
sg.OneLineProgressMeter('My meter title', w, itr-1, 'key')
from matplotlib import animation as anim
fig = plt.figure(figsize=(15,7.5)) # Create a dummy figure
ax = plt.axes() # Set the axis rigid
mywriter = anim.FFMpegWriter()
scale=1
def animate(w):
w = w*scale
plt.clf()
plt.title(str(w) + 'hr')
y = globals()['df_{}'.format(w)]
x = np.array([nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx,nx])
data2 = np.flip(ccz[w*1,:,:],0)
cont = plt.pcolor(x,y,np.array(data2), cmap = 'jet',
vmin = 0, vmax = 0.03
)
plt.colorbar(label='Concentration Profile ($m^3/m^3$)')
plt.fill_between(nx,min(zb[0])-1,zb[w],color = 'yellow')
point = 350
plt.xlim(point,nx.shape[-1])
plt.ylim(min(zb[0,point:point+1]),max(zb[0,point:]))
plt.xlabel('Cross shore distance (m)')
plt.ylabel('Depth (m)')
fig.tight_layout()
return cont,
ani = anim.FuncAnimation(fig, animate, interval = 1, frames=itr)
ani.save('Sed_Con.mp4', writer=mywriter)

How do I animate a scatterplot over a basemap in matplotlib?

The code below generates a animated basemap, but not exactly the one I want: I want the scatterplot from the previous frame to disappear, but it persists through the remainder of the animation.
I suspect it has something to do with my not understanding what the basemap really is. I understand calling it on lat/lons to project them to x/y, but I don't entirely get what's going on when I call event_map.scatter().
import random
import os
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from matplotlib import animation
import pandas as pd
from IPython.display import HTML
# Enables animation display directly in IPython
#(http://jakevdp.github.io/blog/2013/05/12/embedding-matplotlib-animations/)
from tempfile import NamedTemporaryFile
VIDEO_TAG = """<video controls>
<source src="data:video/x-m4v;base64,{0}" type="video/mp4">
Your browser does not support the video tag.
</video>"""
def anim_to_html(anim):
if not hasattr(anim, '_encoded_video'):
with NamedTemporaryFile(suffix='.mp4') as f:
anim.save(f.name, fps=20, extra_args=['-vcodec', 'libx264'])
video = open(f.name, "rb").read()
anim._encoded_video = video.encode("base64")
return VIDEO_TAG.format(anim._encoded_video)
def display_animation(anim):
plt.close(anim._fig)
return HTML(anim_to_html(anim))
animation.Animation._repr_html_ = anim_to_html
FRAMES = 20
POINTS_PER_FRAME = 30
LAT_MIN = 40.5
LAT_MAX = 40.95
LON_MIN = -74.15
LON_MAX = -73.85
FIGSIZE = (10,10)
MAP_BACKGROUND = '.95'
MARKERSIZE = 20
#Make Sample Data
data_frames = {}
for i in range(FRAMES):
lats = [random.uniform(LAT_MIN, LAT_MAX) for x in range(POINTS_PER_FRAME)]
lons = [random.uniform(LON_MIN, LON_MAX) for x in range(POINTS_PER_FRAME)]
data_frames[i] = pd.DataFrame({'lat':lats, 'lon':lons})
class AnimatedMap(object):
""" An animated scatter plot over a basemap"""
def __init__(self, data_frames):
self.dfs = data_frames
self.fig = plt.figure(figsize=FIGSIZE)
self.event_map = Basemap(projection='merc',
resolution='i', area_thresh=1.0, # Medium resolution
lat_0 = (LAT_MIN + LAT_MAX)/2, lon_0=(LON_MIN + LON_MAX)/2, # Map center
llcrnrlon=LON_MIN, llcrnrlat=LAT_MIN, # Lower left corner
urcrnrlon=LON_MAX, urcrnrlat=LAT_MAX) # Upper right corner
self.ani = animation.FuncAnimation(self.fig, self.update, frames=FRAMES, interval=1000,
init_func=self.setup_plot, blit=True,
repeat=False)
def setup_plot(self):
self.event_map.drawcoastlines()
self.event_map.drawcounties()
self.event_map.fillcontinents(color=MAP_BACKGROUND) # Light gray
self.event_map.drawmapboundary()
self.scat = self.event_map.scatter(x = [], y=[], s=MARKERSIZE,marker='o', zorder=10)
return self.scat
def project_lat_lons(self, i):
df = data_frames[i]
x, y = self.event_map(df.lon.values, df.lat.values)
x_y = pd.DataFrame({'x': x, 'y': y}, index=df.index)
df = df.join(x_y)
return df
def update(self, i):
"""Update the scatter plot."""
df = self.project_lat_lons(i)
self.scat = self.event_map.scatter(x = df.x.values, y=df.y.values, marker='o', zorder=10)
return self.scat,
s = AnimatedMap(data_frames)
s.ani

It looks like you're simply adding a new scatter plot at each update. What you should do instead is change the data in the existing path collection at each update. Try something along the lines of
def update(self, i):
"""Update the scatter plot."""
df = self.project_lat_lons(i)
new_offsets = np.vstack([df.x.values, df.y.values]).T
self.scat.set_offsets(new_offsets)
return self.scat,
Note that I haven't tested this.