Develop Reference

plot power spectrum of a signal with wavelet transform

Using this dataset https://philharmonia.co.uk/resources/sound-samples/ I'm trying to plot the power spectrum of note played by a specific instrument.
I'm using librosa to load the audio file and get some information with this code
import librosa
import numpy as np
import matplotlib.pyplot as plt
import pywt
y, sr = librosa.load(file_path)
duration = librosa.get_duration(y=y, sr=sr)
delta_t = duration / len(y)
t0=0
time = np.arange(0, len(y)) * delta_t + t0
I'm also following this https://ataspinar.com/2018/12/21/a-guide-for-using-the-wavelet-transform-in-machine-learning/ guide to plot the power spectrum and I'm using pywavelet library.
The problem that I have with this code is RuntimeWarning: divide by zero encountered in log2 and the plot is not shown.
def plot_wavelet(time, signal, scales,
waveletname = 'cmor',
cmap = plt.cm.seismic,
title = 'Wavelet Transform (Power Spectrum) of signal',
ylabel = 'Period (years)',
xlabel = 'Time'):
dt = time[1] - time[0]
print("dt ", dt)
[coefficients, frequencies] = pywt.cwt(signal, scales, waveletname, dt)
power = (abs(coefficients)) ** 2
period = 1. / frequencies
levels = [0.0625, 0.125, 0.25, 0.5, 1, 2, 4, 8]
contourlevels = np.log2(levels)
fig, ax = plt.subplots(figsize=(15, 10))
im = ax.contourf(time, np.log2(period), np.log2(power), contourlevels, extend='both',cmap=cmap)
ax.set_title(title, fontsize=20)
ax.set_ylabel(ylabel, fontsize=18)
ax.set_xlabel(xlabel, fontsize=18)
yticks = 2**np.arange(np.ceil(np.log2(period.min())), np.ceil(np.log2(period.max())))
ax.set_yticks(np.log2(yticks))
ax.set_yticklabels(yticks)
ax.invert_yaxis()
ylim = ax.get_ylim()
ax.set_ylim(ylim[0], -1)
cbar_ax = fig.add_axes([0.95, 0.5, 0.03, 0.25])
fig.colorbar(im, cax=cbar_ax, orientation="vertical")
plt.show()
scales = np.arange(1, 128)
plot_wavelet(time=time, signal=y, scales=scales, waveletname='gaus5')
Noting that some values in the power array are at -inf.
How can I solve?

How to evenly spread ticks on a colorbar?

I have been trying to make a plot with some evenly spaced tick in my colorbar, but so far my results always give me a colorbar with the distance between the ticks proportional to their values as shown in the image below:
import numpy as np
import matplotlib
import matplotlib as plt
T= [0.01, 0.02, 0.03, 0.04] #values for the colourbar to use in equation in for loop
x=np.linspace[0, 8, 100]
e=1/(np.exp(x)+1) #factor used in equation dependent on the x-axis values
a=6.4*10**(-9)
b= 1.51 # constants for the equation
pof6= [number **6 for number in T]
norm = matplotlib.colors.Normalize(vmin=np.min(pof6), vmax=np.max(pof6)) #colourbar max and min values
c_m = matplotlib.cm.cool
s_m = matplotlib.cm.ScalarMappable(cmap='jet', norm=norm)
s_m.set_array([])
#below is the for loop that uses one value of T at a time, represented as t in the equation
for t in pof6:
plt.plot(x, b*x/(((a*t*x**2/(m**2))+1)**2)*e, color=s_m.to_rgba(t))
func = lambda x,pos: "{:g}".format(x)
fmt = matplotlib.ticker.FuncFormatter(func)
c_bar=plt.colorbar(s_m, format=fmt, ticks=[0.01**6,0.02* 0.03**6, 0.04**6])
plt.legend()
plt.xlabel('y=E/T')
plt.ylabel('$f_{ν_s}$')
c_bar.set_label(r'T(K)')
plt.show()
I have attempted applying some of the solutions suggested here n=on the website, like Spread custom tick labels evenly over colorbar but haven't been successful at that.

You're using a linear norm, where the pof values are very close to each other. It helps to use a LogNorm. The tick formatter can be adapted to show the values in their **6 format.
The code below shifts the four functions a bit, because with the code from the example all plots seem to coincide. At least when I use something like m=2 (m is not defined in the code).
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import colors as mcolors
from matplotlib import ticker as mticker
T = [0.01, 0.02, 0.03, 0.04] # values for the colourbar to use in equation in for loop
x = np.linspace(0, 8, 100)
e = 1 / (np.exp(x) + 1) # factor used in equation dependent on the x-axis values
a = 6.4 * 10 ** (-9)
b = 1.51 # constants for the equation
pof6 = [number ** 6 for number in T]
norm = mcolors.LogNorm(vmin=np.min(pof6), vmax=np.max(pof6)) # colourbar max and min values
s_m = plt.cm.ScalarMappable(cmap='jet', norm=norm)
s_m.set_array([])
m = 2
for t in pof6:
plt.plot(x, b * x / (((a * t * x ** 2 / (m ** 2)) + 1) ** 2) * e + 10*t**(1/6), color=s_m.to_rgba(t))
func = lambda x, pos: "{:g}**6".format(x**(1/6) )
fmt = mticker.FuncFormatter(func)
c_bar = plt.colorbar(s_m, format=fmt, ticks=pof6)
c_bar.set_label(r'T(K)')
# plt.legend() # there are no labels set, so a default legend can't be created
plt.xlabel('y=E/T')
plt.ylabel('$f_{ν_s}$')
plt.show()
If you want a legend, you need to put a label to each curve, for example:
for t in pof6:
plt.plot(x, b * x / (((a * t * x ** 2 / (m ** 2)) + 1) ** 2) * e, color=s_m.to_rgba(t),
label=f'$t = {t**(1/6):g}^6$')
plt.legend()

Calculating PDF given a histogram

I have a heavily right-skewed histogram and would like to calculate the probabilities for a range of Lifetimevalues (Area under the curve, the PDF). For instance, the probability that the Lifetime value is in (0-0.01)
Dataframe consisting of LTV calculated by cumulative revenue/ cumulative installs:
df['LTV'] is
(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0.208125,0.0558879,0.608348,0.212553,0.0865896,
0.728542,0,0.609512,0,0,0,0,0,0,0,0.0801339,0.140657,0.0194118,0,0,0.0634682,
0.339545,0.875902,0.8325,0.0260526,0.0711905,0.169894,0.202969,0.0761538,0,0.342055,
0.42781,0,0,0.192115,0,0,0,0,0,0,0,0,0,0,0,1.6473,0,0.232329,0,2.21329,0.748,0.0424286,
0.455439,0.210282,5.56453,0.427959,0,0.352059,0,0,0.567059,0,0,0,0.384462,1.29476,
0.0103125,0,0.0126923,1.03356,0,0,0.289785,0,0)
I have tried utilizing SKlearn's KernelDensity, however, after fitting it to the histogram it does not capture the over-represented 0s.
import gc
from sklearn.neighbors import KernelDensity
def plot_prob_density(df_lunch, field, x_start, x_end):
plt.figure(figsize = (10, 7))
unit = 0
x = np.linspace(df_lunch.min() - unit, df_lunch.max() + unit, 1000)[:, np.newaxis]
# Plot the data using a normalized histogram
plt.hist(df_lunch, bins=200, density=True, label='LTV', color='blue', alpha=0.2)
# Do kernel density estimation
kd_lunch = KernelDensity(kernel='gaussian', bandwidth=0.00187).fit(df_lunch) #0.00187
# Plot the estimated densty
kd_vals_lunch = np.exp(kd_lunch.score_samples(x))
plt.plot(x, kd_vals_lunch, color='orange')
plt.axvline(x=x_start,color='red',linestyle='dashed')
plt.axvline(x=x_end,color='red',linestyle='dashed')
# Show the plots
plt.xlabel(field, fontsize=15)
plt.ylabel('Probability Density', fontsize=15)
plt.legend(fontsize=15)
plt.show()
gc.collect()
return kd_lunch
kd_lunch = plot_prob_density(final_df['LTV'].values.reshape(-1,1), 'LTV', x_start=0, x_end=0.01)
Then finding the probabilities like this:
def get_probability(start_value, end_value, eval_points, kd):
# Number of evaluation points
N = eval_points
step = (end_value - start_value) / (N - 1) # Step size
x = np.linspace(start_value, end_value, N)[:, np.newaxis] # Generate values in the range
kd_vals = np.exp(kd.score_samples(x)) # Get PDF values for each x
probability = np.sum(kd_vals * step) # Approximate the integral of the PDF
return probability.round(4)
print('Probability of LTV 0-3 tips during LUNCH time: {}\n'
.format(get_probability(start_value = 0,
end_value = 0.01,
eval_points = 100,
kd = kd_lunch)))
However, this method does not yield the appropriate PDF values we were aiming for.
Any suggestions for alternative methods would be appreciated.
PLot:

I have used more or less similar script for my work, here is my script may be it will be helpful for you.
import gc
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.neighbors import KernelDensity
from scipy import stats
data1 = beta_95[0]
def plot_prob_density(data1, x_start, x_end):
plt.figure(figsize = (4, 3.5))
unit = 1.5
x = np.linspace(-20, 20, 1000)[:, np.newaxis]
# Plot the data using a normalized histogram
plt.hist(data1, bins=np.linspace(-20,20,40), density=True, color='r', alpha=0.4)
#plt.show
# Do kernel density estimation
kd_data1 = KernelDensity(kernel='gaussian', bandwidth=1.8).fit(data1)
# Plot the estimated densty
kd_vals_data1 = np.exp(kd_data1.score_samples(x))
plt.plot(x, kd_vals_data1, color='r', label='$N_a$', linewidth = 2)
plt.axvline(x=9.95,color='green',linestyle='dashed', linewidth = 2.0, label='$β_o$')
plt.axvline(x=1.9,color='black',linestyle='dashed', linewidth = 2.0, label='$β_b$')
plt.axvline(x=x_end,color='red',linestyle='dashed', linewidth = 2, label='$β_{95\%}$')
# Show the plots
plt.xlabel('Beta', fontsize=10)
plt.ylabel('Probability Density', fontsize=10)
plt.title('02 hours window', fontsize=12)
plt.xlim(-20, 20)
plt.ylim(0, 0.3)
plt.yticks([0, 0.1, 0.2, 0.3])
plt.legend(fontsize=12, loc='upper left', frameon=False)
plt.show()
gc.collect()
return kd_data1
def get_probability(start_value, end_value, eval_points, kd):
# Number of evaluation points
N = eval_points
step = (end_value - start_value) / (N - 1) # Step size
x = np.linspace(start_value, end_value, N)[:, np.newaxis] # Generate values in the range
kd_vals = np.exp(kd.score_samples(x)) # Get PDF values for each x
probability = np.sum(kd_vals * step) # Approximate the integral of the PDF
return probability.round(4)
data1 = np.array(data1).reshape(-1, 1)
kd_data1 = plot_prob_density(data1, x_start=3.0, x_end=13)
print('Beta-95%: {}\n'
.format(get_probability(start_value = -10,
end_value = 13,
eval_points = 1000,
kd = kd_data1)))

Make dots in matplotlib plots selectable by mouse

scikit-learn has a very nice demo that creates an outlier analysis tool. Here is the
import numpy as np
import pylab as pl
import matplotlib.font_manager
from scipy import stats
from sklearn import svm
from sklearn.covariance import EllipticEnvelope
# Example settings
n_samples = 200
outliers_fraction = 0.25
clusters_separation = [0, 1, 2]
# define two outlier detection tools to be compared
classifiers = {
"One-Class SVM": svm.OneClassSVM(nu=0.95 * outliers_fraction + 0.05,
kernel="rbf", gamma=0.1),
"robust covariance estimator": EllipticEnvelope(contamination=.1)}
# Compare given classifiers under given settings
xx, yy = np.meshgrid(np.linspace(-7, 7, 500), np.linspace(-7, 7, 500))
n_inliers = int((1. - outliers_fraction) * n_samples)
n_outliers = int(outliers_fraction * n_samples)
ground_truth = np.ones(n_samples, dtype=int)
ground_truth[-n_outliers:] = 0
# Fit the problem with varying cluster separation
for i, offset in enumerate(clusters_separation):
np.random.seed(42)
# Data generation
X1 = 0.3 * np.random.randn(0.5 * n_inliers, 2) - offset
X2 = 0.3 * np.random.randn(0.5 * n_inliers, 2) + offset
X = np.r_[X1, X2]
# Add outliers
X = np.r_[X, np.random.uniform(low=-6, high=6, size=(n_outliers, 2))]
# Fit the model with the One-Class SVM
pl.figure(figsize=(10, 5))
for i, (clf_name, clf) in enumerate(classifiers.items()):
# fit the data and tag outliers
clf.fit(X)
y_pred = clf.decision_function(X).ravel()
threshold = stats.scoreatpercentile(y_pred,
100 * outliers_fraction)
y_pred = y_pred > threshold
n_errors = (y_pred != ground_truth).sum()
# plot the levels lines and the points
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
subplot = pl.subplot(1, 2, i + 1)
subplot.set_title("Outlier detection")
subplot.contourf(xx, yy, Z, levels=np.linspace(Z.min(), threshold, 7),
cmap=pl.cm.Blues_r)
a = subplot.contour(xx, yy, Z, levels=[threshold],
linewidths=2, colors='red')
subplot.contourf(xx, yy, Z, levels=[threshold, Z.max()],
colors='orange')
b = subplot.scatter(X[:-n_outliers, 0], X[:-n_outliers, 1], c='white')
c = subplot.scatter(X[-n_outliers:, 0], X[-n_outliers:, 1], c='black')
subplot.axis('tight')
subplot.legend(
[a.collections[0], b, c],
['learned decision function', 'true inliers', 'true outliers'],
prop=matplotlib.font_manager.FontProperties(size=11))
subplot.set_xlabel("%d. %s (errors: %d)" % (i + 1, clf_name, n_errors))
subplot.set_xlim((-7, 7))
subplot.set_ylim((-7, 7))
pl.subplots_adjust(0.04, 0.1, 0.96, 0.94, 0.1, 0.26)
pl.show()
And here is what it looks like:
Is that cool or what?
However, I want the plot to be mouse-sensitive. That is, I want to be able to click on dots and find out what they are, with either a tool-tip or with a pop-up window, or something in a scroller. And I'd also like to be able to click-to-zoom, rather than zoom with a bounding box.
Is there any way to do this?

Not to plug my own project to much, but have a look at mpldatacursor. If you'd prefer, it's also quite easy to implement from scratch.
As a quick example:
import matplotlib.pyplot as plt
import numpy as np
from mpldatacursor import datacursor
x1, y1 = np.random.random((2, 5))
x2, y2 = np.random.random((2, 5))
fig, ax = plt.subplots()
ax.plot(x1, y1, 'ro', markersize=12, label='Series A')
ax.plot(x2, y2, 'bo', markersize=12, label='Series B')
ax.legend()
datacursor()
plt.show()
For this to work with the example code you posted, you'd need to change things slightly. As it is, the artist labels are set in the call to legend, instead of when the artist is created. This means that there's no way to retrieve what's displayed in the legend for a particular artist. All you'd need to do is just pass in the labels as a kwarg to scatter instead of as the second argument to legend, and things should work as you were wanting.

How to plot normal distribution

Given a mean and a variance is there a simple function call which will plot a normal distribution?

import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats
import math
mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 3*sigma, mu + 3*sigma, 100)
plt.plot(x, stats.norm.pdf(x, mu, sigma))
plt.show()

I don't think there is a function that does all that in a single call. However you can find the Gaussian probability density function in scipy.stats.
So the simplest way I could come up with is:
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
# Plot between -10 and 10 with .001 steps.
x_axis = np.arange(-10, 10, 0.001)
# Mean = 0, SD = 2.
plt.plot(x_axis, norm.pdf(x_axis,0,2))
plt.show()
Sources:
http://www.johndcook.com/distributions_scipy.html
http://docs.scipy.org/doc/scipy/reference/stats.html
http://telliott99.blogspot.com/2010/02/plotting-normal-distribution-with.html

Use seaborn instead
i am using distplot of seaborn with mean=5 std=3 of 1000 values
value = np.random.normal(loc=5,scale=3,size=1000)
sns.distplot(value)
You will get a normal distribution curve

If you prefer to use a step by step approach you could consider a solution like follows
import numpy as np
import matplotlib.pyplot as plt
mean = 0; std = 1; variance = np.square(std)
x = np.arange(-5,5,.01)
f = np.exp(-np.square(x-mean)/2*variance)/(np.sqrt(2*np.pi*variance))
plt.plot(x,f)
plt.ylabel('gaussian distribution')
plt.show()

Unutbu answer is correct.
But because our mean can be more or less than zero I would still like to change this :
x = np.linspace(-3 * sigma, 3 * sigma, 100)
to this :
x = np.linspace(-3 * sigma + mean, 3 * sigma + mean, 100)

I believe that is important to set the height, so created this function:
def my_gauss(x, sigma=1, h=1, mid=0):
from math import exp, pow
variance = pow(sigma, 2)
return h * exp(-pow(x-mid, 2)/(2*variance))
Where sigma is the standard deviation, h is the height and mid is the mean.
To:
plt.close("all")
x = np.linspace(-20, 20, 101)
yg = [my_gauss(xi) for xi in x]
Here is the result using different heights and deviations:

I have just come back to this and I had to install scipy as matplotlib.mlab gave me the error message MatplotlibDeprecationWarning: scipy.stats.norm.pdf when trying example above. So the sample is now:
%matplotlib inline
import math
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats
mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 3*sigma, mu + 3*sigma, 100)
plt.plot(x, scipy.stats.norm.pdf(x, mu, sigma))
plt.show()

you can get cdf easily. so pdf via cdf
import numpy as np
import matplotlib.pyplot as plt
import scipy.interpolate
import scipy.stats
def setGridLine(ax):
#http://jonathansoma.com/lede/data-studio/matplotlib/adding-grid-lines-to-a-matplotlib-chart/
ax.set_axisbelow(True)
ax.minorticks_on()
ax.grid(which='major', linestyle='-', linewidth=0.5, color='grey')
ax.grid(which='minor', linestyle=':', linewidth=0.5, color='#a6a6a6')
ax.tick_params(which='both', # Options for both major and minor ticks
top=False, # turn off top ticks
left=False, # turn off left ticks
right=False, # turn off right ticks
bottom=False) # turn off bottom ticks
data1 = np.random.normal(0,1,1000000)
x=np.sort(data1)
y=np.arange(x.shape[0])/(x.shape[0]+1)
f2 = scipy.interpolate.interp1d(x, y,kind='linear')
x2 = np.linspace(x[0],x[-1],1001)
y2 = f2(x2)
y2b = np.diff(y2)/np.diff(x2)
x2b=(x2[1:]+x2[:-1])/2.
f3 = scipy.interpolate.interp1d(x, y,kind='cubic')
x3 = np.linspace(x[0],x[-1],1001)
y3 = f3(x3)
y3b = np.diff(y3)/np.diff(x3)
x3b=(x3[1:]+x3[:-1])/2.
bins=np.arange(-4,4,0.1)
bins_centers=0.5*(bins[1:]+bins[:-1])
cdf = scipy.stats.norm.cdf(bins_centers)
pdf = scipy.stats.norm.pdf(bins_centers)
plt.rcParams["font.size"] = 18
fig, ax = plt.subplots(3,1,figsize=(10,16))
ax[0].set_title("cdf")
ax[0].plot(x,y,label="data")
ax[0].plot(x2,y2,label="linear")
ax[0].plot(x3,y3,label="cubic")
ax[0].plot(bins_centers,cdf,label="ans")
ax[1].set_title("pdf:linear")
ax[1].plot(x2b,y2b,label="linear")
ax[1].plot(bins_centers,pdf,label="ans")
ax[2].set_title("pdf:cubic")
ax[2].plot(x3b,y3b,label="cubic")
ax[2].plot(bins_centers,pdf,label="ans")
for idx in range(3):
ax[idx].legend()
setGridLine(ax[idx])
plt.show()
plt.clf()
plt.close()

import math
import matplotlib.pyplot as plt
import numpy
import pandas as pd
def normal_pdf(x, mu=0, sigma=1):
sqrt_two_pi = math.sqrt(math.pi * 2)
return math.exp(-(x - mu) ** 2 / 2 / sigma ** 2) / (sqrt_two_pi * sigma)
df = pd.DataFrame({'x1': numpy.arange(-10, 10, 0.1), 'y1': map(normal_pdf, numpy.arange(-10, 10, 0.1))})
plt.plot('x1', 'y1', data=df, marker='o', markerfacecolor='blue', markersize=5, color='skyblue', linewidth=1)
plt.show()

For me, this worked pretty well if you are trying to plot a particular pdf
theta1 = {
"a": 0.5,
"cov" : 1,
"mean" : 0
}
x = np.linspace(start = 0, stop = 1000, num = 1000)
pdf = stats.norm.pdf(x, theta1['mean'], theta1['cov']) + theta2['a']
sns.lineplot(x,pdf)