Develop Reference

How to count number of points above a least square fit?

I want to count the points above the least squares fits.
from sklearn.mixture import GaussianMixture
from sklearn import preprocessing
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from scipy import stats
from astropy.io import ascii
from scipy.stats import norm
from astropy.timeseries import LombScargle
from astropy import stats
data4= pd.read_csv('Standard Dev main pop.csv')
names4 = data4.columns
df3 = pd.DataFrame(data4, columns=names4)
df3.head()
#print(c)
data5= pd.read_csv('2 Sigma Main pop.csv')
names5 = data5.columns
df5 = pd.DataFrame(data5, columns=names5)
df5.head()
data6= pd.read_csv('3 Sigma main pop.csv')
names6 = data6.columns
df6 = pd.DataFrame(data6, columns=names6)
df6.head()
a=df5['Mean Mag']
b=df5['Std']
c=df6['Mean Mag']
d=df6['Std']
e=df3['Mean Mag']
f=df3['Std']
ax=plt.scatter(e,f, label=' All sources')
#ay=plt.scatter(c,d, label='3 Sigma from Median Std')
lstsq_coefs = np.polyfit(a, b, deg=2)
lstsq_preds = lstsq_coefs[0]*a**2 + lstsq_coefs[1]*a + lstsq_coefs[2]
plt.plot(a, lstsq_preds, linestyle="dashed", color="red", label="Least squares 2 sigma")
#ay=plt.scatter(c,d, label='3 Sigma from Median Std')
lstsq_coefs1 = np.polyfit(c ,d, deg=2)
lstsq_preds1 = lstsq_coefs1[0]*c**2 + lstsq_coefs1[1]*c + lstsq_coefs1[2]
plt.plot(c, lstsq_preds1, linestyle="dashed", color="black", label="Least squares 3 sigma")
plt.legend(loc='best',fontsize= 16)
plt.gcf().set_size_inches((12,10))
plt.ylim(0,0.1)
plt.show()
I want to count the number of points that lie above each least-squares fit. I have tried some extremely tedious methods which is not feasible in the long run.

You can compare them using numpy module:
import numpy as np
f = np.array(f)
lstsq_preds = np.array(lstsq_preds)
lstsq_preds1 = np.array(lstsq_preds1)
print("Number above least squares #1:", len(f[f > lstsq_preds]))
print("Number above least squares #2:", len(f[f > lstsq_preds]))
Note that, I just transform the arrays to numpy to make sure they are numpy. It might be unnecessary to use these transforming lines since you are dealing with pandas dataframe.

Not able to get any graphs as output

I am using Mac OS Monterey, I am tying to run a python code and get two graphs as output, but not getting any. I have ran the code in both atom and Jupiter notebooks and code is running fine without any errors, but I am still not getting any output.
import glob
from astropy.coordinates import get_icrs_coordinates
from astropy.coordinates import SkyCoord
from astropy.io import fits
import astropy.units as u
import matplotlib as mpl
import numpy as np
import pylab as pl
project_Coords = get_icrs_coordinates('4FGL J1256.9+2736')
ra0=project_Coords.ra.value
dec0=project_Coords.dec.value
def load_event_data():
f = fits.open('project_merged.fits')
ra = f['events'].data.field('ra')
dec = f['events'].data.field('dec')
en = f['events'].data.field('energy')
#ph = f['events'].data.field('pulse_phase')
ph = None
f.close()
return ra,dec,en,ph
def make_counts_map():
ra,dec,en,ph = load_event_data()
mask = en>1000
c = SkyCoord(ra[mask]*u.deg,dec[mask]*u.deg,frame = 'icrs')
c0 = SkyCoord(ra0*u.deg,dec0*u.deg)
gal = c.galactic
gal_b = gal.b.value
gal_l = gal.l.value
b0 = c0.galactic.b.value
l0 = c0.galactic.l.value
l_bins = np.linespace(l0-20,l0+20,41)
b_bins = np.linespace(b0-20,b0+20,41)
#Plot sky map
pl.figure(1); pl.clf()
pl.hist2d(gal_l,gal_b,bins [l_bins,b_bins] , norm=mpl.colors.LogNorm() ,vmin=2e2,vmax=1e4);
pl.xlabel('Galactic Longitude')
pl.ylable('Galactic Latitude')
pl.colorbar()
plt.show()
#plot longitude slice around b= -3
pl.figure(2); pl.clf()
mask = np.abs(gal_b - (-3.15)) < 0.5
pl.gca().set_yselect('log')
l_bins = np.linspace(l0-20,l0+20,101)
pl.hist(gal_l[mask],bins=l_bins)
pl.xlabel('A')
pl.ylable('B')

Fitting a quadratic function in python without numpy polyfit

I am trying to fit a quadratic function to some data, and I'm trying to do this without using numpy's polyfit function.
Mathematically I tried to follow this website https://neutrium.net/mathematics/least-squares-fitting-of-a-polynomial/ but somehow I don't think that I'm doing it right. If anyone could assist me that would be great, or If you could suggest another way to do it that would also be awesome.
What I've tried so far:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
ones = np.ones(3)
A = np.array( ((0,1),(1,1),(2,1)))
xfeature = A.T[0]
squaredfeature = A.T[0] ** 2
b = np.array( (1,2,0), ndmin=2 ).T
b = b.reshape(3)
features = np.concatenate((np.vstack(ones), np.vstack(xfeature), np.vstack(squaredfeature)), axis = 1)
featuresc = features.copy()
print(features)
m_det = np.linalg.det(features)
print(m_det)
determinants = []
for i in range(3):
featuresc.T[i] = b
print(featuresc)
det = np.linalg.det(featuresc)
determinants.append(det)
print(det)
featuresc = features.copy()
determinants = determinants / m_det
print(determinants)
plt.scatter(A.T[0],b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
p2 = np.polyfit(A.T[0],b,2)
plt.plot(u, np.polyval(p2,u), 'b--')
plt.show()
As you can see my curve doesn't compare well to nnumpy's polyfit curve.
Update:
I went through my code and removed all the stupid mistakes and now it works, when I try to fit it over 3 points, but I have no idea how to fit over more than three points.
This is the new code:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
ones = np.ones(3)
A = np.array( ((0,1),(1,1),(2,1)))
xfeature = A.T[0]
squaredfeature = A.T[0] ** 2
b = np.array( (1,2,0), ndmin=2 ).T
b = b.reshape(3)
features = np.concatenate((np.vstack(ones), np.vstack(xfeature), np.vstack(squaredfeature)), axis = 1)
featuresc = features.copy()
print(features)
m_det = np.linalg.det(features)
print(m_det)
determinants = []
for i in range(3):
featuresc.T[i] = b
print(featuresc)
det = np.linalg.det(featuresc)
determinants.append(det)
print(det)
featuresc = features.copy()
determinants = determinants / m_det
print(determinants)
plt.scatter(A.T[0],b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
p2 = np.polyfit(A.T[0],b,2)
plt.plot(u, np.polyval(p2,u), 'r--')
plt.show()

Instead using Cramer's Rule, actually solve the system using least squares. Remember that Cramer's Rule will only work if the total number of points you have equals the desired order of polynomial plus 1.
If you don't have this, then Cramer's Rule will not work as you're trying to find an exact solution to the problem. If you have more points, the method is unsuitable as we will create an overdetermined system of equations.
To adapt this to more points, numpy.linalg.lstsq would be a better fit as it solves the solution to the Ax = b by computing the vector x that minimizes the Euclidean norm using the matrix A. Therefore, remove the y values from the last column of the features matrix and solve for the coefficients and use numpy.linalg.lstsq to solve for the coefficients:
import numpy as np
import matplotlib.pyplot as plt
ones = np.ones(4)
xfeature = np.asarray([0,1,2,3])
squaredfeature = xfeature ** 2
b = np.asarray([1,2,0,3])
features = np.concatenate((np.vstack(ones),np.vstack(xfeature),np.vstack(squaredfeature)), axis = 1) # Change - remove the y values
determinants = np.linalg.lstsq(features, b)[0] # Change - use least squares
plt.scatter(xfeature,b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
plt.show()
I get this plot now, which matches what the dashed curve is in your graph, also matching what numpy.polyfit gives you:

Plotting stick-breaking process in R based on Python code

I'd like to reproduce Python code to R code about Stick-breaking process, which is one of construction schemes for Dirichlet Process. However, the plot I drew within R is quite different in that DP sample distributions are not around the base distribution, H.
The reference Python code is from Austin Rochford's blog.
from matplotlib import pyplot as plt
import numpy as np
import pymc3 as pm
import scipy.stats as ss
import seaborn as sns
from statsmodels.datasets import get_rdataset
from theano import tensor as T
np.random.seed(433)
N=20
K=30
alpha=50
H = ss.norm # base dist
beta = ss.beta.rvs(1,alpha, size=(N,K))
pi = np.empty_like(beta)
pi[:, 0] = beta[:,0]
pi[:, 1:] = beta[:, 1:] * (1-beta[:, :-1]).cumprod(axis=1)
omega = H.rvs(size=(N,K))
x_plot = np.linspace(-3,3,200)
sample_cdfs = (pi[..., np.newaxis]* np.less.outer(omega, x_plot)).sum(axis=1)
fig, ax = plt.subplots(figsize=(8,6))
ax.plot (x_plot, sample_cdfs[0],c="gray", alpha=0.75, label = "DP sample CDFs")
ax.plot(x_plot, sample_cdfs[1:].T, c="gray", alpha=0.75)
ax.plot(x_plot, H.cdf(x_plot), c= "k", label = "Base CDF")
ax.set_title(r'$\alpha = {}$'.format(alpha))
ax.legend(loc=2)
The figure on the right side is the result in Python code.
And I tried to convert it to R code:
library(yarrr)
N=20;K=30;ngrid=200;alpha=50
xgrid = seq(-3,3,length.out=ngrid)
betas = matrix(rbeta(N*K, 1, alpha),nr=N, nc=K)
stick.to.right = c(1, cumprod(1 - betas))[1:K]
pis.temp = stick.to.right * betas
omega = matrix(rnorm(N*K),nr=N,nc=K)
dirac = array(numeric(N*K*ngrid),dim=c(N,K,ngrid))
for(i in 1:N){
for(j in 1:K){
for(k in 1:ngrid){
dirac[i,j,k]=ifelse(omega[i,j]<xgrid[k],TRUE,FALSE)
}
}
}
pis = array(pis.temp,dim=c(N,K,200))
sample_cdfs = apply(pis* dirac,c(1,3),sum)
plot(xgrid,sample_cdfs[1,],col=piratepal("pony"),type="l",lwd=1,ylim=c(0,1))
for(i in 2:N) lines(xgrid,sample_cdfs[i,],col=piratepal("pony")[i])
lines(xgrid,pnorm(xgrid),lwd=2)
The plot I drew is DP with alpha=50:
How can I modify R code to give a similar result as Python code?

Solving nonlinear differential first order equations using Python

I would like to solve a nonlinear first order differential equation using Python.
For instance,
df/dt = f**4
I wrote the following program, but I have an issue with matplotlib, so I don't know if the method I used with scipy is correct.
from scipy.integrate import odeint
import numpy as np
import matplotlib.pyplot as plt
derivate=lambda f,t: f**4
f0=10
t=np.linspace(0,2,100)
f_numeric=scipy.integrate.odeint(derivate,f0,t)
print(f_numeric)
plt.plot(t,f_numeric)
plt.show()
Which results in the following error:
AttributeError: 'float' object has no attribute 'rint'

In this case, you might be better of using Sympy, which allows you to obtain the closed form solutions:
from IPython.display import display
import sympy as sy
from sympy.solvers.ode import dsolve
import matplotlib.pyplot as plt
import numpy as np
sy.init_printing() # LaTeX like pretty printing for IPython
t = sy.symbols("t", real=True)
f = sy.symbols("f", function=True)
eq1 = sy.Eq(f(t).diff(t), f(t)**4) # the equation
sls = dsolve(eq1) # solvde ODE
# print solutions:
print("For ode")
display(eq1)
print("the solutions are:")
for s in sls:
display(s)
# plot solutions:
x = np.linspace(0, 2, 100)
fg, axx = plt.subplots(2, 1)
axx[0].set_title("Real part of solution of $\\frac{d}{dt}f(t)= (f(t))^4$")
axx[1].set_title("Imag. part of solution of $\\frac{d}{dt}f(t)= (f(t))^4$")
fg.suptitle("$C_1=0.1$")
for i, s in enumerate(sls, start=1):
fn1 = s.rhs.subs("C1", .1) # C_1 -> 1
fn2 = sy.lambdify(t, fn1, modules="numpy") # make numpy function
y = fn2(x+0j) # needs to be called with complex number
axx[0].plot(x, np.real(y), label="Sol. %d" % i)
axx[1].plot(x, np.imag(y), label="Sol. %d" % i)
for ax in axx:
ax.legend(loc="best")
ax.grid(True)
axx[0].set_ylabel("Re$\\{f(t)\\}$")
axx[1].set_ylabel("Im$\\{f(t)\\}$")
axx[-1].set_xlabel("$t$")
fg.canvas.draw()
plt.show()
In an IPython shell, you should see the following: