Develop Reference

Curve_Fit not accurate

i tried to fit very fluctual data over time as good as possible. So first i smoothed the data which is working fine. The smoothed data I get from this should further be represented from a fit to get out more of the peaks. As you see in the code I want to use an log-tanh function to fit the data. I am well aware that this problem accured in some of the threads already, but I tried them already and the data is also not very small or very big which i know can also cause problems.
The polynomial fit i tried works also pretty good as you see, but it does not eliminate all the wavy values. They cause problems for the following derivative which is very bad.
import tkinter as tk
from tkinter import filedialog
import numpy as np
import scipy.signal
from scipy.optimize import curve_fit
from numpy import diff
import matplotlib.pyplot as plt
from lmfit.models import StepModel, LinearModel
def loghypfunc(x, A, B, C, D, E):
return A*np.log(1+x)+B*np.tanh(C*x)+D*x+E
def expfunc(t, c0, c1, c2, c3):
return c0+c1*t-c2*np.exp(-c3*t)
def expdecay(x, a, b, c):
return a * np.exp(-b * x) + c
path="C:/Users/Sammy/Documents/Masterarbeit WT/CSM und Kriechdaten/Kriechen/Creep_10mN_00008_LC_20210406_2121_DYN.txt"
dataFile = np.loadtxt(path, delimiter='\t', skiprows=2, usecols=(0, 1, 2, 3, 29, 30), dtype=float)
num_rows, num_cols = dataFile.shape
# time column
time = dataFile[:, [0]].transpose()
time = time.flatten()
refTime = time[0] # get first time in column (reference)
# genullte Testzeit
timeNull = time - refTime
print("time", time)
flatTimeNull = timeNull.flatten() # jetzt ein 1D array (one row)
##################################################################################
# indent displacement column
indentDis = dataFile[:, [4]].transpose()
indentDis = indentDis.flatten()
indentDis = indentDis - indentDis[0]
# the indendt data has to be smoothed so there is not such a big fluctuation
indentSmooth = scipy.signal.savgol_filter(indentDis, 2001, 3)
# null the indent Smooth data
indentSmooth_Null = indentSmooth - indentSmooth[0]
hind_Smooth_flat = indentSmooth_Null.flatten() # jetzt ein 1D array
print('indent smooth', indentSmooth)
######################################################################
p0 = [100, 0.1, 100, 0.1]
c, cov = curve_fit(expfunc, time, indentSmooth, p0)
y_indent = expfunc(indentSmooth, *c)
p0 = [70, 0.5, 50, 0.1, 100]
popt, pcov = curve_fit(loghypfunc, time, indentSmooth, p0, maxfev = 10000)
y_indentTan = loghypfunc(indentSmooth, *popt)
modelh_t = np.poly1d(np.polyfit(time, indentSmooth, 8))
plt.plot(time, indentSmooth, 'r', label="Data smoothed")
plt.scatter(time, modelh_t(time), s=0.1, label="Polyfit")
plt.plot(time, y_indentTan, label="Curve fit Tangens function")
plt.plot(time, y_indent, label="Curve fit exp function")
plt.legend(loc="lower right")
plt.xlabel("time")
plt.ylabel("indent")
plt.show()
These are the two arrays i get the data from
time [ 6.299596 6.349592 6.399589 ... 608.0109 608.060897 608.110894]
indent smooth [120.81411822 121.07093706 121.32748184 ... 476.78825661 476.89357473 476.99915287]
Here the plots
Plots
The question for me now is how to fix it. Is it because of the false optimizied parameters to fit? But python should do that automatic sufficiently good i guess?
My second guess was that the data is timed to compact along this axes, as the array is about 12000 values long. Could this be a reason?
I would be very grateful for any kind of advices regarding the fits.
Regards
Hndrx

Fitting a quadratic function in python without numpy polyfit

I am trying to fit a quadratic function to some data, and I'm trying to do this without using numpy's polyfit function.
Mathematically I tried to follow this website https://neutrium.net/mathematics/least-squares-fitting-of-a-polynomial/ but somehow I don't think that I'm doing it right. If anyone could assist me that would be great, or If you could suggest another way to do it that would also be awesome.
What I've tried so far:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
ones = np.ones(3)
A = np.array( ((0,1),(1,1),(2,1)))
xfeature = A.T[0]
squaredfeature = A.T[0] ** 2
b = np.array( (1,2,0), ndmin=2 ).T
b = b.reshape(3)
features = np.concatenate((np.vstack(ones), np.vstack(xfeature), np.vstack(squaredfeature)), axis = 1)
featuresc = features.copy()
print(features)
m_det = np.linalg.det(features)
print(m_det)
determinants = []
for i in range(3):
featuresc.T[i] = b
print(featuresc)
det = np.linalg.det(featuresc)
determinants.append(det)
print(det)
featuresc = features.copy()
determinants = determinants / m_det
print(determinants)
plt.scatter(A.T[0],b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
p2 = np.polyfit(A.T[0],b,2)
plt.plot(u, np.polyval(p2,u), 'b--')
plt.show()
As you can see my curve doesn't compare well to nnumpy's polyfit curve.
Update:
I went through my code and removed all the stupid mistakes and now it works, when I try to fit it over 3 points, but I have no idea how to fit over more than three points.
This is the new code:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
ones = np.ones(3)
A = np.array( ((0,1),(1,1),(2,1)))
xfeature = A.T[0]
squaredfeature = A.T[0] ** 2
b = np.array( (1,2,0), ndmin=2 ).T
b = b.reshape(3)
features = np.concatenate((np.vstack(ones), np.vstack(xfeature), np.vstack(squaredfeature)), axis = 1)
featuresc = features.copy()
print(features)
m_det = np.linalg.det(features)
print(m_det)
determinants = []
for i in range(3):
featuresc.T[i] = b
print(featuresc)
det = np.linalg.det(featuresc)
determinants.append(det)
print(det)
featuresc = features.copy()
determinants = determinants / m_det
print(determinants)
plt.scatter(A.T[0],b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
p2 = np.polyfit(A.T[0],b,2)
plt.plot(u, np.polyval(p2,u), 'r--')
plt.show()

Instead using Cramer's Rule, actually solve the system using least squares. Remember that Cramer's Rule will only work if the total number of points you have equals the desired order of polynomial plus 1.
If you don't have this, then Cramer's Rule will not work as you're trying to find an exact solution to the problem. If you have more points, the method is unsuitable as we will create an overdetermined system of equations.
To adapt this to more points, numpy.linalg.lstsq would be a better fit as it solves the solution to the Ax = b by computing the vector x that minimizes the Euclidean norm using the matrix A. Therefore, remove the y values from the last column of the features matrix and solve for the coefficients and use numpy.linalg.lstsq to solve for the coefficients:
import numpy as np
import matplotlib.pyplot as plt
ones = np.ones(4)
xfeature = np.asarray([0,1,2,3])
squaredfeature = xfeature ** 2
b = np.asarray([1,2,0,3])
features = np.concatenate((np.vstack(ones),np.vstack(xfeature),np.vstack(squaredfeature)), axis = 1) # Change - remove the y values
determinants = np.linalg.lstsq(features, b)[0] # Change - use least squares
plt.scatter(xfeature,b)
u = np.linspace(0,3,100)
plt.plot(u, u**2*determinants[2] + u*determinants[1] + determinants[0] )
plt.show()
I get this plot now, which matches what the dashed curve is in your graph, also matching what numpy.polyfit gives you:

Numpy: Reshaped arrays behaving strangely

I'm trying to reshape a numpy array [link] then reshape that array again, but am not able to achieve my desired result. My data starts in shape (n_vertices, n_time, n_dimensions). I then transform it into shape (n_time, n_vertices * n_dimensions):
import numpy as np
X = np.load('dance.npy')
n_vertices, n_time, n_dims = X.shape
X = X.reshape(n_time, n_vertices * n_dims)
By visualizing the data, I can see that the transformation above does not distort the internal values:
import mpl_toolkits.mplot3d.axes3d as p3
from mpl_toolkits.mplot3d.art3d import juggle_axes
import matplotlib.pyplot as plt
from IPython.display import HTML
from matplotlib import animation
import matplotlib
matplotlib.rcParams['animation.embed_limit'] = 2**128
def update_points(time, points, df):
points._offsets3d = juggle_axes(df[:,time,0], df[:,time,1], df[:,time,2], 'z')
def get_plot(df, lim=1, frames=200, duration=45, time_axis=1, reshape=False):
if reshape: df = df.reshape(n_vertices, df.shape[time_axis], n_dims)
fig = plt.figure()
ax = p3.Axes3D(fig)
ax.set_xlim(-lim, lim)
ax.set_ylim(-lim, lim)
ax.set_zlim(-lim, lim)
points = ax.scatter(df[:,0,0], df[:,0,1], df[:,0,2], depthshade=False) # x,y,z vals
return animation.FuncAnimation(fig, update_points, frames, interval=duration, fargs=(points, df), blit=False ).to_jshtml()
HTML(get_plot(X, frames=200, time_axis=0, reshape=True))
This shows the data in motion (the vertices are body parts of a dancer, and the visualization looks like a human body). This is all good. However, when I try to visualize just the first 10 time slices of the data, the resulting plot does not show the first few frames of the visualization above -- the form is in fact not human shaped:
HTML(get_plot(X[:20], frames=10, time_axis=0, reshape=True))
Can anyone help me understand why this slicing operation does not match the first few time frames of X? Any suggestions or observations would be very helpful.

It turns out that my reshaping operations weren't manipulating my arrays as I thought they were. The following functions reshape my original array X into the flattened form (with two axes) then back to the unflattened form (with three axes) properly. I added comments and tests to make sure all was as it is expected to be:
from math import floor
def flatten(df, run_tests=True):
'''
df is a numpy array with the following three axes:
df.shape[0] = the index of a vertex
df.shape[1] = the index of a time stamp
df.shape[2] = the index of a dimension (x, y, z)
So df[1][0][2] is the value for the 1st vertex (0-based) at time 0 in dimension 2 (z).
To flatten this dataframe will mean to push the data into shape:
flattened.shape[0] = time index
flattened.shape[1] = [vertex_index*3] + dimension_vertex
So flattened[1][3] will be the 3rd dimension of the 1st index (0-based) at time 1.
'''
if run_tests:
assert df.shape == X.shape and np.all(df == X)
# reshape X such that flattened.shape = time, [x0, y0, z0, x1, y1, z1, ... xn-1, yn-1, zn-1]
flattened = X.swapaxes(0, 1).reshape( (df.shape[1], df.shape[0] * df.shape[2]), order='C' )
if run_tests: # switch to false to skip tests
for idx, i in enumerate(df):
for jdx, j in enumerate(df[idx]):
for kdx, k in enumerate(df[idx][jdx]):
assert flattened[jdx][ (idx*df.shape[2]) + kdx ] == df[idx][jdx][kdx]
return flattened
And to unflatten the flattened data:
def unflatten(df, run_tests=True):
'''
df is a numpy array with the following two axes:
df.shape[0] = time index
df.shape[1] = [vertex_index*3] + dimension_vertex
To unflatten this dataframe will mean to push the data into shape:
unflattened.shape[0] = the index of a vertex
unflattened.shape[1] = the index of a time stamp
unflattened.shape[2] = the index of a dimension (x, y, z)
So df[2][4] == unflattened[1][2][0]
'''
if run_tests:
assert (len(df.shape) == 2) and (df.shape[1] == X.shape[0] * X.shape[2])
unflattened = np.zeros(( X.shape[0], df.shape[0], X.shape[2] ))
for idx, i in enumerate(df):
for jdx, j in enumerate(df[idx]):
kdx = floor(jdx / 3)
ldx = jdx % 3
unflattened[kdx][idx][ldx] = df[idx][jdx]
if run_tests: # set to false to skip tests
for idx, i in enumerate(unflattened):
for jdx, j in enumerate(unflattened[idx]):
for kdx, k in enumerate(unflattened[idx][jdx]):
assert( unflattened[idx][jdx][kdx] == X[idx][jdx][kdx] )
return unflattened
Then to visualize:
import mpl_toolkits.mplot3d.axes3d as p3
from mpl_toolkits.mplot3d.art3d import juggle_axes
import matplotlib.pyplot as plt
from IPython.display import HTML
from matplotlib import animation
import matplotlib
# ask matplotlib to plot up to 2^128 frames in animations
matplotlib.rcParams['animation.embed_limit'] = 2**128
def update_points(time, points, df):
points._offsets3d = juggle_axes(df[:,time,0], df[:,time,1], df[:,time,2], 'z')
def get_plot(df, lim=1, frames=200, duration=45):
if len(df.shape) == 2: df = unflatten(df)
fig = plt.figure()
ax = p3.Axes3D(fig)
ax.set_xlim(-lim, lim)
ax.set_ylim(-lim, lim)
ax.set_zlim(-lim, lim)
points = ax.scatter(df[:,0,0], df[:,0,1], df[:,0,2], depthshade=False) # x,y,z vals
return animation.FuncAnimation(fig,
update_points,
frames,
interval=duration,
fargs=(points, df),
blit=False
).to_jshtml()
HTML(get_plot(unflat, frames=200))
This allows me to slice my time axis without problem:
flat = flatten(X)
unflat = unflatten(flat)
HTML(get_plot(unflat, frames=200))
HTML(get_plot(flat[:20], frames=20))
HTML(get_plot(unflat[:,:20,:], frames=20))

Connect all 2D Points from NumPy 2D Arrays as a triangular meshgrid

I am pretty new to Python and I am trying to plot a triangular grid like this:
import matplotlib.pyplot as plt
import numpy as np
r = 0.25
d = 2*r
s = 0
l1 = np.array([[s,0], [s+d,0], [s+2*d,0], [s+3*d,0]])
l2 = np.array([[s-r,d], [s+r,d], [s+r+d,d], [s+r+2*d,d]])
l3 = np.array([[s,2*d], [s+d,2*d], [s+2*d,2*d], [s+3*d,2*d]])
l4 = np.array([[s-r,3*d], [s+r,3*d], [s+r+d,3*d], [s+r+2*d,3*d]])
l5 = np.array([[s,4*d], [s+d,4*d], [s+2*d,4*d], [s+3*d,4*d]])
plt.scatter(*zip(*l1))
plt.scatter(*zip(*l2))
plt.scatter(*zip(*l3))
plt.scatter(*zip(*l4))
plt.scatter(*zip(*l5))
plt.show
My problem is, that I have no real clue how to connect all points. I have added horizontal lines with plt.plot(*zip(*l1)) for all l but I have no idea how to draw the 'vertical' zigzag lines... Has anybody a 'simple' solution?
Many thanks in advance!

triplot is made for that purpose: plotting triangles.
You can either pass only x and y coordinates (in this case a Delaunay triangulation will be computed), or a full Triangulation object to which you can specify your own triangles.
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.tri as mtri
r = 0.25
d = 2*r
s = 0
def meshgrid_triangles(n, m):
""" Returns triangles to mesh a np.meshgrid of n x m points """
tri = []
for i in range(n-1):
for j in range(m-1):
a = i + j*(n)
b = (i+1) + j*n
d = i + (j+1)*n
c = (i+1) + (j+1)*n
if j%2 == 1:
tri += [[a, b, d], [b, c, d]]
else:
tri += [[a, b, c], [a, c, d]]
return np.array(tri, dtype=np.int32)
x0 = np.arange(4) * d
y0 = np.arange(5) * d
x, y = np.meshgrid(x0, y0)
x[1::2] -= r
triangles = meshgrid_triangles(4, 5)
triangulation = mtri.Triangulation(x.ravel(), y.ravel(), triangles)
plt.scatter(x, y, color='red')
plt.triplot(triangulation, 'g-h')
plt.show()

Using the code the way you have (otherwise look at triplot_demo depending on what you want, as mentioned by #GBy), you can extract or rotate each array so that you just plot the line downwards:
import matplotlib.pyplot as plt
import numpy as np
r = 0.25
d = 2*r
s = 0
l1 = np.array([[s,0], [s+d,0], [s+2*d,0], [s+3*d,0]])
l2 = np.array([[s-r,d], [s+r,d], [s+r+d,d], [s+r+2*d,d]])
l3 = np.array([[s,2*d], [s+d,2*d], [s+2*d,2*d], [s+3*d,2*d]])
l4 = np.array([[s-r,3*d], [s+r,3*d], [s+r+d,3*d], [s+r+2*d,3*d]])
l5 = np.array([[s,4*d], [s+d,4*d], [s+2*d,4*d], [s+3*d,4*d]])
fig = plt.figure(0)
ax = fig.add_subplot(111)
larr = [l1,l2,l3,l4,l5]
# Plot horizontally
for l in larr:
# same as your *zip(*l1), but you can select on a column-wise basis
ax.errorbar(l[:,0], l[:,1], fmt="o", ls="-", color="black")
# Plot zig-zag-horizontally
for i in range(len(larr[0])):
lxtmp = np.array([x[:,0][i] for x in larr])
lytmp = np.array([x[:,1][i] for x in larr])
ax.errorbar(lxtmp, lytmp, fmt="o", ls="-", color="black")
ax.set_ylim([-0.1,2.1])
ax.set_xlim([-0.6,1.6])
plt.show()
EDIT:
lxtmp = np.array([x[:,0][i] for x in larr])
So, x[:,0] means take all of the rows ":" but only the first column "0". For l1 it will return:
l1[:,0]
array([ 0. , 0.5, 1. , 1.5])
which are the x-values for l1. Doing l1[:,1] will return all of the rows from column "1", the y-values. To draw the vertical lines, you want to take all the x and y values from each i-th array, and hence you loop over all the arrays, taking out the i-th element. For example, the 3rd vertical zig-zag line, would be:
lxtmp = [l1[:,0][2], l2[:,0][2], l3[:,0][2], l4[:,0][2], l5[:,0][2]]
lytmp = [l1[:,1][2], l2[:,1][2], l3[:,1][2], l4[:,1][2], l5[:,1][2]]
To simplify and run over each element, I created 'larr' to loop over and 'build' then in a normal python fashion, e.g.,
[i for i in range(1,10)]
[1, 2, 3, 4, 5, 6, 7, 8, 9]

Plot periodic trajectories

I have some data of a particle moving in a corridor with closed boundary conditions.
Plotting the trajectory leads to a zig-zag trajectory.
I would like to know how to prevent plot() from connecting the points where the particle comes back to the start. Some thing like in the upper part of the pic, but without "."
The first idea I had was to find the index where the numpy array a[:-1]-a[1:] becomes positive and then plot from 0 to that index. But how would I get the index of the first occurrence of a positive element of a[:-1]-a[1:]?
Maybe there are some other ideas.

I'd go a different approach. First, I'd determine the jump points not by looking at the sign of the derivative, as probably the movement might go up or down, or even have some periodicity in it. I'd look at those points with the biggest derivative.
Second, an elegant approach to have breaks in a plot line is to mask one value on each jump. Then matplotlib will make segments automatically. My code is:
import pylab as plt
import numpy as np
xs = np.linspace(0., 100., 1000.)
data = (xs*0.03 + np.sin(xs) * 0.1) % 1
plt.subplot(2,1,1)
plt.plot(xs, data, "r-")
#Make a masked array with jump points masked
abs_d_data = np.abs(np.diff(data))
mask = np.hstack([ abs_d_data > abs_d_data.mean()+3*abs_d_data.std(), [False]])
masked_data = np.ma.MaskedArray(data, mask)
plt.subplot(2,1,2)
plt.plot(xs, masked_data, "b-")
plt.show()
And gives us as result:
The disadvantage of course is that you lose one point at each break - but with the sampling rate you seem to have I guess you can trade this in for simpler code.

To find where the particle has crossed the upper boundary, you can do something like this:
>>> import numpy as np
>>> a = np.linspace(0, 10, 50) % 5
>>> a = np.linspace(0, 10, 50) % 5 # some sample data
>>> np.nonzero(np.diff(a) < 0)[0] + 1
array([25, 49])
>>> a[24:27]
array([ 4.89795918, 0.10204082, 0.30612245])
>>> a[48:]
array([ 4.79591837, 0. ])
>>>
np.diff(a) calculates the discrete difference of a, while np.nonzero finds where the condition np.diff(a) < 0 is negative, i.e., the particle has moved downward.

To avoid the connecting line you will have to plot by segments.
Here's a quick way to plot by segments when the derivative of a changes sign:
import numpy as np
a = np.linspace(0, 20, 50) % 5 # similar to Micheal's sample data
x = np.arange(50) # x scale
indices = np.where(np.diff(a) < 0)[0] + 1 # the same as Micheal's np.nonzero
for n, i in enumerate(indices):
if n == 0:
plot(x[:i], a[:i], 'b-')
else:
plot(x[indices[n - 1]:i], a[indices[n - 1]:i], 'b-')

Based on Thorsten Kranz answer a version which adds points to the original data when the 'y' crosses the period. This is important if the density of data-points isn't very high, e.g. np.linspace(0., 100., 100) vs. the original np.linspace(0., 100., 1000). The x position of the curve transitions are linear interpolated. Wrapped up in a function its:
import numpy as np
def periodic2plot(x, y, period=np.pi*2.):
indexes = np.argwhere(np.abs(np.diff(y))>.5*period).flatten()
index_shift = 0
for i in indexes:
i += index_shift
index_shift += 3 # in every loop it adds 3 elements
if y[i] > .5*period:
x_transit = np.interp(period, np.unwrap(y[i:i+2], period=period), x[i:i+2])
add = np.ma.array([ period, 0., 0.], mask=[0,1,0])
else:
# interpolate needs sorted xp = np.unwrap(y[i:i+2], period=period)
x_transit = np.interp(0, np.unwrap(y[i:i+2], period=period)[::-1], x[i:i+2][::-1])
add = np.ma.array([ 0., 0., period], mask=[0,1,0])
x_add = np.ma.array([x_transit]*3, mask=[0,1,0])
x = np.ma.hstack((x[:i+1], x_add, x[i+1:]))
y = np.ma.hstack((y[:i+1], add, y[i+1:]))
return x, y
The code for comparison to the original answer of Thorsten Kranz with lower data-points density.
import matplotlib.pyplot as plt
x = np.linspace(0., 100., 100)
y = (x*0.03 + np.sin(x) * 0.1) % 1
#Thorsten Kranz: Make a masked array with jump points masked
abs_d_data = np.abs(np.diff(y))
mask = np.hstack([np.abs(np.diff(y))>.5, [False]])
masked_y = np.ma.MaskedArray(y, mask)
# Plot
plt.figure()
plt.plot(*periodic2plot(x, y, period=1), label='This answer')
plt.plot(x, masked_y, label='Thorsten Kranz')
plt.autoscale(enable=True, axis='both', tight=True)
plt.legend(loc=1)
plt.tight_layout()