Related
step = [0.1,0.2,0.3,0.4,0.5]
static = []
for x in step:
range = np.arrange(5,10 + x, x)
static.append(range)
# this return a list that looks something like this [[5.,5.1,5.2,...],[5.,5.2,5.4,...],[5.,5.3,5.6,...],...]
Im trying to create standard and dynamic stop/step ranges from 5.0-10. For the standard ranges I used a list with the steps and then looped it to get the different interval lists.
What I want now is to get varying step sizes within the 5.0-10.0 interval. So for example from 5.0-7.3, the step size is 0.2, from 7.3-8.3, the range is 0.5 and then from 8.3-10.0 the lets say the step is 0.8. What I don't understand how to do is to make the dynamic run through and get all the possible combinations.
Using a list of steps and a list of "milestones" that we are going to use to determine the start and end points of each np.arange, we can do this:
import numpy as np
def dynamic_range(milestones, steps) -> list:
start = milestones[0]
dynamic_range = []
for end, step in zip(milestones[1:], steps):
dynamic_range += np.arange(start, end, step).tolist()
start = end
return dynamic_range
print(dynamic_range(milestones=(5.0, 7.3, 8.3, 10.0), steps=(0.2, 0.5, 0.8)))
# [5.0, 5.2, 5.4, 5.6, 5.8, 6.0, 6.2, 6.4, 6.6, 6.8, 7.0,
# 7.2, 7.3, 7.8, 8.3, 8.3, 9.1, 9.9]
Note on performance: this answer assumes that you are going to use a few hundred points in your dynamic range. If you want millions of points, we should try another approach with pure numpy and no list concatenation.
if you want to be it within <5,10> interval then dont add x to 10:
import numpy as np
step = [0.1, 0.2, 0.3, 0.4, 0.5]
static = []
for x in step:
range = np.arange(5, 10, x)
static.append(range)
print(static)
Dinamic:
import numpy as np
step = [0.1, 0.2, 0.3, 0.4, 0.5]
breakingpoints=[6,7,8,9,10]
dinamic = []
i=0
startingPoint=5
for x in step:
#print(breakingpoints[i])
range = np.arange(startingPoint, breakingpoints[i], x)
dinamic.append(range)
i+=1
#print(range[-1])
startingPoint=range[-1]
print(dinamic)
I have the following code, in which DGauss is a function that generates the expected values. The two arrays, on the other hand, allow me to generate a distribution, that I take as observed values.
The code, based on the observed values, extracts a polynomial (for the moment of the seventh degree) that describes its trend.
import matplotlib.pyplot as plt
import numpy as np
from scipy.optimize import curve_fit
def DGauss(x,I1,I2,sigma1,sigma2):
return I1*np.exp(-x*x/(2*sigma1*sigma1)) + I2*np.exp(-x*x/(2*sigma2*sigma2))
Pos = np.array([3.28, 3.13, 3.08, 3.03, 2.98, 2.93, 2.88, 2.83, 2.78, 2.73, 2.68,
2.63, 2.58, 2.53, 2.48, 2.43, 2.38, 2.33, 2.28, 2.23, 2.18, 2.13,
2.08, 2.03, 1.98, 1.93, 1.88, 1.83, 1.78, 1.73, 1.68, 1.63, 1.58,
1.53, 1.48, 1.43, 1.38, 1.33, 1.28, 1.23, 1.18, 1.13, 1.08, 1.03,
0.98, 0.93, 0.88, 0.83, 0.78, 0.73, 0.68, 0.63, 0.58, 0.53, 0.48,
0.43, 0.38, 0.33, 0.28, 0.23, 0.18, 0.13, 0.08, 0.03])
Val = np.array([0.00986279, 0.01529543, 0.0242624 , 0.0287456 , 0.03238484,
0.03285927, 0.03945234, 0.04615091, 0.05701618, 0.0637672 ,
0.07194268, 0.07763934, 0.08565687, 0.09615262, 0.1043281 ,
0.11350606, 0.1199406 , 0.1260062 , 0.14093328, 0.15079665,
0.16651464, 0.18065023, 0.1938894 , 0.2047541 , 0.21794024,
0.22806706, 0.23793043, 0.25164404, 0.2635118 , 0.28075974,
0.29568682, 0.30871501, 0.3311846 , 0.34648062, 0.36984661,
0.38540666, 0.40618835, 0.4283945 , 0.45002014, 0.48303911,
0.50746062, 0.53167057, 0.5548792 , 0.57835128, 0.60256181,
0.62566436, 0.65704847, 0.68289386, 0.71332794, 0.73258027,
0.769608 , 0.78769989, 0.81407275, 0.83358852, 0.85210239,
0.87109068, 0.89456217, 0.91618782, 0.93760247, 0.95680234,
0.96919757, 0.9783219 , 0.98486193, 0.9931429 ])
f = np.linspace(-9,9,2*len(Pos))
plt.errorbar(Pos, Val, xerr=0.02, yerr=2.7e-3, fmt='o')
popt, pcov = curve_fit(DGauss, Pos, Val)
plt.plot(xfull, DGauss(f, *popt), '--', label='Double Gauss')
x = Pos
y = Val
#z, w = np.polyfit(x, y, 7, full=False, cov=True)
p = np.poly1d(z)
u = np.array(p)
xp = np.linspace(1, 6, 100)
_ = plt.plot(xp, p(xp), '-', color='darkviolet')
x = symbols('x')
list = u[::-1]
poly = sum(S("{:7.3f}".format(v))*x**i for i, v in enumerate(list))
eq_latex = sympy.printing.latex(poly)
print(eq_latex)
#LOOP SUGGESTED BY #Fourier
dof = [1,2,3,4,5,6,7,8,9,10]
for i in dof:
z = np.polyfit(x, y, i, full=False, cov=True)
chi = np.sum((np.polyval(z, x) - y) ** 2)
chinorm = chi/i
plt.plot(chinorm)
What I would like to do now is to make a fit by varying the order of the polynomial to figure out which is the minimum order I need to have a good fit and not exceed the number of free parameters. In particular, I would like to make this fit with different orders and plot the chi-squared, which must be normalized with respect to the number of degrees of freedom.
Could someone help me kindly?
Thanks!
Based on the posted code this should work for your purpose:
chiSquares = []
dofs = 10
for i in np.arange(1,dofs+1):
z = np.polyfit(x, y, i, full=False, cov=False)
chi = np.sum((np.polyval(z, x) - y) ** 2) / np.std(y) #ideally you should divide this using an error for Val array
chinorm = chi/i
chiSquares.append(chinorm)
plt.plot(np.arange(1,dofs+1),chiSquares)
If not evident from the plot, you can further use the F-test to check how much dof is really needed:
n = len(y)
for d, (rss1,rss2) in enumerate(zip(chiSquares,chiSquares[1:])):
p1 = d + 1
p2 = d + 2
F = (rss1-rss2/(p2-p1)) / (rss2/(n-p2))
p = 1.0 - scipy.stats.f.cdf(F,p1,p2)
print 'F-stats: {:.3f}, p-value: {:.5f}'.format(F,p)
Hi i have an array of float [time,position] coordinates in a sparse format, eg
times = [0.1, 0.1, 1.5, 1.9, 1.9, 1.9]
posit = [2.1, 3.5, 0.4, 1.3, 2.7, 3.5]
and an array of velocities, eg
vel = [0.5,0.7,1.0]
I have to multiply each positions at the i-th time with the i-th element of vel.
In numpy is quite simple with a for:
import numpy
times = numpy.array([0.1, 0.1, 1.5, 1.9, 1.9, 1.9])
posit = numpy.array([2.1, 3.5, 0.4, 1.3, 2.7, 3.5])
vel = numpy.array([0.5,0.7,1.0])
uniqueTimes = numpy.unique(times, return_index=True)
uniqueIndices = uniqueTimes[1]
uniqueTimes = uniqueTimes[0]
numIndices = numpy.size(uniqueTimes)-1
iterator = numpy.arange(numIndices)+1
for i in iterator:
posit[uniqueIndices[i-1]:uniqueIndices[i]] = posit[uniqueIndices[i-1]:uniqueIndices[i]]*vel[i-1]
In tensorflow i can gather every information i need with
import tensorflow as tf
times = tf.constant([0.1, 0.1, 1.5, 1.9, 1.9, 1.9])
posit = tf.constant([2.1, 3.5, 0.4, 1.3, 2.7, 3.5])
vel = tf.constant([0.5,0.7,1.0])
uniqueTimes, uniqueIndices, counts = tf.unique_with_counts(times)
uniqueIndices = tf.cumsum(tf.pad(tf.unique_with_counts(uniqueIndices)[2],[[1,0]]))[:-1]
but i can't figure how to do the product. With int elements i could use sparse to dense tensors and use tf.matmul, but with float i can't.
Moreover, looping is difficult, since map_fn and while_loop require same size of each 'row', but i have different number of positions at each times. For the same reason i can't work separately each time and update the final positions tensor with tf.concat. Any help? Maybe with scatter_update or Variable assignment?
Following answer from vijai m, i have differences up to 1.5% between numpy and tensorflow code. You can check it using these data
times [0.1, 0.1, 0.2, 0.2]
posit [58.98962402, 58.9921875, 60.00390625, 60.00878906]
vel [0.99705114,0.99974157]
They return
np: [ 58.81567188 58.8182278 60.00390625 60.00878906]
tf: [ 58.81567001 58.81822586 59.98839951 59.9932785 ]
differences: [ 1.86388465e-06 1.93737304e-06 1.55067444e-02 1.55105566e-02]
Your numpy code doesn't work. I hope this is what you are looking for:
uniqueTimes, uniqueIndices, counts = tf.unique_with_counts(times)
out = tf.gather_nd(vel,uniqueIndices[:,None])*posit
I am having an error like this as shown below
Error
Traceback (most recent call last):
File "/Users/tej/Desktop/Allfiles/TSAlgorithm/Rots/slidingwindow.py", line 41, in <module>
segments = segment.slidingwindowsegment(data, fit.regression, fit.sumsquared_error, max_error)
File "/Users/tej/Desktop/Allfiles/TSAlgorithm/Rots/segment.py", line 29, in slidingwindowsegment
result_segment = create_segment(sequence,(seq_range[0],seq_range[1]))
File "/Users/tej/Desktop/Allfiles/TSAlgorithm/Rots/fit.py", line 15, in regression
p, error = leastsquareslinefit(sequence,seq_range)
File "/Users/tej/Desktop/Allfiles/TSAlgorithm/Rots/wrappers.py", line 10, in leastsquareslinefit
y = array(sequence[seq_range[0]:seq_range[1]+1])
TypeError: slice indices must be integers or None or have an __index__ method
Code
from numpy import arange, array, ones
from numpy.linalg import lstsq
def leastsquareslinefit(sequence,seq_range):
"""Return the parameters and error for a least squares line fit of one segment of a sequence"""
x = arange(seq_range[0],seq_range[1]+1)
y = array(sequence[seq_range[0]:seq_range[1]+1])
A = ones((len(x),2),float)
A[:,0] = x
(p,residuals,rank,s) = lstsq(A,y)
try:
error = residuals[0]
except IndexError:
error = 0.0
return (p,error)
Error line is
y = array(sequence[seq_range[0]:seq_range[1]+1])
sequence = [2.07879, 1.10188, 0.568894, 0.394302, 0.794052, 0.428718, 0.47567, 0.452418, 0.426734, 0.768852, 0.870319, 1.30586, 0.380166, 0.569896, 0.756903, 0.586858, 0.641127, 0.692155, 1.03697, 1.03722, 1.56939, 0.281659, 0.312457, 1.33759, 1.70201, 1.35018, 1.87431, 1.65084, 1.1157, 0.562875, 0.544611, 0.556123, 0.973077, 0.535477, 0.644177, 0.784439, 0.668507, 0.58048, 0.761036, 1.27403, 0.502488, 0.383824, 0.635096, 0.553752, 0.620955, 0.727158, 0.579525, 0.912165]
seq_range = (0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0, 10.5, 11.0, 11.5, 12.0, 12.5, 13.0, 13.5, 14.0, 14.5, 15.0, 15.5, 16.0, 16.5, 17.0, 17.5, 18.0, 18.5, 19.0, 19.5, 20.0, 20.5, 21.0, 21.5, 22.0, 22.5, 23.0)
The program where i read the sequence and seq_range
def slidingwindowsegment(sequence, create_segment, compute_error, max_error, seq_range=None):
if not seq_range:
f=0.1
a=0
b=len(sequence)-1
seq_range=[x * f for x in range(a, b)]
seq_range=tuple(seq_range)
#seq_range = (0,len(sequence)-1)
#print (type(seq_range))
start = seq_range[0]
end = start
result_segment = create_segment(sequence,(seq_range[0],seq_range[1]))
while end < seq_range[1]:
end += 1
test_segment = create_segment(sequence,(start,end))
error = compute_error(sequence,test_segment)
if error <= max_error:
result_segment = test_segment
else:
break
if end == seq_range[1]:
return [result_segment]
else:
return [result_segment] + slidingwindowsegment(sequence, create_segment, compute_error, max_error, (end-1,seq_range[1]))
Your line of code is saying that you're accessing the elements of sequence from the index 0.0to 0.5+1 = 1.5. This is clearly not possibile, since 0.0 and 1.5are not an integer values
I wanted to use the built-in range function for floats, but apparently it doesn't work and from a quick research, i understood that there isn't a built in option for that and that I'll need to code my own function for this. So I did:
def fltrange(mini, maxi, step):
lst = []
while mini < maxi:
lst.append(mini)
mini += step
return lst
rang = fltrange(-20.0, 20.1, 0.1)
print(rang)
input()
but this is what I get:
result
the step should be just 0.1000000..., but instead it's about (sometimes it changes) 0.100000000000001.
Thanks in advance.
Fun fact: 1/10 can't be exactly represented by floating point numbers. The closest you can get is 0.1000000000000000055511151231257827021181583404541015625. The rightmost digits usually get left out when you print them, but they're still there. This explains the accumulation of errors as you continually add more 0.1s to the sum.
You can eliminate some inaccuracy (but not all of it) by using a multiplication approach instead of a cumulative sum:
def fltrange(mini, maxi, step):
lst = []
width = maxi - mini
num_steps = int(width/step)
for i in range(num_steps):
lst.append(mini + i*step)
return lst
rang = fltrange(-20.0, 20.1, 0.1)
print(rang)
Result (newlines added by me for clarity):
[-20.0, -19.9, -19.8, -19.7, -19.6, -19.5, -19.4, -19.3, -19.2, -19.1,
-19.0, -18.9, -18.8, -18.7, -18.6, -18.5, -18.4, -18.3, -18.2, -18.1,
-18.0, -17.9, -17.8, -17.7, -17.6, -17.5, -17.4, -17.3, -17.2, -17.1,
-17.0, -16.9, -16.8, -16.7, -16.6, -16.5, -16.4, -16.3, -16.2, -16.1,
-16.0, -15.899999999999999, -15.8, -15.7, -15.6, -15.5, -15.399999999999999, -15.3, -15.2, -15.1, -15.0,
...
19.1, 19.200000000000003, 19.300000000000004, 19.400000000000006, 19.5, 19.6, 19.700000000000003, 19.800000000000004, 19.900000000000006, 20.0]
You can use numpy for it. There are a few functions for your needs.
import numpy as np # of course :)
linspace :
np.linspace(1, 10, num=200)
array([ 1. , 1.04522613, 1.09045226, 1.13567839,
1.18090452, 1.22613065, 1.27135678, 1.31658291,
...
9.68341709, 9.72864322, 9.77386935, 9.81909548,
9.86432161, 9.90954774, 9.95477387, 10. ])
arange :
np.arange(1., 10., 0.1)
array([ 1. , 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2. ,
2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3. , 3.1,
...
8.7, 8.8, 8.9, 9. , 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7,
9.8, 9.9])
P.S. However, it's not technically a generator, which is a range in Python3 (xrange for Python2.x).