I want to plot Real time in a way that updates fast.
The data I have:
arrives via serial port at 62.5 Hz
data corresponds to 32 sensors (so plot 32 lines vs time).
32points *62.5Hz = 2000 points/sec
The problem with my current plotting loop is that it runs slower than 62.5[Hz], meaning I miss some data coming in from serial port.
I am looking for any solution to this problem that allows for:
All data from serial port to be saved.
Plots the data (even skipping a few points/using averages/eliminating old points and only keeping the most recent)
Here is my code, I am using random data to simulate the serial port data.
import numpy as np
import time
import matplotlib.pyplot as plt
#extra plot debugging
hz_ = [] #list of speed
time_=[] #list for time vs Hz plot
#store all data generated
store_data = np.zeros((1, 33))
#only data to plot
to_plot = np.zeros((1, 33))
#color each line
colours = [f"C{i}" for i in range (1,33)]
fig,ax = plt.subplots(1,1, figsize=(10,8))
ax.set_xlabel('time(s)')
ax.set_ylabel('y')
ax.set_ylim([0, 300])
ax.set_xlim([0, 200])
start_time = time.time()
for i in range (100):
loop_time = time.time()
#make data with col0=time and col[1:11] = y values
data = np.random.randint(1,255,(1,32)).astype(float) #simulated data, usually comes in at 62.5 [Hz]
data = np.insert(data, 0, time.time()-start_time).reshape(1,33) #adding time for first column
store_data = np.append(store_data, data , axis=0)
to_plot = store_data[-100:,]
for i in range(1, to_plot.shape[1]):
ax.plot(to_plot[:,0], to_plot[:,i],c = colours[i-1], marker=(5, 2), linewidth=0, label=i)
#ax.lines = ax.lines[-33:] #This soluition speeds it up, to clear old code.
fig.canvas.draw()
fig.canvas.flush_events()
Hz = 1/(time.time()-loop_time)
#for time vs Hz plot
hz_.append(Hz)
time_.append( time.time()-start_time)
print(1/(time.time()-loop_time), "Hz - frequncy program loops at")
#extra fig showing how speed drops off vs time
fig,ax = plt.subplots(1,1, figsize=(10,8))
fig.suptitle('Decreasingn Speed vs Time', fontsize=20)
ax.set_xlabel('time(s)')
ax.set_ylabel('Hz')
ax.plot(time_, hz_)
fig.show()
I also tried while using
ax.lines = ax.lines[-33:]
to remove older points, and this speed up the plotting, but still slower than the speed i aquire data.
Any library/solution to make sure I collect all data and plot the general trendlines (so even not all points) is ok. Maybe something that runs acquiring data and plotting in parallel?
You could try to have two separate processes:
one for acquiring and storing the data
one for plotting the data
Below there are two basic scripts to get the idea.
You first run gen.py which starts to generate numbers and save them in a file.
Then, in the same directory, you can run plot.py which will read the last part of the file and will update the a Matplotlib plot.
Here is the gen.py script to generate data:
#!/usr/bin/env python3
import time
import random
LIMIT_TIME = 100 # s
DATA_FILENAME = "data.txt"
def gen_data(filename, limit_time):
start_time = time.time()
elapsed_time = time.time() - start_time
with open(filename, "w") as f:
while elapsed_time < limit_time:
f.write(f"{time.time():30.12f} {random.random():30.12f}\n") # produces 64 bytes
f.flush()
elapsed = time.time() - start_time
gen_data(DATA_FILENAME, LIMIT_TIME)
and here is the plot.py script to plot the data (reworked from this one):
#!/usr/bin/env python3
import io
import time
import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib.animation
BUFFER_LEN = 64
DATA_FILENAME = "data.txt"
PLOT_LIMIT = 20
ANIM_FILENAME = "video.gif"
fig, ax = plt.subplots(1, 1, figsize=(10,8))
ax.set_title("Plot of random numbers from `gen.py`")
ax.set_xlabel("time / s")
ax.set_ylabel("random number / #")
ax.set_ylim([0, 1])
def get_data(filename, buffer_len, delay=0.0):
with open(filename, "r") as f:
f.seek(0, io.SEEK_END)
data = f.read(buffer_len)
if delay:
time.sleep(delay)
return data
def animate(i, xs, ys, limit=PLOT_LIMIT, verbose=False):
# grab the data
try:
data = get_data(DATA_FILENAME, BUFFER_LEN)
if verbose:
print(data)
x, y = map(float, data.split())
if x > xs[-1]:
# Add x and y to lists
xs.append(x)
ys.append(y)
# Limit x and y lists to 10 items
xs = xs[-limit:]
ys = ys[-limit:]
else:
print(f"W: {time.time()} :: STALE!")
except ValueError:
print(f"W: {time.time()} :: EXCEPTION!")
else:
# Draw x and y lists
ax.clear()
ax.set_ylim([0, 1])
ax.plot(xs, ys)
# save video (only to attach here)
#anim = mpl.animation.FuncAnimation(fig, animate, fargs=([time.time()], [None]), interval=1, frames=3 * PLOT_LIMIT, repeat=False)
#anim.save(ANIM_FILENAME, writer='imagemagick', fps=10)
#print(f"I: Saved to `{ANIM_FILENAME}`")
# show interactively
anim = mpl.animation.FuncAnimation(fig, animate, fargs=([time.time()], [None]), interval=1)
plt.show()
plt.close()
Note that I have also included and commented out the portion of code that I used to generate the animated GIF above.
I believe this should be enough to get you going.
I've got this script running to update every time a new value is added to my csv file (it's a manual log taking in values from a machine):
from itertools import count
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
plt.style.use('fivethirtyeight')
x_vals = []
y_vals = []
counter=0
index = count()
def animate(i):
data = pd.read_csv('C:/Users/Owner/Downloads/Manual-Log-27Nov2020-150846.csv')
data=data.dropna()
data['Date And Time']=data['Date']+' '+data['Time']
Date = data['Date And Time']
Temp = data['Temperature']
Pressure = data['Pressure']
pH = data['pH']
plt.cla()
plt.plot(Date, Temp, label='Temperature')
plt.plot(Date, Pressure, label='Pressure')
plt.plot(Date, pH, label='pH')
plt.legend(loc='upper left')
plt.xlabel('Date and Time')
plt.ylabel('Y Values')
plt.xticks(rotation=90)
plt.tight_layout()
ani = FuncAnimation(plt.gcf(), animate, interval=30000)
plt.tight_layout()
plt.show()
Since it's such a large file (20k lines or so) the resulting graph is huge and you can't really read the data properly. Is there a way I could only get it to show the 20 most recent readings?
you can slice it or get first n records. you can apply the following after your code line : data=data.dropna()
df=pd.DataFrame({'Count':[2,33,4,6,8,9],'apha':['A','B','C','D','E','F']})
df_sorted=sorted_df = df.sort_values(by=['Count'], ascending=True) # in case needed
df_limited=df_sorted.head(3) # this is one you are looking for
I'm trying to plot the exponents of 2 vs the 2 to the power of the exponent but I keep getting a linear graph instead of the curve. I'm not sure what I'm doing wrong.
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
from random import seed
from random import random
import math
tiktok=0
#Desired style
style.use('dark_background')
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
T= open('live_graphText.txt','w+')
T.truncate(0)
T.close()
test = f=open('live_graphText.txt','w+')
for i in range(10):
test.write("%d,%d\n"%(i,math.pow(2,i)))
tiktok = tiktok+i
test.close()
In addition I'm trying to use the live graph animation from matplotlib.animation with a file i created. to keep automatically keep adding points to the file but it seems the function isn't even being called. I'm not sure if I just have it called in the wrong place or the function just doesn't make sense
#This is not being called
def edit(tik):
global tiktok
f=open('live_graphText.txt','a+')
f.write("%d,%d\n"%(tik,math.pow(2,tik)))
print("HelloWorld")
tiktok = tiktok +1
f.close()
def animate(i):
#Opening the file to read
graph_data = open('live_graphText.txt','r').read()
#Split the lines by \n
lines=graph_data.split('\n')
xs =[]
ys =[]
for line in lines:
#This if statement ignores any white space at the end of the file
if len(line)>1:
x,y = line.split(',')
xs.append(x)
ys.append(y)
#print(xs)
#print(ys)
ax1.clear()
ax1.plot(xs,ys)
#The parameters are (Where to plot the function, the function we are plotting, the interval we want to plot in milliseconds)
ani = animation.FuncAnimation(fig,animate,interval=1000)
plt.show()
while tiktok<20:
edit(tiktok)
print(tiktok)
plt.show()
Any help is appreciated!
This Python program plots lines dynamically from data in a csv file. When the program first starts it dynamically draws points that already exist in the file. This part works as expected. I'd like for any new points added to the file to be subsequently drawn. The problem is that i continues to increment so by the time a new item is added to my csv file the value of i is usually much higher than the index from the csv so it never gets plotted. How can I prevent the count of i continuing on until there is an applicable value in the csv file?
import numpy as np
from itertools import count
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
plt.style.use('fivethirtyeight')
data = pd.read_csv('csv_data.csv')
x_vals = []
y_vals1 = []
y_vals2 = []
index = count()
def animate(i):
x = data['x_value']
y1 = data['total_1']
y2 = data['total_2']
x_vals.append(x[i])
y_vals1.append(y1[i])
y_vals2.append(y2[i])
plt.cla()
plt.plot(x_vals, y_vals1, label='Channel 1')
plt.plot(x_vals, y_vals2, label='Channel 2')
plt.legend(loc='upper left')
plt.tight_layout()
ani = FuncAnimation(plt.gcf(), animate, interval=100)
plt.show()
I'm plotting real-time data using parsed data from a file that is being repeatedly opened. I'm deriving and plotting two different values on the same chart. The Y axis scales up and down according the values of each. The "snr" (green) value is plotting fine but the "data_rate"(red) value seems to be static. See screenshot.
import matplotlib.pyplot as plt
import csv
import datetime
from matplotlib.animation import FuncAnimation
import numpy as np
x = []
y = []
z = []
rssi_val = []
def animate(i):
with open('stats.txt', 'r') as searchfile:
time = (searchfile.read(8))
for line in searchfile:
if 'agrCtlRSSI:' in line:
rssi_val = line[16:20]
rssi_val=int(rssi_val)
if 'agrCtlNoise:' in line:
noise_val = (line[16:20])
noise_val=int(noise_val)
if 'maxRate:' in line:
data_rate = (line[16:20])
data_rate=int(data_rate)
snr = ((noise_val - rssi_val) * -1)
#begin test
y.append(snr)
x.append(time)
z.append(data_rate)
#end test
#begin test
plt.cla()
#Verify values are not empty
print("SNR = ", snr)
print("DR = ", data_rate)
plt.plot(snr,label='Signal')
plt.plot(data_rate,label='Data Rate')
plt.legend(loc='upper right')
plt.plot(x,y,z)
ax=plt.gca()
ax.tick_params('x',labelrotation=60)
#end test
ani = FuncAnimation(plt.gcf(), animate, interval=1000)
plt.show()
The simplest way is to call plot twice.
plt.plot(x,y)
plt.plot(x,z)
See "Plotting multiple sets of data" here for more options.