Develop Reference

Plotting or time.sleep or csv saving datas in a while loop induces lags between sensed and read datas from a force sensor

I am quite fresh using Python. I need it to acquire serial datas from a force sensor plugged into a COMPORT (6). With the code below, I have no problem to store the datas in a list and save it afterwards. I can also print each data without noticing any lag.
However, when I try to implement a plot in my while loop, a rather annoying shift appears between the time I touch my sensor and the datas are written (from few to tens of seconds). I first thought it was because matplotlib is a memory consuming library, but even when i add the simple line "time.sleep(0.00001), which is a very short pause compared to the rate of acquisition (60 FPS), I get the same lags. I also tried to save my datas in a csv file and plot the datas in a different function by using multiprocess but even saving datas triggers the lag also.
This is problematic as visualizing my live datas is an important part of my experiment.
Could someone please help me with this particular issue ?
Thank you so much.
import serial
import struct
import platform
import multiprocessing
import time
import numpy as np
import csv
# from pylab import *
import matplotlib.pyplot as plt
class MeasurementConverter:
def convertValue(self, bytes):
pass
class ForceMeasurementConverterKG(MeasurementConverter):
def __init__(self, F_n, S_n, u_e):
self.F_n = F_n
self.S_n = S_n
self.u_e = u_e
def convertValue(self, value):
A = struct.unpack('>H', value)[0]
# return (A - 0x8000) * (self.F_n / self.S_n) * (self.u_e / 0x8000)
return self.F_n / self.S_n * ((A - 0x8000) / 0x8000) * self.u_e * 2
class GSV3USB:
def __init__(self, com_port, baudrate=38400):
com_path = f'/dev/ttyUSB{com_port}' if platform.system(
) == 'Linux' else f'COM{com_port}'
# print(f'Using COM: {com_path}')
self.sensor = serial.Serial(com_path, baudrate)
self.converter = ForceMeasurementConverterKG(10, 0.499552, 2)
def read_value(self):
self.sensor.read_until(b'\xA5')
read_val = self.sensor.read(2)
return self.converter.convertValue(read_val)
# initialization of datas
gsv_data=[]
temps=[]
t_0=time.time()
def data_gsv():
dev = GSV3USB(6)
# fig=plt.figure()
# ax = fig.add_subplot(111)
i=0
# line1, = ax.plot(temps, gsv_data)
try:
while True:
gsv_data.append(dev.read_value())
t1=time.time()-t_0
temps.append(t1)
# I can print the datas without noticing any lags
print(dev.read_value())
# I cannot plot the datas without noticing any lags
plt.plot(temps,gsv_data)
plt.draw ()
plt.axis([temps[i]-6,temps[i]+6,-2,10])
plt.pause(0.00001)
plt.clf()
i=i+1
# I cannot pause without noticing any lags
time.sleep(0.0001)
# I cannot save datas without noticing any lags
with open('student_gsv.csv', 'w') as f:
write = csv.writer(f)
write.writerow(gsv_data)
except KeyboardInterrupt:
print("Exiting")
return
if __name__ == "__main__":
data_gsv()```

Your main loop is constructed in a way that each time when you acquire a value with gsv_data.append(dev.read_value()), the plot is drawn again. That probably takes time, and if the data frequency with that the measuring device (GSV-3 USB) send measured data is quite high - let's say >10 frames/s - you'll get these "lags".
I would lower the plot update rate:
append the measured data to an array (as you already do)
Use a counter or a comparison of time difference to get a plot update rate of about 4 times per second (250ms). This is an update rate often used for being consumed by the human eye.

Python : AttributeError: 'Serial' object has no attribute 'is_open' for real time data

hello i have a problem with my current code. i want to plot a real time signal for my ecg from my udoo. buat when i connect the serial it getting like this :
Traceback (most recent call last):
File "plot_serial.py", line 10, in <module>
if userial.is_open:
AttributeError: 'Serial' object has no attribute 'is_open'
here's my code :
import serial
import time
import matplotlib.pyplot as plt
import numpy as np
ubaudrate = 9600
uport = '/dev/ttyMCC' # set the correct port before run it
userial = serial.Serial(uport, ubaudrate, timeout=1)
userial.timeout = 10
if userial.is_open:
while True:
size = userial.inWaiting()
if size:
data = userial.read(size)
plt.figure
plt.plot(size, data, 'b', alpha=0.75)
plt.legend(('Sinyal Jantung'), loc='best')
plt.grid(True)
plt.show()
#print data
else:
print ('no data')
time.sleep(1)
else:
print ('serial not open')
# z1serial.close() # close z1serial if z1serial is open.
thanks for your help before, since i'm newbie in python programming

It seems you are using older version of pySerial. You can either get later version of pySerial, or use isOpen() to replace is_open.
The data from serial.read is likely str, so you need to convert it into numeric.
data = np.float64(np.frombuffer(data.encode(), 'B'))
This supposes each byte of the data represents a point.
I also see another problem in your code to plot. You may drop the first input 'size', like
plt.plot(data, 'b', alpha=0.75)

how to make matplotlib update from continuous data source?

I am trying to make a simple graph from a data gathered from a continuous real-time data source.
My code for using matplotlib is below:
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
from serialdata import SerialData
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
def animate(i):
xar = []
yar = []
#Open Serial Port and Receive Continuous Data
#in format of number,number
a = SerialData()
b = a.setSerial('COM3', 9600)
dataArray = a.getSerial(9999)
for eachLine in dataArray:
if len(eachLine)>1:
x,y = eachLine.split(',')
xar.append(int(x))
yar.append(int(y))
ax1.clear()
ax1.plot(xar,yar)
ani = animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
The serialdata.py just yields the data every time it gets from the data source:
import serial
from time import sleep
class SerialData:
def __init__(self):
pass
def setSerial(self, port, baudrate):
self.port = port
self.baudrate = baudrate
print("Opening Serial Port...")
self.ser = serial.Serial(self.port, self.baudrate, timeout=1)
sleep(2)
print("Setup Successful")
def getSerial(self, read):
while True:
self.data = self.ser.read(read)
if len(self.data)>0:
yield self.data.decode('utf-8')
sleep(.1)
self.ser.close()
supposedly it should send data in form of:
1,2
2,5
3,7
4(autoincrement),5(random number)
and works just fine when I just make them print on the CLI.
However I cannot make it work with the matplotlib.
There is no specific error.
It just shows
Opening Serial Port...
Setup Successful
and... thats it. nothing happens then.
What is wrong with my code?
I did more research and found that I shouldn't be using show()
so I rewrote my code as following:
import time
import numpy as np
import matplotlib.pyplot as plt
from serialdata import SerialData
plt.axis([0, 1000, 0, 1])
plt.ion()
plt.show()
for i in range(1000):
# y = np.random.random()
a = SerialData()
b = a.setSerial('COM3', 9600)
dataArray = a.getSerial(9999)
print("Data Gathering...")
for eachLine in dataArray:
if len(eachLine)>1:
y = eachLine.split(',')[1]
plt.scatter(i, y)
plt.draw()
time.sleep(0.05)
But, the result is the same.

I don't have a serial port, but this is what I tried to do to figure it out:
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
import random
from time import sleep
class MockData():
def __init__(self, n):
self.n = n #number of data points to return
def getData(self):
start = 0 #mock autoincrement
for i in range(self.n):
yield (start, random.randint(0, 100)) #yield a tuple, not a str (x(autoincrem),y(random))
start+=1
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
def animate(i):
xar = []
yar = []
#Open Serial Port and Receive Continuous Data
#in format of number,number
a = MockData(10)
dataArray = a.getData() #this is a generator that will yield (x,y) tuples 10 times.
for eachLine in dataArray:
x,y = eachLine
xar.append(int(x))
yar.append(int(y))
ax1.clear()
ax1.plot(xar,yar)
ani = animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
This code will fail\be very slow:
If I request a large amount of data at a time: a=MockData(1000) will execute for ~2-3seconds per frame
If readout times are long i.e.
def getData(self):
start = 0
for i in range(self.n):
yield (start, random.randint(0, 100))
start+=1
time.sleep(1)
this will execute ~10 seconds per frame
Both
So as far as I can conclude issue is in your SerialData class, specifically in the getSerial method. I know that because that's the method that's responsible for retrieving the actual data. Since I don't have a serial port I can't exactly test where, but I can wager a few guesses.
def getSerial(self, read):
while True:
self.data = self.ser.read(read)
if len(self.data)>0:
yield self.data.decode('utf-8')
sleep(.1)
self.ser.close()
self.data = self.ser.read(read) Issue here is that you request 9999 bytes to be read. 9600 bauds is about 1200 bytes/s. To read 9 999 bytes it would take ~10seconds. And that's if there's actually 9999 new bytes to be read. If there weren't the read function would continue waiting until it read in that amount. This is equal to my 1) test case, except the waiting period would be sleep(10). The if check therefore is already in the read function so you don't have to check again.
self.data.decode('utf-8') Check out how long it takes to decode 9999 bytes:
>>> from timeit import Timer
>>> ts = Timer("s.decode('utf-8')", "s = b'1'*9999")
>>> ts.timeit()
2.524627058740407
Now, granted, this isn't the same conversion as yours, but since I don't have a serial port on my laptop I can't test it. Anyhow, it seems like it's very slow. Right now, you have my case 2) with sleep(12)
sleep(.1) this now just seems like adding an insult to injury.
Your code doesn't report an error because it works, it just takes over 3 minutes to read in the first set of data and plot it.
My recommendation is that you ignore this approach, read in data, almost literally, byte per byte, plot them as they come. Have an array in which you just append new bytes as they come and plot that. You could easily pull something like:
serialPort = serial.Serial(self.port, self.baudrate, timeout=1)
data = list() #some global list to append to
def animate():
d = serialPort.read(2) #this will hang until it completes
data.append(d) #does the data really have to be in string?
ax.plot(data)
ani = animation.FuncAnimation(fig, animate, interval=1000) #optionally send in repeat_delay?
plt.show()
though part about this is that I'm not sure how animation behaves if the data array starts getting really big, so you might consider shifting your x axis to the right, deleting your old data and creating a new array periodically.
Hopefully I was of some help in pointing out which things you need to test for. Start timing all your read-ins from serial port to get a feeling for how long do things take. Don't forget to make sure you're actually reading anything at all from the serial port also, if you're not, the serial.read will just wait there.
I've also never done anything like this, so it's possible I'm way off course. Last time I dealt with serial ports was robotics championship from elementary (rofl).

Plotting 3D Data from Gyro Encoder

I am collecting data from the gyro encoder via serial port. I would like to plot the real time data. Below is the python code:
import serial
import struct
import numpy as np
#import csv
ser = serial.Serial("COM5",115200)
buffer = []
while 1:
buffer += ser.read(ser.inWaiting())
'\x7e\x5d' in 'buffer'
val = ser.read(18)
(NA, timestamp, rsvd, gyro_xout, gyro_yout, gyro_zout, level, encoder_angle) = struct.unpack(">HHhhhhlh",val)
print (NA, timestamp, rsvd, gyro_xout, gyro_yout, gyro_zout, level, encoder_angle)
I want to plot gyro_xout, gyro_yout, and gyro_zout. I am new to python. I would greatly appreciate if someone can help in plotting the 3D values. Thanks!!

Reading real time values with pySerial while plotting

So here is the deal, I have a module which sends out data over the serial port at 9600 baud and I am using the matplotlib to plot that data in real time. I wrote this code
#! python
############ import section ###################
import serial
import struct
import numpy as np
import matplotlib.pyplot as plt
###############################################
############ Serial Configuration #############
com = serial.Serial(14,9600)
print ("Successfully opened " + com.portstr)
###############################################
def get_new_values():
s = com.read(20)
raw_data = struct.unpack('!BBBBBBBBBBBBBBBBBBBB', s)
j = 0;
norm_data = []
for i in range(0,20,2):
big_byte = raw_data[i+1]+(raw_data[i]*256)
norm_data.append(big_byte)
j = j+1
return norm_data
x = range(0,10,1)
y = get_new_values()
plt.ion()
line, = plt.plot(x,y)
while(1):
a = get_new_values()
line.set_ydata(a)
plt.draw()
com.close()
print ("Port Closed Successfully")
I get 20 bytes and then make 10 big bytes (2 byte data is split as two 1 byte values while transmitting). But I just noticed that I am not getting real time values from this. I am on windows 7 home basic if that matters.
Any clue why this is happening?
EDIT
Also, whenever I click on the plot it hangs.

I know the code below seems long and overly complex for your simple problem, but manual optimization is usually more code and more potential bugs. That's why premature optimization is almost always a mistake.
In __init__ it sets up the plot, setting references for the axis, canvas, line (it starts with a line drawn off screen), and the pre-animation background. Additionally __init__ registers callbacks to handle resizing and shutdown. The on_resize callback is needed to update the background (used for the blit) when the window is resized. The on_close callback uses a lock to update the running status. I haven't eliminated all race conditions with this, but this works to prevent a _tkinter.TclError caused by trying to blit to a terminated Tk app. I only tested with Tk, and on only one machine. YMMV, and I'm open to suggestions.
In the run method I've added a call to canvas.flush_events(). This should keep the plot window from hanging if you try to drag the window around and resize it. The while loop in this method calls self.get_new_values() to set the data in the plot. It then updates the plot using the configured method. If self.blit is true, it uses canvas.blit, else it uses pyplot.draw.
The variable spf (samples per frame) controls how many samples are plotted in a frame. You can use it in your implementation of get_new_values to determine the number of bytes to read (e.g. 2 * self.spf for 2 bytes per sample). I've set the default to 120, which is 5 frames per second if your data rate is 600 samples per second. You have to find the sweet spot that maximizes throughput vs time resolution in the graph while also keeping up with the incoming data.
Reading your data into a NumPy array instead of using a Python list will likely speed up processing. Also, it would give you easy access to tools to downsample and analyze the signal. You can read into a NumPy array directly from a byte string, but make sure you get the endianess right:
>>> data = b'\x01\xff' #big endian 256 + 255 = 511
>>> np.little_endian #my machine is little endian
True
>>> y = np.fromstring(data, dtype=np.uint16); y #so this is wrong
array([65281], dtype=uint16)
>>> if np.little_endian: y = y.byteswap()
>>> y #fixed
array([511], dtype=uint16)
Code:
from __future__ import division
from matplotlib import pyplot
import threading
class Main(object):
def __init__(self, samples_per_frame=120, blit=True):
self.blit = blit
self.spf = samples_per_frame
pyplot.ion()
self.ax = pyplot.subplot(111)
self.line, = self.ax.plot(range(self.spf), [-1] * self.spf)
self.ax.axis([0, self.spf, 0, 65536])
pyplot.draw()
self.canvas = self.ax.figure.canvas
self.background = self.canvas.copy_from_bbox(self.ax.bbox)
self.canvas.mpl_connect('resize_event', self.on_resize)
self.canvas.mpl_connect('close_event', self.on_close)
self.lock = threading.Lock()
self.run()
def get_new_values(self):
import time
import random
#simulate receiving data at 9600 bps (no cntrl/crc)
time.sleep(2 * self.spf * 8 / 9600)
y = [random.randrange(65536) for i in range(self.spf)]
self.line.set_ydata(y)
def on_resize(self, event):
self.line.set_ydata([-1] * self.spf)
pyplot.draw()
self.background = self.canvas.copy_from_bbox(self.ax.bbox)
def on_close(self, event):
with self.lock:
self.running = False
def run(self):
with self.lock:
self.running = True
while self.running:
self.canvas.flush_events()
with self.lock:
self.get_new_values()
if self.running:
if self.blit:
#blit for a higher frame rate
self.canvas.restore_region(self.background)
self.ax.draw_artist(self.line)
self.canvas.blit(self.ax.bbox)
else:
#versus a regular draw
pyplot.draw()
if __name__ == '__main__':
Main(samples_per_frame=120, blit=True)