Related
I'm using a Raspberry Pi to plot live data from serial, but eventually run out of memory. I'm not sure if/how I can close the figure, but still have a live data display.
Would it be possible to create and close a new figure with every animate?
My code at the moment:
import serial
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('TkAgg') #comment out for debugging
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import numpy as np
import gc
# Create figure for plotting
fig = plt.figure()
xs = []
ysAC = []
ysDC = []
ser = serial.Serial('/dev/ttyUSB0', 115200, timeout=1)
ser.flush()
# This function is called periodically from FuncAnimation
def animate(i, xs, ysAC, ysDC):
values = getValues()
wAC = values[1]
wDC = values[2]
# Add x and y to lists
xs.append(i)
ysAC.append(wAC)
ysDC.append(wDC)
# Limit x and y lists to 10 items
xs = ['T-9','T-8','T-7','T-6','T-5','T-4','T-3','T-2','T-1','Now']
ysDC = ysDC[-10:]
ysAC = ysAC[-10:]
# Draw x and y lists
axRT1.clear()
if len(ysDC) == 10:
lineAC, = axRT1.plot(xs, ysAC, 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot(xs, ysDC, 'g--', label='Solar', linewidth = 4)
gc.collect()
#fig.clf()
#plt.close()
def getValues():
if ser.in_waiting > 0:
line = ser.readline().decode('utf-8').rstrip()
return list(line.split(","))
# Set up plot to call animate() function periodically
ani = animation.FuncAnimation(fig, animate, fargs=(xs, ysAC, ysDC), interval=1000, blit=False)
plt.get_current_fig_manager().full_screen_toggle()
plt.ioff()
plt.show()
plt.draw()
The crude way of clearing the plots marked below fixed it for me:
import time
import serial
import datetime as dt
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('TkAgg') #comment out for debugging
import matplotlib.pyplot as plt
import matplotlib.cbook as cbook
import matplotlib.animation as animation
from decimal import Decimal
import pandas as pd
import numpy as np
import os.path as path
import re
import gc
import os
count = 0
# Create figure for plotting
fig = plt.figure()
fig.patch.set_facecolor('whitesmoke')
hFont = {'fontname':'sans-serif', 'weight':'bold', 'size':'12'}
xs = ['T-9','T-8','T-7','T-6','T-5','T-4','T-3','T-2','T-1','Now']
ysTemp = []
ysAC = []
ysDC = []
axRT1 = fig.add_subplot(2, 2, 1)
axRT2 = axRT1.twinx() # instantiate a second axes that shares the same x-axis
#Draw x and y lists
axRT1.clear()
axRT2.clear()
axRT1.set_ylim([0, 4])
axRT2.set_ylim([10, 70])
axRT1.set_ylabel('Power Consumption kW', **hFont)
axRT2.set_ylabel('Temperature C', **hFont)
axRT1.set_xlabel('Seconds', **hFont)
axRT1.set_title('Power Consumption and Temperature - Real Time', **hFont)
lineTemp, = axRT2.plot([], [], 'r', label='Temp', linewidth = 4)
lineAC, = axRT1.plot([], [], 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot([], [], 'g--', label='Solar', linewidth = 4)
fig.legend([lineAC, lineDC,lineTemp], ['Mains', 'Solar', 'Temp'], fontsize=20)
ser = serial.Serial('/dev/ttyUSB0', 115200, timeout=1)
ser.flush()
# This function is called periodically from FuncAnimation
def animate(i, xs, ysTemp, ysAC, ysDC):
values = getValues()
if values != 0:
temp_c = Decimal(re.search(r'\d+',values[3]).group())
if temp_c < 0: temp_c = 0
wAC = round(Decimal(re.search(r'\d+', values[1]).group())/1000, 2)
if wAC < 0.35: wAC = 0
aDC = float(re.search(r'\d+', values[2]).group()) #remove characters
vDC = float(re.search(r'\d+', values[4][:5]).group()) #remove characters
wDC = aDC * vDC
wDC = round(abs(Decimal(wDC))/1000, 2)
# Add x and y to lists
ysTemp.append(temp_c)
ysAC.append(wAC)
ysDC.append(wDC)
# Limit x and y lists to 10 items
ysTemp = ysTemp[-10:]
ysDC = ysDC[-10:]
ysAC = ysAC[-10:]
if len(ysTemp) == 10:
axRT2.lines = [] #This crude way of clearing the plots worked
axRT1.lines =[] #This crude way of clearing the plots worked
lineTemp, = axRT2.plot(xs, ysTemp, 'r', label='Temp', linewidth = 4)
lineAC, = axRT1.plot(xs, ysAC, 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot(xs, ysDC, 'g--', label='Solar', linewidth = 4)
def getValues():
measureList = 0
if ser.in_waiting > 0:
line = ser.readline().decode('utf-8').rstrip()
print(line)
if line.count(',') == 4:
measureList = list(line.split(","))
return measureList
# Set up plot to call animate() function periodically
ani = animation.FuncAnimation(fig, animate, fargs=(xs, ysTemp, ysAC, ysDC), interval=1000, blit=False)
plt.get_current_fig_manager().full_screen_toggle()
plt.ioff()
plt.show()
plt.draw()
I am trying to monitor sensor data in real-time, but the plot shows nothing, below is just an example.
can anyone explain to me how come the result is showing nothing?
import datetime
import random
import matplotlib.pyplot as plt
from drawnow import *
from matplotlib.dates import AutoDateLocator, AutoDateFormatter, date2num
i = 0
x = 0
y = 0
FirstTime = str('00:00')
LastTime = str('00:00')
def CreatePlot():
figure = plt.subplot()
plt.plot([],[])
date_datetime = datetime.datetime.strptime(LastTime, '%H:%M')
int_date = date2num( date_datetime)
locator = AutoDateLocator()
figure.xaxis.set_major_locator(locator)
figure.xaxis.set_major_formatter( AutoDateFormatter(locator) )
min_date = date2num( datetime.datetime.strptime(FirstTime, '%H:%M') )
max_date = date2num( datetime.datetime.strptime(LastTime, '%H:%M') )
plt.xlim(min_date, max_date)
plt.plot(x,y, 'r-')
plt.gcf().autofmt_xdate()
while True:
x = datetime.datetime.now() + datetime.timedelta(minutes=i)
x = datetime.datetime.strftime(x,'%H:%M')
if i == 0:
FirstTime = x
else:
LastTime = x
y = (2*i)+2
if i>500:
break
else:
drawnow(CreatePlot)
plt.pause(0.0001)
i+=1
I solved the issue, so I am gonna explain it to help someone else like me,
the first issue is changing date format to string with strftime,
plotting string in x-axis is not auto-formattable,
also following commands are redundant:
min_date = date2num( datetime.datetime.strptime(FirstTime, '%H:%M') )
max_date = date2num( datetime.datetime.strptime(LastTime, '%H:%M') )
plt.xlim(min_date, max_date)
in addition, to make a better view someone can add the following commands too:
from matplotlib.ticker import AutoMinorLocator
from matplotlib.dates import AutoDateLocator, AutoDateFormatter
.
.
.
.
ax0 = plt.subplot(2,2,1)
locator = AutoDateLocator()
ax0.xaxis.set_major_locator(locator)
formatter = AutoDateFormatter(locator)
ax0.xaxis.set_major_formatter(formatter)
ax0.xaxis.set_minor_locator(AutoMinorLocator())
I have code that produces a live graph, updating every few seconds. It all functions EXACTLY as I want other than a single issue, the x axis keeps adding new values but never removing old ones
in the example code below, because I limit the dataframe to 6 columns, I expect to never see more than 6 measurements represented on my x-axis. Instead, the graph continues to update and eventually the poiunts are too close together.
from matplotlib import pyplot
from matplotlib.animation import FuncAnimation
import pandas as pd
from datetime import datetime
import threading
import random
import time
measurements = ['abc','bcd','afr','reg','wow']
counter = 0
figure = pyplot.figure()
measurement_frame = pd.DataFrame(index = measurements)
def get_live(counter2, col_num):
measurement_frame.iat[counter2,col_num] = random.randint(50,80)
def add_to_dataframe():
global measurement_frame
#timey = datetime.now().strftime('%H:%M:%S')
timey = datetime.now().time()
if measurement_frame.shape[1] == 6:
measurement_frame.drop(measurement_frame.columns[0], axis = 1, inplace = True)
measurement_frame[timey] = measurements
col_num = measurement_frame.shape[1]-1
print(col_num)
counter2 = 0
for item in measurements:
t = threading.Thread(target=get_live, args=(counter2, col_num,))
t.start()
counter2 = counter2 +1
t.join()
print(measurement_frame.columns[0])
time.sleep(1)
def update(frame):
add_to_dataframe()
x_data = measurement_frame.columns
print(x_data[0])
y1_data = measurement_frame.loc[measurement_frame.index[0]]
y2_data = measurement_frame.loc[measurement_frame.index[1]]
y3_data = measurement_frame.loc[measurement_frame.index[2]]
y4_data = measurement_frame.loc[measurement_frame.index[3]]
y5_data = measurement_frame.loc[measurement_frame.index[4]]
line, = pyplot.plot_date(x_data, y1_data, '-', color = 'b')
line2, = pyplot.plot_date(x_data, y2_data, '-', color = 'g')
line3, = pyplot.plot_date(x_data, y3_data, '-', color = 'r')
line4, = pyplot.plot_date(x_data, y4_data, '-', color = 'm')
line5, = pyplot.plot_date(x_data, y5_data, '-', color = 'y')
line.set_data(x_data, y1_data)
line2.set_data(x_data, y2_data)
line3.set_data(x_data, y3_data)
line4.set_data(x_data, y4_data)
line5.set_data(x_data, y5_data)
figure.gca().set_xlim(x_data[0])
figure.gca().autoscale()
print(figure.gca().get_xlim())
return line, line2, line3, line4, line5,
animation = FuncAnimation(figure, update, interval=1000)
pyplot.show()
What I need is that after the dataframe reaches maximum size, the far left measurements are removed, so as to not exceed a set number of measurements on the screen at once. Note that the dataframe already drops unneeded columns before adding a new one when it reaches a certain size, but my graph does not reflect that
using autoscale tries to keep old data in view. If you drop autoscale and use
figure.gca().set_xlim(left =x_data[0], right = datetime.now().time())
it works as intended
the full code is now
from matplotlib import pyplot
from matplotlib.animation import FuncAnimation
import pandas as pd
from datetime import datetime
import threading
import random
import time
measurements = ['abc','bcd','afr','reg','wow']
counter = 0
figure = pyplot.figure()
measurement_frame = pd.DataFrame(index = measurements)
def get_live(counter2, col_num):
measurement_frame.iat[counter2,col_num] = random.randint(50,80)
def add_to_dataframe():
global measurement_frame
#timey = datetime.now().strftime('%H:%M:%S')
timey = datetime.now().time()
if measurement_frame.shape[1] == 6:
measurement_frame.drop(measurement_frame.columns[0], axis = 1, inplace = True)
measurement_frame[timey] = measurements
col_num = measurement_frame.shape[1]-1
print(col_num)
counter2 = 0
for item in measurements:
t = threading.Thread(target=get_live, args=(counter2, col_num,))
t.start()
counter2 = counter2 +1
t.join()
print(measurement_frame.columns[0])
time.sleep(1)
def update(frame):
add_to_dataframe()
x_data = measurement_frame.columns
print(x_data[0])
y1_data = measurement_frame.loc[measurement_frame.index[0]]
y2_data = measurement_frame.loc[measurement_frame.index[1]]
y3_data = measurement_frame.loc[measurement_frame.index[2]]
y4_data = measurement_frame.loc[measurement_frame.index[3]]
y5_data = measurement_frame.loc[measurement_frame.index[4]]
line, = pyplot.plot_date(x_data, y1_data, '-', color = 'b')
line2, = pyplot.plot_date(x_data, y2_data, '-', color = 'g')
line3, = pyplot.plot_date(x_data, y3_data, '-', color = 'r')
line4, = pyplot.plot_date(x_data, y4_data, '-', color = 'm')
line5, = pyplot.plot_date(x_data, y5_data, '-', color = 'y')
line.set_data(x_data, y1_data)
line2.set_data(x_data, y2_data)
line3.set_data(x_data, y3_data)
line4.set_data(x_data, y4_data)
line5.set_data(x_data, y5_data)
figure.gca().set_xlim(left =x_data[0], right = datetime.now().time())
print(figure.gca().get_xlim())
return line, line2, line3, line4, line5,
animation = FuncAnimation(figure, update, interval=1000)
pyplot.show()
I have folder names which are date time formated as
2018-08-21 to 2018-10-16
Inside each folder there is a zip files which contains time values which is a linear scale, the time goes up linearly.
I'm trying to plot for each day, which has a lot of .bz2 file time series data, the time value at that date.
Right now I'm trying to do that:
timearr = np.asarray(data1['time'])
ax.plot(np.asarray(timeStamps), timearr)
ax.set_title('title')
ax.set_ylabel('date vs time ')
ax.grid(True)
# Format the x-axis for dates (label formatting, rotation)
fig.autofmt_xdate(rotation=45)
fig.tight_layout()
plt.show()
but I get an error message, that both dimensions doesn't match.
dateStamps are list[2018-08-21
2018-08-22
2018-08-23
2018-08-24
2018-08-25]
data1['time'] = list of EPOC values.
Unfortunately, I don't know the detailed file structure, so I have to guess a little were the problem actually is
Here is some code to generate some folders with generic bz2 files:
import bz2
import numpy as np
import datetime
import os
startDate = datetime.datetime(2000,5,2,10,15,0,0)
for day in range(5):
theDate = startDate + datetime.timedelta(days=day)
folder = "{}".format( theDate.replace( microsecond = 0 ).strftime("%Y-%m-%d") )
os.mkdir( folder )
data = ""
for k in range(100):
zzz = theDate + datetime.timedelta(seconds=137*k)
data += "{} ".format( zzz.replace( microsecond = 0 ).strftime("%H:%M:%S") )
d = zzz.day
m = zzz.minute
data += " {}\n".format( .17 * d + .003 * m**2 -.001 * m )
myZip = bz2.BZ2File(os.path.join( folder, 'dat.bz2' ), 'w' )
myZip.write( data )
myZip.close()
Those folders and files a treat with:
import bz2
import numpy as np
import datetime
import os
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
"""
SE posts I used
https://stackoverflow.com/questions/1574088/plotting-time-in-python-with-matplotlib
https://stackoverflow.com/questions/11264521/date-ticks-and-rotation-in-matplotlib
"""
def split_data( inData ):
rows=data.strip().split('\n')
rowcol = [x.split() for x in rows ]
x,y = zip(*rowcol)
y = [float(z) for z in y ]
x = [ datetime.datetime.strptime(z, '%H:%M:%S') for z in x]
return x,y
dataDict = dict()
for root, dirs, files in os.walk("."):
for name in files:
if name.split('.')[-1]=='bz2':
base = os.path.basename( root )
myPath = (os.path.join(root, name))
bz = bz2.BZ2File( myPath, 'r' )
data = bz.read()
dataDict[ base ] = split_data( data )
myFmt = mdates.DateFormatter('%H:%M')
fig = plt.figure()
ax = fig.add_subplot( 1, 1, 1 )
for key, dt in dataDict.iteritems():
ax.plot( *dt , label=key )
ax.xaxis.set_major_formatter(myFmt)
for label in ax.get_xmajorticklabels():
label.set_rotation(30)
ax.set_ylabel('data (arb. u.)')
ax.set_xlabel('time')
ax.legend( loc=0 )
plt.tight_layout()
plt.show()
Providing:
Hope I got it right.
Can anybody help how to optimize the plot function in python? I use Matplotlib to plot financial data.Here small function for plotting OHLC data. The time increase significantly if I add indicators or other data.
import numpy as np
import datetime
from matplotlib.collections import LineCollection
from pylab import *
import urllib2
def test_plot(OHLCV):
bar_width = 1.3
date_offset = 0.5
fig = figure(figsize=(50, 20), facecolor='w')
ax = fig.add_subplot(1, 1, 1)
labels = ax.get_xmajorticklabels()
setp(labels, rotation=0)
month = MonthLocator()
day = DayLocator()
timeFmt = DateFormatter('%Y-%m-%d')
colormap = OHLCV[:,1] < OHLCV[:,4]
color = np.zeros(colormap.__len__(), dtype = np.dtype('|S5'))
color[:] = 'red'
color[np.where(colormap)] = 'green'
dates = date2num( OHLCV[:,0])
lines_hl = LineCollection( zip(zip(dates, OHLCV[:,2]), zip(dates, OHLCV[:,3])))
lines_hl.set_color(color)
lines_hl.set_linewidth(bar_width)
lines_op = LineCollection( zip(zip((np.array(dates) - date_offset).tolist(), OHLCV[:,1]), zip((np.array(dates)).tolist(), parsed_table[:,1])))
lines_op.set_color(color)
lines_op.set_linewidth(bar_width)
lines_cl = LineCollection( zip(zip((np.array(dates) + date_offset).tolist(), OHLCV[:,4]), zip((np.array(dates)).tolist(), parsed_table[:,4])))
lines_cl.set_color(color)
lines_cl.set_linewidth(bar_width)
ax.add_collection(lines_hl, autolim=True)
ax.add_collection(lines_cl, autolim=True)
ax.add_collection(lines_op, autolim=True)
ax.xaxis.set_major_locator(month)
ax.xaxis.set_major_formatter(timeFmt)
ax.xaxis.set_minor_locator(day)
ax.autoscale_view()
ax.xaxis.grid(True, 'major')
ax.grid(True)
ax.set_title('EOD test plot')
ax.set_xlabel('Date')
ax.set_ylabel('Price , $')
fig.savefig('test.png', dpi = 50, bbox_inches='tight')
close()
if __name__=='__main__':
data_table = urllib2.urlopen(r"http://ichart.finance.yahoo.com/table.csv?s=IBM&a=00&b=1&c=2012&d=00&e=15&f=2013&g=d&ignore=.csv").readlines()[1:][::-1]
parsed_table = []
#Format: Date, Open, High, Low, Close, Volume
dtype = (lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(),float, float, float, float, int)
for row in data_table:
field = row.strip().split(',')[:-1]
data_tmp = [i(j) for i,j in zip(dtype, field)]
parsed_table.append(data_tmp)
parsed_table = np.array(parsed_table)
import time
bf = time.time()
count = 100
for i in xrange(count):
test_plot(parsed_table)
print('Plot time: %s' %(time.time() - bf) / count)
The result is something like this. Average time execution on each plot is aproximately 2.6s. Charting in R is much faster, but I didn't measure the performance and I don't want use Rpy, so I bielive that my code is inefficient.
This solution reuses a Figure instance and saves plots asynchronously. You could change this to have as many figures as there are processors, do that many plots asynchronously, and it should speed things up even more. As it is, this takes ~1s per plot, down from 2.6 on my machine.
import numpy as np
import datetime
import urllib2
import time
import multiprocessing as mp
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pylab import *
from matplotlib.collections import LineCollection
class AsyncPlotter():
def __init__(self, processes=mp.cpu_count()):
self.manager = mp.Manager()
self.nc = self.manager.Value('i', 0)
self.pids = []
self.processes = processes
def async_plotter(self, nc, fig, filename, processes):
while nc.value >= processes:
time.sleep(0.1)
nc.value += 1
print "Plotting " + filename
fig.savefig(filename)
plt.close(fig)
nc.value -= 1
def save(self, fig, filename):
p = mp.Process(target=self.async_plotter,
args=(self.nc, fig, filename, self.processes))
p.start()
self.pids.append(p)
def join(self):
for p in self.pids:
p.join()
class FinanceChart():
def __init__(self, async_plotter):
self.async_plotter = async_plotter
self.bar_width = 1.3
self.date_offset = 0.5
self.fig = plt.figure(figsize=(50, 20), facecolor='w')
self.ax = self.fig.add_subplot(1, 1, 1)
self.labels = self.ax.get_xmajorticklabels()
setp(self.labels, rotation=0)
line_hl = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
line_op = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
line_cl = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
self.lines_hl = self.ax.add_collection(line_hl, autolim=True)
self.lines_op = self.ax.add_collection(line_cl, autolim=True)
self.lines_cl = self.ax.add_collection(line_op, autolim=True)
self.ax.set_title('EOD test plot')
self.ax.set_xlabel('Date')
self.ax.set_ylabel('Price , $')
month = MonthLocator()
day = DayLocator()
timeFmt = DateFormatter('%Y-%m-%d')
self.ax.xaxis.set_major_locator(month)
self.ax.xaxis.set_major_formatter(timeFmt)
self.ax.xaxis.set_minor_locator(day)
def test_plot(self, OHLCV, i):
colormap = OHLCV[:,1] < OHLCV[:,4]
color = np.zeros(colormap.__len__(), dtype = np.dtype('|S5'))
color[:] = 'red'
color[np.where(colormap)] = 'green'
dates = date2num( OHLCV[:,0])
date_array = np.array(dates)
xmin = min(dates)
xmax = max(dates)
ymin = min(OHLCV[:,1])
ymax = max(OHLCV[:,1])
self.lines_hl.set_segments( zip(zip(dates, OHLCV[:,2]), zip(dates, OHLCV[:,3])))
self.lines_hl.set_color(color)
self.lines_hl.set_linewidth(self.bar_width)
self.lines_op.set_segments( zip(zip((date_array - self.date_offset).tolist(), OHLCV[:,1]), zip(date_array.tolist(), OHLCV[:,1])))
self.lines_op.set_color(color)
self.lines_op.set_linewidth(self.bar_width)
self.lines_cl.set_segments( zip(zip((date_array + self.date_offset).tolist(), OHLCV[:,4]), zip(date_array.tolist(), OHLCV[:,4])))
self.lines_cl.set_color(color)
self.lines_cl.set_linewidth(self.bar_width)
self.ax.set_xlim(xmin,xmax)
self.ax.set_ylim(ymin,ymax)
self.ax.xaxis.grid(True, 'major')
self.ax.grid(True)
self.async_plotter.save(self.fig, '%04i.png'%i)
if __name__=='__main__':
print "Starting"
data_table = urllib2.urlopen(r"http://ichart.finance.yahoo.com/table.csv?s=IBM&a=00&b=1&c=2012&d=00&e=15&f=2013&g=d&ignore=.csv").readlines()[1:][::-1]
parsed_table = []
#Format: Date, Open, High, Low, Close, Volume
dtype = (lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(),float, float, float, float, int)
for row in data_table:
field = row.strip().split(',')[:-1]
data_tmp = [i(j) for i,j in zip(dtype, field)]
parsed_table.append(data_tmp)
parsed_table = np.array(parsed_table)
import time
bf = time.time()
count = 10
a = AsyncPlotter()
_chart = FinanceChart(a)
print "Done with startup tasks"
for i in xrange(count):
_chart.test_plot(parsed_table, i)
a.join()
print('Plot time: %.2f' %(float(time.time() - bf) / float(count)))