Develop Reference

Matplotlib live data animation runs out of memory

I'm using a Raspberry Pi to plot live data from serial, but eventually run out of memory. I'm not sure if/how I can close the figure, but still have a live data display.
Would it be possible to create and close a new figure with every animate?
My code at the moment:
import serial
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('TkAgg') #comment out for debugging
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import numpy as np
import gc
# Create figure for plotting
fig = plt.figure()
xs = []
ysAC = []
ysDC = []
ser = serial.Serial('/dev/ttyUSB0', 115200, timeout=1)
ser.flush()
# This function is called periodically from FuncAnimation
def animate(i, xs, ysAC, ysDC):
values = getValues()
wAC = values[1]
wDC = values[2]
# Add x and y to lists
xs.append(i)
ysAC.append(wAC)
ysDC.append(wDC)
# Limit x and y lists to 10 items
xs = ['T-9','T-8','T-7','T-6','T-5','T-4','T-3','T-2','T-1','Now']
ysDC = ysDC[-10:]
ysAC = ysAC[-10:]
# Draw x and y lists
axRT1.clear()
if len(ysDC) == 10:
lineAC, = axRT1.plot(xs, ysAC, 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot(xs, ysDC, 'g--', label='Solar', linewidth = 4)
gc.collect()
#fig.clf()
#plt.close()
def getValues():
if ser.in_waiting > 0:
line = ser.readline().decode('utf-8').rstrip()
return list(line.split(","))
# Set up plot to call animate() function periodically
ani = animation.FuncAnimation(fig, animate, fargs=(xs, ysAC, ysDC), interval=1000, blit=False)
plt.get_current_fig_manager().full_screen_toggle()
plt.ioff()
plt.show()
plt.draw()

The crude way of clearing the plots marked below fixed it for me:
import time
import serial
import datetime as dt
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('TkAgg') #comment out for debugging
import matplotlib.pyplot as plt
import matplotlib.cbook as cbook
import matplotlib.animation as animation
from decimal import Decimal
import pandas as pd
import numpy as np
import os.path as path
import re
import gc
import os
count = 0
# Create figure for plotting
fig = plt.figure()
fig.patch.set_facecolor('whitesmoke')
hFont = {'fontname':'sans-serif', 'weight':'bold', 'size':'12'}
xs = ['T-9','T-8','T-7','T-6','T-5','T-4','T-3','T-2','T-1','Now']
ysTemp = []
ysAC = []
ysDC = []
axRT1 = fig.add_subplot(2, 2, 1)
axRT2 = axRT1.twinx() # instantiate a second axes that shares the same x-axis
#Draw x and y lists
axRT1.clear()
axRT2.clear()
axRT1.set_ylim([0, 4])
axRT2.set_ylim([10, 70])
axRT1.set_ylabel('Power Consumption kW', **hFont)
axRT2.set_ylabel('Temperature C', **hFont)
axRT1.set_xlabel('Seconds', **hFont)
axRT1.set_title('Power Consumption and Temperature - Real Time', **hFont)
lineTemp, = axRT2.plot([], [], 'r', label='Temp', linewidth = 4)
lineAC, = axRT1.plot([], [], 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot([], [], 'g--', label='Solar', linewidth = 4)
fig.legend([lineAC, lineDC,lineTemp], ['Mains', 'Solar', 'Temp'], fontsize=20)
ser = serial.Serial('/dev/ttyUSB0', 115200, timeout=1)
ser.flush()
# This function is called periodically from FuncAnimation
def animate(i, xs, ysTemp, ysAC, ysDC):
values = getValues()
if values != 0:
temp_c = Decimal(re.search(r'\d+',values[3]).group())
if temp_c < 0: temp_c = 0
wAC = round(Decimal(re.search(r'\d+', values[1]).group())/1000, 2)
if wAC < 0.35: wAC = 0
aDC = float(re.search(r'\d+', values[2]).group()) #remove characters
vDC = float(re.search(r'\d+', values[4][:5]).group()) #remove characters
wDC = aDC * vDC
wDC = round(abs(Decimal(wDC))/1000, 2)
# Add x and y to lists
ysTemp.append(temp_c)
ysAC.append(wAC)
ysDC.append(wDC)
# Limit x and y lists to 10 items
ysTemp = ysTemp[-10:]
ysDC = ysDC[-10:]
ysAC = ysAC[-10:]
if len(ysTemp) == 10:
axRT2.lines = [] #This crude way of clearing the plots worked
axRT1.lines =[] #This crude way of clearing the plots worked
lineTemp, = axRT2.plot(xs, ysTemp, 'r', label='Temp', linewidth = 4)
lineAC, = axRT1.plot(xs, ysAC, 'b:', label='Mains', linewidth = 4)
lineDC, = axRT1.plot(xs, ysDC, 'g--', label='Solar', linewidth = 4)
def getValues():
measureList = 0
if ser.in_waiting > 0:
line = ser.readline().decode('utf-8').rstrip()
print(line)
if line.count(',') == 4:
measureList = list(line.split(","))
return measureList
# Set up plot to call animate() function periodically
ani = animation.FuncAnimation(fig, animate, fargs=(xs, ysTemp, ysAC, ysDC), interval=1000, blit=False)
plt.get_current_fig_manager().full_screen_toggle()
plt.ioff()
plt.show()
plt.draw()

Plotting Max Temps with Polynomial Fitment Line

I am retrieving data from http://data.rcc-acis.org/StnData. That is working well, I can plot the data as I want to visualize it using matplotlib. What I am struggling with is plotting a 4 polynomial line for the values. The code I have is listed below. I am using Python 3.7. Any help is greatly appreciated.
import urllib
import datetime
import urllib.request
import ast
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import scipy.interpolate as ntrpl
import numpy as np
import matplotlib as mpl
# create a static day of the month
#dayofmonth = ('01-01')
dayofmonth = ('11-26')
# testing out a range of years
y = datetime.datetime.today().year
years = list(range(y-1,y-50, -1))
i = 0
dateList=[]
minTList=[]
maxTList=[]
for year in years:
sdate = (str(year) + '-' + dayofmonth)
#print(sdate)
url = "http://data.rcc-acis.org/StnData"
values = {
"sid": "KGGW",
"date": sdate,
"elems": "maxt,mint",
"meta": "name",
"output": "json"
}
data = urllib.parse.urlencode(values).encode("utf-8")
req = urllib.request.Request(url, data)
response = urllib.request.urlopen(req)
results = response.read()
results = results.decode()
results = ast.literal_eval(results)
if i < 1:
n_label = results['meta']['name']
i = 2
for x in results["data"]:
date,maxT,minT = x
#setting the string of date to datetime
date = date[0:4]
date_obj = datetime.datetime.strptime(date,'%Y')
dateList.append(date_obj)
minTList.append(minT)
maxTList.append(maxT)
minT_int = list(map(int,minTList))
maxT_int = list(map(int,maxTList))
n_label = n_label + " " + dayofmonth
#p1 = np.polyfit(newDates,h_temps,3)
def plot_graph(dates,h_temps,l_temps,label):
#converting the dates into a readable format for pyplot
newDates = mdates.date2num(dates)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y'))
plt.title(label)
plt.gcf().autofmt_xdate()
plt.plot(newDates,l_temps)
plt.plot(newDates,h_temps)
#p1 = np.polyfit(x,y,3)
p1 = np.polyfit(newDates,h_temps,3)
print(p1)
plt.plot(newDates,p1)
plt.show()
#print
plot_graph(dateList,maxT_int,minT_int,n_label)

Plotting Date vs time from files

I have folder names which are date time formated as
2018-08-21 to 2018-10-16
Inside each folder there is a zip files which contains time values which is a linear scale, the time goes up linearly.
I'm trying to plot for each day, which has a lot of .bz2 file time series data, the time value at that date.
Right now I'm trying to do that:
timearr = np.asarray(data1['time'])
ax.plot(np.asarray(timeStamps), timearr)
ax.set_title('title')
ax.set_ylabel('date vs time ')
ax.grid(True)
# Format the x-axis for dates (label formatting, rotation)
fig.autofmt_xdate(rotation=45)
fig.tight_layout()
plt.show()
but I get an error message, that both dimensions doesn't match.
dateStamps are list[2018-08-21
2018-08-22
2018-08-23
2018-08-24
2018-08-25]
data1['time'] = list of EPOC values.

Unfortunately, I don't know the detailed file structure, so I have to guess a little were the problem actually is
Here is some code to generate some folders with generic bz2 files:
import bz2
import numpy as np
import datetime
import os
startDate = datetime.datetime(2000,5,2,10,15,0,0)
for day in range(5):
theDate = startDate + datetime.timedelta(days=day)
folder = "{}".format( theDate.replace( microsecond = 0 ).strftime("%Y-%m-%d") )
os.mkdir( folder )
data = ""
for k in range(100):
zzz = theDate + datetime.timedelta(seconds=137*k)
data += "{} ".format( zzz.replace( microsecond = 0 ).strftime("%H:%M:%S") )
d = zzz.day
m = zzz.minute
data += " {}\n".format( .17 * d + .003 * m**2 -.001 * m )
myZip = bz2.BZ2File(os.path.join( folder, 'dat.bz2' ), 'w' )
myZip.write( data )
myZip.close()
Those folders and files a treat with:
import bz2
import numpy as np
import datetime
import os
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
"""
SE posts I used
https://stackoverflow.com/questions/1574088/plotting-time-in-python-with-matplotlib
https://stackoverflow.com/questions/11264521/date-ticks-and-rotation-in-matplotlib
"""
def split_data( inData ):
rows=data.strip().split('\n')
rowcol = [x.split() for x in rows ]
x,y = zip(*rowcol)
y = [float(z) for z in y ]
x = [ datetime.datetime.strptime(z, '%H:%M:%S') for z in x]
return x,y
dataDict = dict()
for root, dirs, files in os.walk("."):
for name in files:
if name.split('.')[-1]=='bz2':
base = os.path.basename( root )
myPath = (os.path.join(root, name))
bz = bz2.BZ2File( myPath, 'r' )
data = bz.read()
dataDict[ base ] = split_data( data )
myFmt = mdates.DateFormatter('%H:%M')
fig = plt.figure()
ax = fig.add_subplot( 1, 1, 1 )
for key, dt in dataDict.iteritems():
ax.plot( *dt , label=key )
ax.xaxis.set_major_formatter(myFmt)
for label in ax.get_xmajorticklabels():
label.set_rotation(30)
ax.set_ylabel('data (arb. u.)')
ax.set_xlabel('time')
ax.legend( loc=0 )
plt.tight_layout()
plt.show()
Providing:
Hope I got it right.

Asking User For Datetime Before Plotting Selected Points

I'm using Python 2.7.3, How can I ask the user to enter "from datetime" & "to datetime" before plotting the selected criteria onto a graph
For eg. when user
from datetime: 21/7/2014 0:00
to datetime: 22/7/2014 23:57
from datetime import datetime
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
x = []
y = []
t = []
fig = plt.figure()
rect = fig.patch
rect.set_facecolor('#31312e')
readFile = open('data.txt', 'r')
sepFile = readFile.read().split('\n')
readFile.close()
for idx, plotPair in enumerate(sepFile):
if plotPair in '. ':
# skip. or space
continue
if idx > 1: # to skip the first line
xAndY = plotPair.split(',')
time_string = xAndY[0]
time_string1 = datetime.strptime(time_string, '%d/%m/%Y %H:%M')
t.append(time_string1)
y.append(float(xAndY[1]))
ax1 = fig.add_subplot(1, 1, 1, axisbg='white')
ax1.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y %H:%M'))
ax1.plot(t, y, 'c', linewidth=3.3)
plt.title('IRRADIANCE')
plt.xlabel('TIME')
fig.autofmt_xdate(rotation=45)
fig.tight_layout()
plt.show()

Try this.
#!/usr/bin/env python
#-*- coding:utf-8 -*-
from datetime import datetime
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
x = []
y = []
t = []
fig = plt.figure()
rect = fig.patch
rect.set_facecolor('#31312e')
readFile = open('C:\\Temp\\001.txt', 'r')
sepFile = readFile.read().split('\n')
readFile.close()
startTime = raw_input('please enter start time in format like 21/7/2014 0:00 :')
endTime = raw_input('please enter end time in format like 22/7/2014 23:57 :')
# startTime = '21/7/2014 0:02'
# endTime = '22/7/2014 23:58'
startTime = datetime.strptime(startTime, '%d/%m/%Y %H:%M')
endTime = datetime.strptime(endTime, '%d/%m/%Y %H:%M')
for idx, plotPair in enumerate(sepFile):
if plotPair in '. ':
# skip. or space
continue
if idx > 1: # to skip the first line
xAndY = plotPair.split(',')
time_string = xAndY[0]
time_string1 = datetime.strptime(time_string, '%d/%m/%Y %H:%M')
if startTime<=time_string1 <=endTime:
t.append(time_string1)
y.append(float(xAndY[1]))
ax1 = fig.add_subplot(1, 1, 1, axisbg='white')
ax1.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y %H:%M'))
ax1.plot(t, y, 'c', linewidth=3.3)
plt.title('IRRADIANCE')
plt.xlabel('TIME')
fig.autofmt_xdate(rotation=45)
fig.tight_layout()
plt.show()

Very slow plot with Matlpotlib

Can anybody help how to optimize the plot function in python? I use Matplotlib to plot financial data.Here small function for plotting OHLC data. The time increase significantly if I add indicators or other data.
import numpy as np
import datetime
from matplotlib.collections import LineCollection
from pylab import *
import urllib2
def test_plot(OHLCV):
bar_width = 1.3
date_offset = 0.5
fig = figure(figsize=(50, 20), facecolor='w')
ax = fig.add_subplot(1, 1, 1)
labels = ax.get_xmajorticklabels()
setp(labels, rotation=0)
month = MonthLocator()
day = DayLocator()
timeFmt = DateFormatter('%Y-%m-%d')
colormap = OHLCV[:,1] < OHLCV[:,4]
color = np.zeros(colormap.__len__(), dtype = np.dtype('|S5'))
color[:] = 'red'
color[np.where(colormap)] = 'green'
dates = date2num( OHLCV[:,0])
lines_hl = LineCollection( zip(zip(dates, OHLCV[:,2]), zip(dates, OHLCV[:,3])))
lines_hl.set_color(color)
lines_hl.set_linewidth(bar_width)
lines_op = LineCollection( zip(zip((np.array(dates) - date_offset).tolist(), OHLCV[:,1]), zip((np.array(dates)).tolist(), parsed_table[:,1])))
lines_op.set_color(color)
lines_op.set_linewidth(bar_width)
lines_cl = LineCollection( zip(zip((np.array(dates) + date_offset).tolist(), OHLCV[:,4]), zip((np.array(dates)).tolist(), parsed_table[:,4])))
lines_cl.set_color(color)
lines_cl.set_linewidth(bar_width)
ax.add_collection(lines_hl, autolim=True)
ax.add_collection(lines_cl, autolim=True)
ax.add_collection(lines_op, autolim=True)
ax.xaxis.set_major_locator(month)
ax.xaxis.set_major_formatter(timeFmt)
ax.xaxis.set_minor_locator(day)
ax.autoscale_view()
ax.xaxis.grid(True, 'major')
ax.grid(True)
ax.set_title('EOD test plot')
ax.set_xlabel('Date')
ax.set_ylabel('Price , $')
fig.savefig('test.png', dpi = 50, bbox_inches='tight')
close()
if __name__=='__main__':
data_table = urllib2.urlopen(r"http://ichart.finance.yahoo.com/table.csv?s=IBM&a=00&b=1&c=2012&d=00&e=15&f=2013&g=d&ignore=.csv").readlines()[1:][::-1]
parsed_table = []
#Format: Date, Open, High, Low, Close, Volume
dtype = (lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(),float, float, float, float, int)
for row in data_table:
field = row.strip().split(',')[:-1]
data_tmp = [i(j) for i,j in zip(dtype, field)]
parsed_table.append(data_tmp)
parsed_table = np.array(parsed_table)
import time
bf = time.time()
count = 100
for i in xrange(count):
test_plot(parsed_table)
print('Plot time: %s' %(time.time() - bf) / count)
The result is something like this. Average time execution on each plot is aproximately 2.6s. Charting in R is much faster, but I didn't measure the performance and I don't want use Rpy, so I bielive that my code is inefficient.

This solution reuses a Figure instance and saves plots asynchronously. You could change this to have as many figures as there are processors, do that many plots asynchronously, and it should speed things up even more. As it is, this takes ~1s per plot, down from 2.6 on my machine.
import numpy as np
import datetime
import urllib2
import time
import multiprocessing as mp
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from pylab import *
from matplotlib.collections import LineCollection
class AsyncPlotter():
def __init__(self, processes=mp.cpu_count()):
self.manager = mp.Manager()
self.nc = self.manager.Value('i', 0)
self.pids = []
self.processes = processes
def async_plotter(self, nc, fig, filename, processes):
while nc.value >= processes:
time.sleep(0.1)
nc.value += 1
print "Plotting " + filename
fig.savefig(filename)
plt.close(fig)
nc.value -= 1
def save(self, fig, filename):
p = mp.Process(target=self.async_plotter,
args=(self.nc, fig, filename, self.processes))
p.start()
self.pids.append(p)
def join(self):
for p in self.pids:
p.join()
class FinanceChart():
def __init__(self, async_plotter):
self.async_plotter = async_plotter
self.bar_width = 1.3
self.date_offset = 0.5
self.fig = plt.figure(figsize=(50, 20), facecolor='w')
self.ax = self.fig.add_subplot(1, 1, 1)
self.labels = self.ax.get_xmajorticklabels()
setp(self.labels, rotation=0)
line_hl = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
line_op = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
line_cl = LineCollection(([[(734881,1), (734882,5), (734883,9), (734889,5)]]))
self.lines_hl = self.ax.add_collection(line_hl, autolim=True)
self.lines_op = self.ax.add_collection(line_cl, autolim=True)
self.lines_cl = self.ax.add_collection(line_op, autolim=True)
self.ax.set_title('EOD test plot')
self.ax.set_xlabel('Date')
self.ax.set_ylabel('Price , $')
month = MonthLocator()
day = DayLocator()
timeFmt = DateFormatter('%Y-%m-%d')
self.ax.xaxis.set_major_locator(month)
self.ax.xaxis.set_major_formatter(timeFmt)
self.ax.xaxis.set_minor_locator(day)
def test_plot(self, OHLCV, i):
colormap = OHLCV[:,1] < OHLCV[:,4]
color = np.zeros(colormap.__len__(), dtype = np.dtype('|S5'))
color[:] = 'red'
color[np.where(colormap)] = 'green'
dates = date2num( OHLCV[:,0])
date_array = np.array(dates)
xmin = min(dates)
xmax = max(dates)
ymin = min(OHLCV[:,1])
ymax = max(OHLCV[:,1])
self.lines_hl.set_segments( zip(zip(dates, OHLCV[:,2]), zip(dates, OHLCV[:,3])))
self.lines_hl.set_color(color)
self.lines_hl.set_linewidth(self.bar_width)
self.lines_op.set_segments( zip(zip((date_array - self.date_offset).tolist(), OHLCV[:,1]), zip(date_array.tolist(), OHLCV[:,1])))
self.lines_op.set_color(color)
self.lines_op.set_linewidth(self.bar_width)
self.lines_cl.set_segments( zip(zip((date_array + self.date_offset).tolist(), OHLCV[:,4]), zip(date_array.tolist(), OHLCV[:,4])))
self.lines_cl.set_color(color)
self.lines_cl.set_linewidth(self.bar_width)
self.ax.set_xlim(xmin,xmax)
self.ax.set_ylim(ymin,ymax)
self.ax.xaxis.grid(True, 'major')
self.ax.grid(True)
self.async_plotter.save(self.fig, '%04i.png'%i)
if __name__=='__main__':
print "Starting"
data_table = urllib2.urlopen(r"http://ichart.finance.yahoo.com/table.csv?s=IBM&a=00&b=1&c=2012&d=00&e=15&f=2013&g=d&ignore=.csv").readlines()[1:][::-1]
parsed_table = []
#Format: Date, Open, High, Low, Close, Volume
dtype = (lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(),float, float, float, float, int)
for row in data_table:
field = row.strip().split(',')[:-1]
data_tmp = [i(j) for i,j in zip(dtype, field)]
parsed_table.append(data_tmp)
parsed_table = np.array(parsed_table)
import time
bf = time.time()
count = 10
a = AsyncPlotter()
_chart = FinanceChart(a)
print "Done with startup tasks"
for i in xrange(count):
_chart.test_plot(parsed_table, i)
a.join()
print('Plot time: %.2f' %(float(time.time() - bf) / float(count)))