Develop Reference

Animating heatmap with python seaborn

This is driving me crazy. I'm trying to create an animation of a heatmap where every frame shows a different time stamp. I have created the data structures, imported ffmpeg, and I tried another package too (celluloid that seems very good). I tried an example with a simple line plot and it does animate. The result is always the same: I get the last frame, not an animation. Here is my code in case anyone can spot the error... Thank you in advance.
------------- Code --------------
from matplotlib import animation, rc
from IPython.display import HTML
data2 = df[['ProducerName', 'MemOccupied']]
data2 = data2.sort_index().sort_values('ProducerName', kind='mergesort')
data2 = data2[['MemOccupied']]
fig = plt.figure()
camera = Camera(fig)
df_heatmap = pd.DataFrame(data2)
df_heatmap = df_heatmap.dropna()
df_heatmap = df_heatmap.resample('1T').mean()
count = df_heatmap.count()
index_series = df_heatmap.index
def init():
#plt.clf()
ax = sns.heatmap(data2[data2.index == index_series[1]], cbar_kws={'label': 'Memory Occupied (GB)'}, cmap='Blues_r')
ax.set(ylabel=None)
#return (ax,)
def animate(i):
plt.clf()
ax = sns.heatmap(data2[data2.index == index_series[i]], cbar_kws={'label': 'Memory Occupied (GB)'}, cmap='Blues_r')
ax.set(ylabel=None)
#return (ax,)
anim = animation.FuncAnimation(fig = fig, func = animate, init_func=init, frames=count-1,repeat = True, interval = 200)
HTML(anim.to_html5_video())

Live scatter plot fetching data from firebase every 2 second

I have a mobile application which does some calculation and throws x,y coordinates and are updated on firebase every 2 seconds.
Next i want those coordinates to be plotted on a floor plan live. For that i am using Scatter plot over the floor plan image. But i cannot make it live as soon as the data is fetched need help with that.
Here is the code till now:
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.animation as animation
xs = []
ys = []
fig = plt.figure()
scat = plt.scatter(xs, ys, c='r', s=100)
def main():
graph_data = open("testfile.txt","r").read()
lines = graph_data.split("\n")
for line in lines:
if len(line)>1:
x,y = line.split(",")
xs.append(x)
ys.append(y)
plt.scatter(xs,ys)
print(xs)
print(ys)
ani = animation.FuncAnimation(fig,main(),fargs=(scat))
plt.show()
Getting error with animation.FuncAnimation TypeError: NoneType object argument after * must be an iterable, not PathCollection

You can fetch data in a separate thread while updating the plot in the main one. Here is a complete working example:
#!/usr/bin/env python3
import time
from queue import Queue
from threading import Thread, Event
import numpy as np
import matplotlib.pyplot as plt
FETCH_DELAY = 2
def fetch_data(queue, stop):
while not stop.is_set():
x, y = np.random.randn(2)
queue.put((x, y))
time.sleep(FETCH_DELAY)
def limits(array, offset=1):
return array.min() - offset, array.max() + offset
def main():
stop = Event()
queue = Queue()
worker = Thread(target=fetch_data, args=(queue, stop))
worker.start()
plt.ion()
fig, ax = plt.subplots()
plot = ax.scatter([], [])
try:
while True:
data = queue.get()
data = np.array(data)
plt_data = plot.get_offsets()
plt_data = np.vstack((plt_data, data))
plot.set_offsets(plt_data)
fig.canvas.draw()
xmin, xmax = limits(plt_data[:, 0])
ymin, ymax = limits(plt_data[:, 1])
ax.set_xlim(xmin, xmax)
ax.set_ylim(ymin, ymax)
queue.task_done()
except KeyboardInterrupt:
pass
finally:
stop.set()
worker.join()
if __name__ == '__main__':
main()
Save it as plot_update.py file and run it from the command line:
python3 plot_update.py

Here is the solution without using threads it becomes very simple:
import matplotlib.pyplot as plt
import matplotlib.animation
import numpy as np
from firebase import firebase
firebase = firebase.FirebaseApplication('Firebase url', None)
fig, ax = plt.subplots()
x, y = [],[]
sc = ax.scatter(x,y,c=np.random.rand(3,))
plt.xlim(12,13)
plt.ylim(77,78)
def animate(i):
xs = firebase.get('/Lat',None)
ys = firebase.get('/Long',None)
xs = round(xs,2)
ys = round(ys,2)
file = open("testfile.txt","a+")
file.write("{0},{1} \n".format(xs,ys))
file.close()
graph_data = open("testfile.txt","r").read()
lines = graph_data.split("\n")
for line in lines:
if len(line)>1:
xs,ys = line.split(",")
x.append(xs)
y.append(ys)
sc.set_offsets(np.c_[x,y])
ani = matplotlib.animation.FuncAnimation(fig, animate,
frames=2, interval=500, repeat=True)
plt.show()

Animated Line Graph with Matplotlib/Python [duplicate]

This question already has answers here:
matplotlib animated line plot stays empty
(3 answers)
Closed 5 years ago.
I'm having trouble getting my line graph to animate.
Background: I'm working on a program that will handle simulating network latency and I'm trying to graph the latency so I can see how well my program is keeping up with the load of commands coming out of the controller.
I've setup my figure:
# First set up the figure, the axis, and the plot element we want to animate
fig = plt.figure()
ax = plt.axes(xlim=(0,2), ylim = (-2,2))
line, = ax.plot([], [], lw=2)
setup an init() and animate() function
# initialization function: plot the background of each frame
def init():
line.set_data([], [])
return line,
def animate(i):
line.set_data(x[i], y[i])
return line,
then in my DelayedTask.process() function (where I measure the time between the intended execution and actual execution) I append the values and index to my x,y lists.
delta = self.time - datetime.now()
lock = threading.Lock()
lock.acquire()
x.append(len(x))
y.append(delta.total_seconds())
lock.release()
finally at the bottom of my program, I create the animation function.
anim = animation.FuncAnimation(fig, animate, init_func=init, frames=200, interval=20, blit=True)
Unfortunately, the graph shows up, but the numbers won't plot. I've put a breakpoint in the animate() function and in the deltas are filling in the list, but it won't show any lines on the graph.
Here is the full code:
import multiprocessing, requests, threading
import decimal
import random
import time
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import animation, rc
from queue import Queue
from multiprocessing.dummy import Pool as ThreadPool
from threading import Thread
from datetime import datetime, timedelta
class WorkQueue:
def __init__(self, threads=6):
self.threads = threads
def process(self, work):
pool = ThreadPool(self.threads)
results = pool.map(DelayedTask.process, work)
pool.close()
pool.join()
class DelayedTask:
def __init__(self, func, delay, message):
print("DelayTask.__init__: {0}".format((func.__name__, delay, message)))
self.func = func
self.time = datetime.now() + timedelta(milliseconds=delay)
self.message = message
def process(self):
delta = self.time - datetime.now()
lock = threading.Lock()
lock.acquire()
x.append(len(x))
y.append(delta.total_seconds())
lock.release()
if delta.total_seconds() > 0.01:
print('DelayTask.Process: Sleeping {0} milliseconds\n'.format(round(delta.total_seconds() * 1000)))
time.sleep(delta.total_seconds())
self.func(self.message)
elif delta.total_seconds() < 0.01 and delta.total_seconds() > 0:
print('DelayTask.Process: Processing with {0} milliseconds remaining\n'.format(round(delta.total_seconds() * 1000)))
self.func(self.message)
else:
print("DelayTask.Process: Processing task: {0} milliseconds late\n".format(round(delta.total_seconds() * -1000)))
self.func(self.message)
return True
def __str__(self):
return str((self.func.__name__, self.time, self.message))
def get(url):
print("Requesting {0}".format(url))
r = requests.get(url=url)
print("get(url): Received response for {0} with Status Code {1}".format(url, r.status_code))
aggregatorq = multiprocessing.Queue()
# First set up the figure, the axis, and the plot element we want to animate
fig = plt.figure()
ax = plt.axes(xlim=(0,2), ylim = (-2,2))
line, = ax.plot([], [], lw=2)
x = []
y = []
# initialization function: plot the background of each frame
def init():
line.set_data([], [])
return line,
def animate(i):
line.set_data(x[i], y[i])
return line,
def collector():
bucket = []
while len(bucket) <= 10:
task = aggregatorq.get()
print("collector: aggregating Tasks\n")
bucket.append(DelayedTask(task['func'], task['delay'], task['message']))
if(len(bucket) == 10):
bucket.sort(key=lambda x: x.time, reverse=False)
firsttask = bucket[0]
firsttime = firsttask.time - datetime.now()
if firsttime.total_seconds() >= 0:
print('collector: Sleeping {0} seconds until first task in bucket\n'.format(firsttime.total_seconds()))
time.sleep(firsttime.total_seconds())
queue = WorkQueue(10)
queue.process(bucket)
bucket.clear()
def controller():
print("Starting Controller\n")
finishtime = datetime.now() + timedelta(seconds=5)
print("controller: Will finish at {0}\n".format(finishtime))
sites = ["att", "google", "hulu", "msn", "yahoo", "gmail"]
while True:
if datetime.now() > finishtime:
print("Controller Finished")
return;
else:
pass
print("{0} remaining in controller..".format(finishtime - datetime.now()))
requestdelay = random.randint(1, 20)
randomsite = random.randint(0, len(sites)-1)
aggregatorq.put({'func': get, 'delay': requestdelay, 'message': 'http://www.{0}.com'.format(sites[randomsite])})
t = threading.Thread(target=controller)
t2 = threading.Thread(target=collector)
# call the animator. blit=True means only re-draw the parts that have changed.
anim = animation.FuncAnimation(fig, animate, init_func=init,
frames=200, interval=20, blit=True)
def main():
t.start()
t2.start()
plt.show()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('Interrupted')
t.join()
t2.join()
try:
sys.exit(0)
except SystemExit:
os._exit(0)

Your problem is in the update function. Using the statement
line.set_data(x[i], y[i])
You assign exactly one data point to your line every time update is called. Therefore you cannot see any line, as lines are only plotted between data points. To fix the problem, leave out the indexing:
line.set_data(x, y)
This way all your collected data will be plotted.

python real time plot of serial port data has huge lag

I am trying to read from serial port and plot the data in graph using matplot.
Following is my code :
I see that because of plot, there is huge lag (data in queue goes up to 10000 bytes) hence i dont see real time plot coming. Can you please help me if i am doing anything wrong.
<
import serial # import Serial Library
import numpy # Import numpy
import matplotlib.pyplot as plt
Accelerometer= []
COM_read = serial.Serial('COM5', 9600, timeout=None,parity='N',stopbits=1,rtscts=0) #Creating our serial object name
plt.ion() #Tell matplotlib you want interactive mode to plot live data
cnt=0
COM_read.flushInput()
COM_read.flushOutput()
def makeFig(): #Create a function that makes our desired plot
plt.title('My Live Streaming Sensor Data') #Plot the title
plt.grid(True) #Turn the grid on
plt.ylabel('Acc in g') #Set ylabels
plt.plot(Accelerometer, 'ro-', label='Accelerometer g') #plot the temperature
plt.legend(loc='upper left') #plot the legend
plt.ylim(15000,30000) #Set limits of second y axis- adjust to readings you are getting
print "came through"
while True: # While loop that loops forever
print COM_read.inWaiting()
while (COM_read.inWaiting()==0): #Wait here until there is data
pass #do nothing
s = COM_read.readline() #read the line of text from the serial port
decx = int(s[0:4],16)
decy = int(s[5:9],16)
decz = int(s[10:14],16)
if decx > 32768:
decx = decx - 65536;
if decy > 32768:
decy = decy - 65536;
if decz > 32768:
decz = decz - 65536;
#print decx,decy,decz
res = ((decx*decx) + (decy*decy) + (decz*decz))**(1.0/2.0)
Accelerometer.append(res) #Build our Accelerometer array by appending temp readings
drawnow(makeFig) #Call drawnow to update our live graph
plt.pause(.000001) #Pause Briefly. Important to keep drawnow from crashing
cnt=cnt+1
if(cnt>50): #If you have 50 or more points, delete the first one from the array
Accelerometer.pop(0) #This allows us to just see the last 50 data points
>
----------------Based on Dr.John's suggestion, the code is written as follows -----
import serial
import matplotlib
import threading, time
import pylab
import matplotlib.pyplot as plt
import matplotlib.animation as anim
class MAIN:
#ser =0;
def __init__(self):
self.res = 0.0
self.ser = serial.Serial('COM5', 9600, timeout=None,parity='N',stopbits=1,rtscts=0)
self.ser.flushInput()
elf.ser.flushOutput()
c = self.ser.read(1);
while(c != '\n'):
=self.ser.read(1)
return
def acquire_data(self):
s = self.ser.readline()
decx = int(s[0:4],16)
decy = int(s[5:9],16)
decz = int(s[10:14],16)
if decx > 32768:
decx = decx - 65536;
if decy > 32768:
decy = decy - 65536;
if decz > 32768:
decz = decz - 65536;
self.res = ((decx*decx) + (decy*decy) + (decz*decz))**(1.0/2.0)
print self.res
return
ex = MAIN()
threading.Timer(2,ex.acquire_data).start() #starts function acquire_data every 2 sec and resets self.res with 0.5Hz
fig, ax = plt.subplots()
ax.plot(time.time(), self.res, 'o-')
print "closed"

A while true loop is a bad idea in most cases.
Non-OO-Programming is a bad idea in most cases.
Appending plot-data to lists is a bad idea in most cases (laggy).
Start the data aquisition at defined times with
import threading, time
threading.Timer(2, acquire_data).start() #starts function acquire_data every 2 sec and resets self.res with 0.5Hz
then instead of the while-loop, please define class based functions for your own benefit:
class MAIN:
def __init__(self):
self.res = 0
def acquire_data(self):
....
return self.res
and plot with
fig, ax = plt.subplots()
ax.plot(time.time(), self.res, 'o-')
Greets Dr. Cobra

python matplotlib: plotting in another process

EDIT: The ultimate requirement for such a Python program is: Receive data from UART from a external circuitry (which probably is equipped with some sensors), the program will process these data, and draw a dynamically updated curve on the computer screen.
So, I want to plot dynamically, the following test script starts a sub-process, and in that process, it accepts data from parent process through a Queue, and plot data accordingly.
But when the script is run, only an empty figure is shown, I can see the console prints "put:" and "got:" messages, meaning both parent and subprocess are running and communicating, but nothing happens in the GUI figure window.
Furthermore, the GUI window is not responsive and if I click on the window, it will crash.
The system is Windows 10, 64 bit. Python version is 2.7 (32bit)
What's the problem here? thank you!
import matplotlib.pyplot as plt
import multiprocessing as mp
import random
import numpy
import time
def worker(q):
plt.ion()
ln, = plt.plot([], [])
plt.show()
while True:
obj = q.get()
n = obj + 0
print "sub : got:", n
ln.set_xdata(numpy.append(ln.get_xdata(), n))
ln.set_ydata(numpy.append(ln.get_ydata(), n))
plt.draw()
if __name__ == '__main__':
queue = mp.Queue()
p = mp.Process(target=worker, args=(queue,))
p.start()
while True:
n = random.random() * 5
print "main: put:", n
queue.put(n)
time.sleep(1.0)

You have to rescale, otherwise nothing will appear:
This works on my computer :
import matplotlib.pyplot as plt
import multiprocessing as mp
import random
import numpy
import time
def worker(q):
#plt.ion()
fig=plt.figure()
ax=fig.add_subplot(111)
ln, = ax.plot([], [])
fig.canvas.draw() # draw and show it
plt.show(block=False)
while True:
obj = q.get()
n = obj + 0
print "sub : got:", n
ln.set_xdata(numpy.append(ln.get_xdata(), n))
ln.set_ydata(numpy.append(ln.get_ydata(), n))
ax.relim()
ax.autoscale_view(True,True,True)
fig.canvas.draw()
if __name__ == '__main__':
queue = mp.Queue()
p = mp.Process(target=worker, args=(queue,))
p.start()
while True:
n = random.random() * 5
print "main: put:", n
queue.put(n)
time.sleep(1.0)

Till now I'd like to flag my following sample program as the answer to my question. It definitely is not the perfect one, or maybe it's even not the correct way to do that in Python and matplotlib.
I think the important thing to not cause unresponsiveness on the figure is not to hang the "UI" thread, which when the UI is shown, matplotlib probably is running a event loop on it, so if I put any time.sleep(0.1) or call Queue.get() which block the thread execution, the figure window will just hang.
So instead of blocking the thread at "Queue.get()", I choose to use "Queue.get_nowait()" as a polling method for incoming new data. The UI thread (ie, matplotlib figure window updating worker) will only block at matplotlib.pyplot.pause(), which will not suspend the event loop I believe.
If there is another call in matplotlib that can block and wait for a signal, I think that would be better than this polling approach.
At first I see multiprocessing examples with matplotlib, so I was trying to use multiple processes for concurrency. But it seems that you just need to take care of the synchronization yourself, it is okay to use multithreading instead. And multithreading has the benefit of sharing data within one process. So the following program utilized threading module instead of multiprocessing.
The following is my test program, I can run it on Windows 7 (64 bit) with Python 2.7, and the Figure Window is responsive at this rate of updating, you can drag it, resize it and so on.
#!/usr/bin/python
# vim: set fileencoding=utf-8:
import random
import time
import Queue
import threading
import numpy as np
import matplotlib.pyplot as plt
## Measurement data that are shared among threads
val1 = []
val2 = []
lock = threading.Lock()
def update_data_sync(x, y):
lock.acquire()
val1.append(x)
val2.append(y)
if len(val1) > 50:
del val1[0]
if len(val2) > 50:
del val2[0]
lock.release()
def get_data_sync():
lock.acquire()
v1 = list(val1)
v2 = list(val2)
lock.release()
return (v1, v2)
def worker(queue):
plt.ion()
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.margins(0.05, 0.05)
#ax.set_autoscale_on(True)
ax.autoscale(enable=True, axis='both')
#ax.autoscale(enable=True, axis='y')
ax.set_ylim(0, 1)
line1, line2 = ax.plot([], [], 'b-', [], [], 'r-')
while True:
need_draw = False
is_bye = False
while True:
## Try to exhaust all pending messages
try:
msg = queue.get_nowait()
if msg is None:
print "thread: FATAL, unexpected"
sys.exit(1)
if msg == 'BYE':
print "thread: got BYE"
is_bye = True
break
# Assume default message is just let me draw
need_draw = True
except Queue.Empty as e:
# Not 'GO' or 'BYE'
break
## Flow control
if is_bye:
break
if not need_draw:
plt.pause(0.33)
continue;
## Draw it
(v1, v2) = get_data_sync()
line1.set_xdata(range(1, len(v1) + 1, 1))
# Make a clone of the list to avoid competition on the same dataset
line1.set_ydata(v1)
line2.set_xdata(line1.get_xdata())
line2.set_ydata(v2)
## Adjust view
#ax.set_xlim(0, len(line1.get_ydata()) + 1)
#ax.set_ylim(0, 1)
## (??) `autoscale' does not work here...
#ax.autoscale(enable=True, axis='x')
#ax.autoscale(enable=True, axis='y')
ax.relim()
ax.autoscale_view(tight=True, scalex=True, scaley=False)
## "Redraw"
## (??) Maybe pyplot.pause() can ensure visible redraw
fig.canvas.draw()
print "thread: DRAW"
plt.pause(0.05)
## Holy lengthy outermost `while' loop ends here
print "thread: wait on GUI"
plt.show(block=True)
plt.close('all')
print "thread: worker exit"
return
def acquire_data():
# Fake data for testing
if not hasattr(acquire_data, 'x0'):
acquire_data.x0 = 0.5
x = int(random.random() * 100) / 100.0
while np.abs(x - acquire_data.x0) > 0.5:
x = int(random.random() * 100) / 100.0
acquire_data.x0 = x
y = 0.75 * np.abs(np.cos(i * np.pi / 10)) + 0.15
return (x, y)
if __name__ == "__main__":
queue = Queue.Queue()
thr = threading.Thread(target=worker, args=(queue, ))
thr.start()
for i in range(200):
(x, y) = acquire_data()
update_data_sync(x, y)
#print "main: val1: {}. val2: {}".format(x, y)
queue.put("GO")
time.sleep(0.1)
queue.put('BYE')
print "main: waiting for children"
thr.join()
print "main: farewell"

Made samll modifications to above code. I used Process, Queue, Value, Array in multiprocessing package to reduce the code complexity. Script will plot [0,0], [1,1], [2,2] etc on a graph
from multiprocessing import Process, Queue, Value, Array
import time
import matplotlib.pyplot as plt
from queue import Empty
def f(q, num, arr1, arr2):
plt.ion()
fig = plt.figure()
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.margins(0.05, 0.05)
#ax.set_autoscale_on(True)
ax.autoscale(enable=True, axis='both')
#ax.autoscale(enable=True, axis='y')
ax.set_ylim(0, 100)
line1, line2 = ax.plot([], [], 'b-', [], [], 'r-')
while True :
try:
#val = q.get_nowait()
val = q.get(timeout = 0.8) # reducing the timeout value will improve the response time on graph whie mouse is moved on it
#val = q.get()
if val == 'Exit':
break
if val == 'Go':
x = num.value
#print("num.value = ", x)
v1 = arr1[0:num.value]
v2 = arr2[0:num.value]
#print("v1 :", v1)
#print("v2 :", v2)
line1.set_xdata(range(1, len(v1) + 1, 1))
line1.set_ydata(v1)
line2.set_xdata(line1.get_xdata())
line2.set_ydata(v2)
ax.relim()
ax.autoscale_view(tight=True, scalex=True, scaley=False)
fig.canvas.draw()
#print ("thread: DRAW")
plt.pause(0.05)
except Empty as e:
x = num.value
line1.set_xdata(range(1, len(v1) + 1, 1))
line1.set_ydata(v1)
line2.set_xdata(line1.get_xdata())
line2.set_ydata(v2)
ax.relim()
ax.autoscale_view(tight=True, scalex=True, scaley=False)
fig.canvas.draw()
plt.pause(0.05)
continue
if __name__ == '__main__':
q = Queue()
num = Value('i', 0)
arr1 = Array('d', range(100))
arr2 = Array('d', range(100))
p = Process(target=f, args=(q,num, arr1, arr2, ))
p.start()
for i in range(10):
arr1[i] = i
arr2[i] = i
num.value = i+1
q.put("Go") # prints "[42, None, 'hello']"
time.sleep(1)
q.put("Exit")
p.join()