I'm trying o thread the following code and send data to it (at random intervals) but I can't figure out how. I'm saving all the data to a txt file and reading the info from there, it isn't working very well. Is it possible to create a function that sends data to a specific thread( like : SendDataToThread(data, ThreadNumber) )? and how would I go about reading the data sent? I've seen a few solutions using queue but I was unable to understand them. here is the script I am temporarily using to plot the graph which I found here. sorry if the question seems simple but I've never before had to messed with threading or matplotlib.
import matplotlib.pyplot as plt
from threading import Thread
plt.ion()
class DynamicUpdate():
#Suppose we know the x range
min_x = 0
max_x = 10
def on_launch(self):
#Set up plot
self.figure, self.ax = plt.subplots()
self.lines, = self.ax.plot([],[], 'o')
#Autoscale on unknown axis and known lims on the other
self.ax.set_autoscaley_on(True)
self.ax.set_xlim(self.min_x, self.max_x)
#Other stuff
self.ax.grid()
...
def on_running(self, xdata, ydata):
#Update data (with the new _and_ the old points)
self.lines.set_xdata(xdata)
self.lines.set_ydata(ydata)
#Need both of these in order to rescale
self.ax.relim()
self.ax.autoscale_view()
#We need to draw *and* flush
self.figure.canvas.draw()
self.figure.canvas.flush_events()
#Example
def __call__(self):
# read/plot data
Here's some example code which shows how to do several of the things that were asked about. This uses multithreading rather than multiprocessing, and shows some examples of using queues, starting/stopping worker threads and updating a matplotlib plot with additional data.
(Part of the code comes from answers to other questions including this one and this one.)
The code shows a possible implementation of an asynchronous worker, to which data can be sent for subsequent processing. The worker uses an internal queue to buffer the data, and an internal thread (loop) that reads data from the queue, does some processing and sends the result for display.
An asynchronous plotter implementation is also shown. Results can be sent to this plotter from multiple workers. (This also uses an internal queue for buffering; this is done to allow the main program thread itself to call the function that updates the plot, which appears to be a requirement with matplotlib.)
NB This was written for Python 2.7 on OSX. Hope some of it may be useful.
import time
import threading
import Queue
import math
import matplotlib.pyplot as plt
class AsynchronousPlotter:
"""
Updates a matplotlib data plot asynchronously.
Uses an internal queue to buffer results passed for plotting in x, y pairs.
NB the output_queued_results() function is intended be called periodically
from the main program thread, to update the plot with any waiting results.
"""
def output_queued_results(self):
"""
Plots any waiting results. Should be called from main program thread.
Items for display are x, y pairs
"""
while not self.queue.empty():
item = self.queue.get()
x, y = item
self.add_point(x, y)
self.queue.task_done()
def queue_result_for_output(self, x, y):
"""
Queues an x, y pair for display. Called from worker threads, so intended
to be thread safe.
"""
self.lock.acquire(True)
self.queue.put([x, y])
self.lock.release()
def redraw(self):
self.ax.relim()
self.ax.autoscale_view()
self.fig.canvas.draw()
plt.pause(0.001)
def add_point(self, x, y):
self.xdata.append(x)
self.ydata.append(y)
self.lines.set_xdata(self.xdata)
self.lines.set_ydata(self.ydata)
self.redraw()
def __init__(self):
self.xdata=[]
self.ydata=[]
self.fig = plt.figure()
self.ax = self.fig.add_subplot(111)
self.lines, = self.ax.plot(self.xdata, self.ydata, 'o')
self.ax.set_autoscalex_on(True)
self.ax.set_autoscaley_on(True)
plt.ion()
plt.show()
self.lock = threading.Lock()
self.queue = Queue.Queue()
class AsynchronousWorker:
"""
Processes data asynchronously.
Uses an internal queue and internal thread to handle data passed in.
Does some processing on the data in the internal thread, and then
sends result to an asynchronous plotter for display
"""
def queue_data_for_processing(self, raw_data):
"""
Queues data for processing by the internal thread.
"""
self.queue.put(raw_data)
def _worker_loop(self):
"""
The internal thread loop. Runs until the exit signal is set.
Processes the supplied raw data into something ready
for display.
"""
while True:
try:
# check for any data waiting in the queue
raw_data = self.queue.get(True, 1)
# process the raw data, and send for display
# in this trivial example, change circle radius -> area
x, y = raw_data
y = y**2 * math.pi
self.ap.queue_result_for_output(x, y)
self.queue.task_done()
except Queue.Empty:
pass
finally:
if self.esig.is_set():
return
def hang_up(self):
self.esig.set() # set the exit signal...
self.loop.join() # ... and wait for thread to exit
def __init__(self, ident, ap):
self.ident = ident
self.ap = ap
self.esig = threading.Event()
self.queue = Queue.Queue()
self.loop = threading.Thread(target=self._worker_loop)
self.loop.start()
if __name__ == "__main__":
ap = AsynchronousPlotter()
num_workers = 5 # use this many workers
# create some workers. Give each worker some ID and tell it
# where it can find the output plotter
workers = []
for worker_number in range (num_workers):
workers.append(AsynchronousWorker(worker_number, ap))
# supply some data to the workers
for worker_number in range (num_workers):
circle_number = worker_number
circle_radius = worker_number * 4
workers[worker_number].queue_data_for_processing([circle_number, circle_radius])
# wait for workers to finish then tell the plotter to plot the results
# in a longer-running example we would update the plot every few seconds
time.sleep(2)
ap.output_queued_results();
# Wait for user to hit return, and clean up workers
raw_input("Hit Return...")
for worker in workers:
worker.hang_up()
I kinda improved the code I can send a value to it when it is being created so that is good, but with multiprocessing I can't really figure out how to make the plot show. When I call the plot without multiprocessing it works so it might be something simple that I can't see. Also I'm trying to study the code you left a link to but to me, it's not very clear. I'm also trying to save the processes to a list so that later I can try to send the data directly to the process while the process is running(I think it's with pipe that I do this but, I'm not sure)
import matplotlib.pyplot as plt
from multiprocessing import Process
plt.ion()
class DynamicUpdate():
#Suppose we know the x range
min_x = 0
max_x = 10
def __init__(self, x):
self.number = x
def on_launch(self):
#Set up plot
self.figure, self.ax = plt.subplots()
self.lines, = self.ax.plot([],[], 'o')
#Autoscale on unknown axis and known lims on the other
self.ax.set_autoscaley_on(True)
self.ax.set_xlim(self.min_x, self.max_x)
#Other stuff
self.ax.grid()
...
def on_running(self, xdata, ydata):
#Update data (with the new _and_ the old points)
self.lines.set_xdata(xdata)
self.lines.set_ydata(ydata)
#Need both of these in order to rescale
self.ax.relim()
self.ax.autoscale_view()
#We need to draw *and* flush
self.figure.canvas.draw()
self.figure.canvas.flush_events()
#Example
def __call__(self):
print(self.number)
import numpy as np
import time
self.on_launch()
xdata = []
ydata = []
for x in np.arange(0,10,0.5):
xdata.append(x)
ydata.append(np.exp(-x**2)+10*np.exp(-(x-7)**2))
self.on_running(xdata, ydata)
time.sleep(1)
return xdata, ydata
_processes_=[]
for i in range(0,2):
_processes_.append(Process(target=DynamicUpdate(i)))
p = Process(target=_processes_[i])
p.start()
# tried adding p.join(), but it didn't change anything
p.join()
Related
so I've been thinking about this for a couple days now and I cant figure it out, I've searched around but couldn't find the answer I was looking for, so any help would be greatly appreciated.
Essentially what I am trying to do is call a method on a group of objects in my main thread from a separate thread, just once after 2 seconds and then the thread can exit, I'm just using threading as a way of creating a non-blocking 2 second pause (if there are other ways of accomplishing this please let me know.
So I have a pyqtplot graph/plot that updates from a websocket stream and the gui can only be updated from the thread that starts it (the main one).
What happens is I open a websocket stream fill up a buffer for about 2 seconds, make an REST request, apply the updates from the buffer to the data from the REST request and then update the data/plot as new messages come in. Now the issue is I can't figure out how to create a non blocking 2 second pause in the main thread without creating a child thread. If I create a child thread and pass the object that contains the dictionary I want to update after 2 seconds, I get issues regarding updating the plot from a different thread. So what I THINK is happening is when that new spawned thread is spawned the reference to the object I want to update is actually the object itself, or the data (dictionary) containing the update data is now in a different thread as the gui and that causes issues.
open websocket --> start filling buffer --> wait 2 seconds --> REST request --> apply updates from buffer to REST data --> update data as new websocket updates/messages come in.
Unfortunately the websocket and gui only start when you run pg.exec() and you can't break them up to start individually, you create them and then start them together (or at least I have failed to find a way to start them individually, alternatively I also tried using a separate library to handle websockets however this requires starting a thread for incoming messages as well)
This is the minimum reproducible example, sorry it's pretty long but I couldn't really break it down anymore without removing required functionality as well as preserving context:
import json
import importlib
from requests.api import get
import functools
import time
import threading
import numpy as np
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore
QtWebSockets = importlib.import_module(pg.Qt.QT_LIB + '.QtWebSockets')
class coin():
def __init__(self):
self.orderBook = {'bids':{}, 'asks':{}}
self.SnapShotRecieved = False
self.last_uID = 0
self.ordBookBuff = []
self.pltwgt = pg.PlotWidget()
self.pltwgt.show()
self.bidBar = pg.BarGraphItem(x=[0], height=[1], width= 1, brush=(25,25,255,125), pen=(0,0,0,0))
self.askBar = pg.BarGraphItem(x=[1], height=[1], width= 1, brush=(255,25,25,125), pen=(0,0,0,0))
self.pltwgt.addItem(self.bidBar)
self.pltwgt.addItem(self.askBar)
def updateOrderBook(self, message):
for side in ['a','b']:
bookSide = 'bids' if side == 'b' else 'asks'
for update in message[side]:
if float(update[1]) == 0:
try:
del self.orderBook[bookSide][float(update[0])]
except:
pass
else:
self.orderBook[bookSide].update({float(update[0]): float(update[1])})
while len(self.orderBook[bookSide]) > 1000:
del self.orderBook[bookSide][(min(self.orderBook['bids'], key=self.orderBook['bids'].get)) if side == 'b' else (max(self.orderBook['asks'], key=self.orderBook['asks'].get))]
if self.SnapShotRecieved == True:
self.bidBar.setOpts(x0=self.orderBook['bids'].keys(), height=self.orderBook['bids'].values(), width=1 )
self.askBar.setOpts(x0=self.orderBook['asks'].keys(), height=self.orderBook['asks'].values(), width=1 )
def getOrderBookSnapshot(self):
orderBookEncoded = get('https://api.binance.com/api/v3/depth?symbol=BTCUSDT&limit=1000')
if orderBookEncoded.ok:
rawOrderBook = orderBookEncoded.json()
orderBook = {'bids':{}, 'asks':{}}
for orders in rawOrderBook['bids']:
orderBook['bids'].update({float(orders[0]): float(orders[1])})
for orders in rawOrderBook['asks']:
orderBook['asks'].update({float(orders[0]): float(orders[1])})
last_uID = rawOrderBook['lastUpdateId']
while self.ordBookBuff[0]['u'] <= last_uID:
del self.ordBookBuff[0]
if len(self.ordBookBuff) == 0:
break
if len(self.ordBookBuff) >= 1 :
for eachUpdate in self.ordBookBuff:
self.last_uID = eachUpdate['u']
self.updateOrderBook(eachUpdate)
self.ordBookBuff = []
self.SnapShotRecieved = True
else:
print('Error retieving order book.') #RESTfull request failed
def on_text_message(message, refObj):
messaged = json.loads(message)
if refObj.SnapShotRecieved == False:
refObj.ordBookBuff.append(messaged)
else:
refObj.updateOrderBook(messaged)
def delay(myObj):
time.sleep(2)
myObj.getOrderBookSnapshot()
def main():
pg.mkQApp()
refObj = coin()
websock = QtWebSockets.QWebSocket()
websock.connected.connect(lambda : print('connected'))
websock.disconnected.connect(lambda : print('disconnected'))
websock.error.connect(lambda e : print('error', e))
websock.textMessageReceived.connect(functools.partial(on_text_message, refObj=refObj))
url = QtCore.QUrl("wss://stream.binance.com:9443/ws/btcusdt#depth#1000ms")
websock.open(url)
getorderbook = threading.Thread(target = delay, args=(refObj,), daemon=True) #, args = (lambda : websocketThreadExitFlag,)
getorderbook.start()
pg.exec()
if __name__ == "__main__":
main()
Is it possible to create a Mayavi visualization that is updated on a timed bases rather than through Trait events?
I have a visualization that I need to update continually, but the data I am updating is coming from an external source (I.E. not an event from a user input from the graphical interface).
In the mean time, I need to be running a separate thread - so the Mayavi visualization can't control the main loop.
Can this be done? And if so How??
Any help would be very greatly appreciated.
Some dummy code for how I'm trying to tackle this is below:
import numpy
from mayavi.sources.array_source import ArraySource
from pyface.api import GUI
from mayavi.modules.api import Surface
from mayavi.api import Engine
import threading
import time
# Class runs a given function on a given thread at a given scan time
class TimedThread(threading.Thread):
def __init__(self, thread, scan_time, funct, *funct_args):
threading.Thread.__init__(self)
# Thread for the function to operate in
self.thread = thread
# Defines the scan time the function is to be run at
self.scan_time = scan_time
# Function to be run
self.run_function = funct
# Function arguments
self.funct_args = funct_args
def run(self):
while True:
# Locks the relevant thread
self.thread.acquire()
# Begins timer for elapsed time calculation
start_time = time.time()
# Runs the function that was passed to the thread
self.run_function(*self.funct_args)
# Wakes up relevant threads to listen for the thread release
self.thread.notify_all()
# Releases thread
self.thread.release()
# Calculates the elapsed process time & sleeps for the remainder of the scan time
end_time = time.time()
elapsed_time = end_time - start_time
sleep_time = self.scan_time - elapsed_time
if sleep_time > 0:
time.sleep(sleep_time)
else:
print 'Process time exceeds scan time'
# Function to update the visualisation
def update_visualisation(source):
print("Updating Visualization...")
# Pretend the data is being updated externally
x = numpy.array([0, numpy.random.rand()])
y = z = x
data = [x, y, z]
source.scalar_data = data
GUI.invoke_later(source.update)
# Function to run the visualisation
def run_main():
print 'Running Main Controller'
if __name__ == '__main__':
c = threading.Condition()
# Create a new Engine for Mayavi and start it
engine = Engine()
engine.start()
# Create a new Scene
engine.new_scene()
# Create the data
x = numpy.linspace(0, 10, 2)
y = z = x
data = [x, y, z]
# Create a new Source, map the data to the source and add it to the Engine
src = ArraySource()
src.scalar_data = data
engine.add_source(src)
# Create a Module
surf = Surface()
# Add the Module to the Engine
engine.add_module(surf)
# Create timed thread classes
visualisation_thread = TimedThread(c, 2.0, update_visualisation, src)
main_thread = TimedThread(c, 1.0, run_main)
# Start & join the threads
main_thread.start()
visualisation_thread.start()
main_thread.join()
visualisation_thread.join()
Found solution in the following link:
Animating a mayavi points3d plot
Solved by using the #mlab.animator to call the update function and using the yield command to release the animation to allow for user manipulation.
Solution below:
import numpy as np
import threading
import time
from mayavi import mlab
from mayavi.api import Engine
# Class runs a given function on a given thread at a given scan time
class SafeTimedThread(threading.Thread):
def __init__(self, thread_condition, scan_time, funct, *funct_args):
threading.Thread.__init__(self)
# Thread condition for the function to operate with
self.tc = thread_condition
# Defines the scan time the function is to be run at
self.scan_time = scan_time
# Function to be run
self.run_function = funct
# Function arguments
self.funct_args = funct_args
def run(self):
while True:
# Locks the relevant thread
self.tc.acquire()
# Begins timer for elapsed time calculation
start_time = time.time()
# Runs the function that was passed to the thread
self.run_function(*self.funct_args)
# Wakes up relevant threads to listen for the thread release
self.tc.notify_all()
# Releases thread
self.tc.release()
# Calculates the elapsed process time & sleep for the remainder of the scan time
end_time = time.time()
elapsed_time = end_time - start_time
sleep_time = self.scan_time - elapsed_time
if sleep_time > 0:
time.sleep(sleep_time)
else:
print 'Process time exceeds scan time'
# Function to run the main controller
def run_main():
print 'Running Main Controller'
def init_vis():
# Creates a new Engine, starts it and creates a new scene
engine = Engine()
engine.start()
engine.new_scene()
# Initialise Plot
data = np.random.random((3, 2))
x = data[0]
y = data[1]
z = data[2]
drawing = mlab.plot3d(x, y, z, np.ones_like(x))
return drawing
#mlab.animate(delay=500, ui=False)
def update_visualisation(drawing):
while True:
print ('Updating Visualisation')
# Pretend to receive data from external source
data = np.random.random((3, 2))
x = data[0]
y = data[1]
z = data[2]
drawing.mlab_source.set(x=x, y=y, z=z)
yield
if __name__ == '__main__':
# Create Condition for Safe Threading
c = threading.Condition()
# Create display window
dwg = init_vis()
# Create safe timed thread for main thread and start
main_thread = SafeTimedThread(c, 1.0, run_main).start()
# Update using mlab animator
vis_thread = update_visualisation(dwg)
mlab.show()
I am trying to plot real time data coming to the computer using python. Data comes in a ROS topic and I use 'rospy' to subscribe to the topic in order to get data.
This is the code I wrote
import rospy
from sensor_msgs.msg import ChannelFloat32
import matplotlib.pyplot as plt
N = 200
i = 0
topic = "chatter"
x = range(N)
lmotor = [0]*N
rmotor = [0]*N
plt.ion()
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlim([0,N])
ax.set_ylim([-1,1])
line1, = ax.plot(lmotor, 'r-')
line2, = ax.plot(rmotor, 'g')
def plotThrottle(data):
global x, lmotor, rmotor, i
[x[i],lmotor[i],rmotor[i], tmp] = data
line1.set_ydata(lmotor)
line1.set_xdata(x)
line2.set_ydata(rmotor)
line2.set_xdata(x)
fig.canvas.draw()
def callBack(packet):
data = list(packet.values)
plotThrottle(data)
def listner():
rospy.init_node('listener', anonymous=True)
rospy.Subscriber(topic, ChannelFloat32, callBack)
rospy.spin()
if __name__ == '__main__':
listner()
My problem is when I call plotThrottle() with the data I got from rostopic, I get following error.
[ERROR]
[WallTime: 1454388985.317080] bad callback: <function callBack at 0x7f13d98ba6e0>
Traceback (most recent call last):
File "/opt/ros/indigo/lib/python2.7/dist-packages/rospy/topics.py", line 720, in _invoke_callback
cb(msg)
File "dummy2.py", line 41, in callBack
plotThrottle(data)
File "dummy2.py", line 37, in plotThrottle
fig.canvas.draw()
File "/usr/lib/pymodules/python2.7/matplotlib/backends/backend_tkagg.py", line 349, in draw
tkagg.blit(self._tkphoto, self.renderer._renderer, colormode=2)
File "/usr/lib/pymodules/python2.7/matplotlib/backends/tkagg.py", line 13, in blit
tk.call("PyAggImagePhoto", photoimage, id(aggimage), colormode, id(bbox_array))
RuntimeError: main thread is not in main loop
But if I use the same function and pass some data generated within the code (some random data) plot works fine.
I am an absolute beginner to python. I searched about this error and it says that this is because of some threading problem. But I don't understand how to fix this code. I am really grateful if someone can explain the problem and help fix this code.
Here you have two threads running, rospy.spin() and top.mainloop() (from Tkinter, backend of matplotlib in your case).
From this answer:
The problems stem from the fact that the _tkinter module attempts to
gain control of the main thread via a polling technique when
processing calls from other threads.
Your Tkinter code in Thread-1 is trying to peek into the main thread
to find the main loop, and it's not there.
From this answer:
If there is another blocking call that keeps your program running,
there is no need to call rospy.spin(). Unlike in C++ where spin() is
needed to process all the threads, in python all it does is block.
So you can use plt.show(block=True) to keep your program from closing, in that case you will use Tkinter mainloop, redrawing your canvas without problems.
The listener fuction should look like this:
def listener():
rospy.init_node('listener', anonymous=True)
rospy.Subscriber(topic, ChannelFloat32, callBack)
# rospy.spin()
plt.show(block=True)
Anyway this seems a bit a workaround for other alternatives see again this answer or simply use separate node for plotting i.e. ros suggested tools like rqt_graph.
Since this is an old post and still seems to be active in the community, I am going to provide an example, in general, how can we do real-time plotting. Here I used matplotlib FuncAnimation function.
import matplotlib.pyplot as plt
import rospy
import tf
from nav_msgs.msg import Odometry
from tf.transformations import quaternion_matrix
import numpy as np
from matplotlib.animation import FuncAnimation
class Visualiser:
def __init__(self):
self.fig, self.ax = plt.subplots()
self.ln, = plt.plot([], [], 'ro')
self.x_data, self.y_data = [] , []
def plot_init(self):
self.ax.set_xlim(0, 10000)
self.ax.set_ylim(-7, 7)
return self.ln
def getYaw(self, pose):
quaternion = (pose.orientation.x, pose.orientation.y, pose.orientation.z,
pose.orientation.w)
euler = tf.transformations.euler_from_quaternion(quaternion)
yaw = euler[2]
return yaw
def odom_callback(self, msg):
yaw_angle = self.getYaw(msg.pose.pose)
self.y_data.append(yaw_angle)
x_index = len(self.x_data)
self.x_data.append(x_index+1)
def update_plot(self, frame):
self.ln.set_data(self.x_data, self.y_data)
return self.ln
rospy.init_node('lidar_visual_node')
vis = Visualiser()
sub = rospy.Subscriber('/dji_sdk/odometry', Odometry, vis.odom_callback)
ani = FuncAnimation(vis.fig, vis.update_plot, init_func=vis.plot_init)
plt.show(block=True)
Note: change the rospy.Subscriber('/dji_sdk/odometry', Odometry, vis.odom_callback) as you need and do necessary changes accordingly.
my problem seem to be similar to This Thread however, while I think I am following the advised method, I still get a PicklingError. When I run my process locally without sending to an IPython Cluster Engine the function works fine.
I am using zipline with IPyhon's notebook, so I first create a class based on zipline.TradingAlgorithm
Cell [ 1 ]
from IPython.parallel import Client
rc = Client()
lview = rc.load_balanced_view()
Cell [ 2 ]
%%px --local # This insures that the Class and modules exist on each engine
import zipline as zpl
import numpy as np
class Agent(zpl.TradingAlgorithm): # must define initialize and handle_data methods
def initialize(self):
self.valueHistory = None
pass
def handle_data(self, data):
for security in data.keys():
## Just randomly buy/sell/hold for each security
coinflip = np.random.random()
if coinflip < .25:
self.order(security,100)
elif coinflip > .75:
self.order(security,-100)
pass
Cell [ 3 ]
from zipline.utils.factory import load_from_yahoo
start = '2013-04-01'
end = '2013-06-01'
sidList = ['SPY','GOOG']
data = load_from_yahoo(stocks=sidList,start=start,end=end)
agentList = []
for i in range(3):
agentList.append(Agent())
def testSystem(agent,data):
results = agent.run(data) #-- This is how the zipline based class is executed
#-- next I'm just storing the final value of the test so I can plot later
agent.valueHistory.append(results['portfolio_value'][len(results['portfolio_value'])-1])
return agent
for i in range(10):
tasks = []
for agent in agentList:
#agent = testSystem(agent,data) ## On its own, this works!
#-- To Test, uncomment the above line and comment out the next two
tasks.append(lview.apply_async(testSystem,agent,data))
agentList = [ar.get() for ar in tasks]
for agent in agentList:
plot(agent.valueHistory)
Here is the Error produced:
PicklingError Traceback (most recent call last)/Library/Python/2.7/site-packages/IPython/kernel/zmq/serialize.pyc in serialize_object(obj, buffer_threshold, item_threshold)
100 buffers.extend(_extract_buffers(cobj, buffer_threshold))
101
--> 102 buffers.insert(0, pickle.dumps(cobj,-1))
103 return buffers
104
PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
If I override the run() method from zipline.TradingAlgorithm with something like:
def run(self, data):
return 1
Trying something like this...
def run(self, data):
return zpl.TradingAlgorithm.run(self,data)
results in the same PicklingError.
then the passing off to the engines works, but obviously the guts of the test are not performed. As run is a method internal to zipline.TradingAlgorithm and I don't know everything that it does, how would I make sure it is passed through?
It looks like the zipline TradingAlgorithm object is not pickleable after it has been run:
import zipline as zpl
class Agent(zpl.TradingAlgorithm): # must define initialize and handle_data methods
def handle_data(self, data):
pass
agent = Agent()
pickle.dumps(agent)[:32] # ok
agent.run(data)
pickle.dumps(agent)[:32] # fails
But this suggests to me that you should be creating the Agents on the engines, and only passing data / results back and forth (ideally, not passing data across at all, or at most once).
Minimizing data transfers might look something like this:
define the class:
%%px
import zipline as zpl
import numpy as np
class Agent(zpl.TradingAlgorithm): # must define initialize and handle_data methods
def initialize(self):
self.valueHistory = []
def handle_data(self, data):
for security in data.keys():
## Just randomly buy/sell/hold for each security
coinflip = np.random.random()
if coinflip < .25:
self.order(security,100)
elif coinflip > .75:
self.order(security,-100)
load the data
%%px
from zipline.utils.factory import load_from_yahoo
start = '2013-04-01'
end = '2013-06-01'
sidList = ['SPY','GOOG']
data = load_from_yahoo(stocks=sidList,start=start,end=end)
agent = Agent()
and run the code:
def testSystem(agent, data):
results = agent.run(data) #-- This is how the zipline based class is executed
#-- next I'm just storing the final value of the test so I can plot later
agent.valueHistory.append(results['portfolio_value'][len(results['portfolio_value'])-1])
# create references to the remote agent / data objects
agent_ref = parallel.Reference('agent')
data_ref = parallel.Reference('data')
tasks = []
for i in range(10):
for j in range(len(rc)):
tasks.append(lview.apply_async(testSystem, agent_ref, data_ref))
# wait for the tasks to complete
[ t.get() for t in tasks ]
And plot the results, never fetching the agents themselves
%matplotlib inline
import matplotlib.pyplot as plt
for history in rc[:].apply_async(lambda : agent.valueHistory):
plt.plot(history)
This is not quite the same code you shared - three agents bouncing back and forth on all your engines, whereas this has on agent per engine. I don't know enough about zipline to say whether that's useful to you or not.
I am having a problem getting matplotlib to work well with interactive plotting... what I see is that after displaying a few frames of my simulated data matplotlib hangs-and doesn't display any more.
Basically I've been playing around a bit with science simulations - and would like to be able to plot my results as they are being made - rather than at the end - using pylab.show().
I found a cookbook example from a while back that seems to do what I would want - in simple terms (although obv. the data is different). The cookbook is here...http://www.scipy.org/Cookbook/Matplotlib/Animations#head-2f6224cc0c133b6e35c95f4b74b1b6fc7d3edca4
I have searched around a little and I know that some people had these problems before - Matplotlib animation either freezes after a few frames or just doesn't work but it seems at the time there were no good solutions. I was wondering if someone has since found a good solution here.
I have tried a few 'backends' on matplotlib....TkAgg seems to work for a few frames.... qt4agg doesn't show the frames. I haven't yet got GTK to install properly.
I am running the most recent pythonxy(2.7.3).
Anyone have any advice?
import matplotlib
matplotlib.use('TkAgg') # 'Normal' Interactive backend. - works for several frames
#matplotlib.use('qt4agg') # 'QT' Interactive backend. - doesn't seem to work at all
#matplotlib.use('GTKAgg') # 'GTK' backend - can't seem to get this to work.... -
import matplotlib.pyplot as plt
import time
import numpy as np
plt.ion()
tstart = time.time() # for profiling
x = np.arange(0,2*np.pi,0.01) # x-array
line, = plt.plot(x,np.sin(x))
#plt.ioff()
for i in np.arange(1,200):
line.set_ydata(np.sin(x+i/10.0)) # update the data
line.axes.set_title('frame number {0}'.format(i))
plt.draw() # redraw the canvas
print 'FPS:' , 200/(time.time()-tstart)
EDIT:
edited code - to get rid of some style issues brought up.
Ok... So I have mangled together something that may sort of work for me....
Basically it is something like a watered down gui - but i'm hoping that it is a class i can import and basically forget about the details of (here's hoping).
I should say though - this is my first attempt at threading OR guis in python - so this code comes with a health warning.
** I'm not going to mark the question as answered though - because i'm sure someone more experienced will have a better solution.
'''
JP
Attempt to get multiple updating of matplotlibs working.
Uses WX to create an 'almost' gui with a mpl in the middle of it.
Data can be queued to this object - or you can directly plot to it.
Probably will have some limitations atm
- only really thinking about 2d plots for now -
but presumably can work around this for other implimentations.
- the working code seems to need to be put into another thread.
Tried to put the wx mainloop into another thread,
but it seemed unhappy. :(
Classes of Interest :
GraphData - A silly class that holds data to be plotted.
PlotFigure - Class of wx frame type.
Holds a mpl figure in it + queue to queue data to.
The frame will plot the data when it refreshes it's canvas
ThreadSimulation - This is not to do with the plotting
it is a test program.
Modified version of:
Copyright (C) 2003-2005 Jeremy O'Donoghue and others
License: This work is licensed under the PSF. A copy should be included
with this source code, and is also available at
http://www.python.org/psf/license.html
'''
import threading
import collections
import time
import numpy as np
import matplotlib
matplotlib.use('WXAgg')
from matplotlib.backends.backend_wxagg import FigureCanvasWxAgg
from matplotlib.backends.backend_wx import NavigationToolbar2Wx
from matplotlib.figure import Figure
import wx
class GraphData(object):
'''
A silly class that holds data to be plotted.
'''
def __init__(self, xdatainit, ydatainit):
self.xdata = xdatainit
self.ydata = ydatainit
class PlotFigure(wx.Frame):
def __init__(self ):
'''
Initialises the frame.
'''
wx.Frame.__init__(self, None, -1, "Test embedded wxFigure")
self.timerid = wx.NewId()
self.fig = Figure((5,4), 75)
self.canvas = FigureCanvasWxAgg(self, -1, self.fig)
self.toolbar = NavigationToolbar2Wx(self.canvas)
self.toolbar.Realize()
# On Windows, default frame size behaviour is incorrect
# you don't need this under Linux
tw, th = self.toolbar.GetSizeTuple()
fw, fh = self.canvas.GetSizeTuple()
self.toolbar.SetSize(wx.Size(fw, th))
# Now put all into a sizer
sizer = wx.BoxSizer(wx.VERTICAL)
# This way of adding to sizer allows resizing
sizer.Add(self.canvas, 1, wx.LEFT|wx.TOP|wx.GROW)
# Best to allow the toolbar to resize!
sizer.Add(self.toolbar, 0, wx.GROW)
self.SetSizer(sizer)
self.Fit()
wx.EVT_TIMER(self, self.timerid, self.onTimer)
self.dataqueue = collections.deque()
# Add an axes and a line to the figure.
self.axes = self.fig.add_subplot(111)
self.line, = self.axes.plot([],[])
def GetToolBar(self):
'''
returns default toolbar.
'''
return self.toolbar
def onTimer(self, evt):
'''
Every timer period this is called.
Want to redraw the canvas.
'''
#print "onTimer"
if len(self.dataqueue) > 0 :
data = self.dataqueue.pop()
x = data.xdata
y = data.ydata
xmax = max(x)
xmin = min(x)
ymin = round(min(y), 0) - 1
ymax = round(max(y), 0) + 1
self.axes.set_xbound(lower=xmin, upper=xmax)
self.axes.set_ybound(lower=ymin, upper=ymax)
self.line.set_xdata(x)
self.line.set_ydata(y)
# Redraws the canvas - does this even if the data isn't updated...
self.canvas.draw()
def onEraseBackground(self, evt):
'''
this is supposed to prevent redraw flicker on some X servers...
'''
pass
class ThreadSimulation(threading.Thread):
'''
Simulation Thread - produces data to be displayed in the other thread.
'''
def __init__(self, nsimloops, datastep, pltframe, slowloop = 0):
threading.Thread.__init__(self)
self.nsimloops = nsimloops
self.datastep = datastep
self.pltframe = pltframe
self.slowloop=slowloop
def run(self):
'''
This is the simulation function.
'''
nsimloops = self.nsimloops
datastep = self.datastep
pltframe = self.pltframe
print 'Sim Thread: Starting.'
tstart = time.time() # for profiling
# Define Data to share between threads.
x = np.arange(0,2*np.pi,datastep) # x-array
y = np.sin(x )
# Queues up the data and removes previous versions.
pltframe.dataqueue.append(GraphData(x,y))
for i in range(len(pltframe.dataqueue)-1):
pltframe.dataqueue.popleft()
pltframe.dataqueue
for i in np.arange(1, nsimloops):
x = x + datastep
y = np.sin(x)
# Queues up the data and removes previous versions.
pltframe.dataqueue.append(GraphData(x,y))
for i in range(len(pltframe.dataqueue)-1):
pltframe.dataqueue.popleft()
#pltframe.dataqueue
if self.slowloop > 0 :
time.sleep(self.slowloop)
tstop= time.time()
print 'Sim Thread: Complete.'
print 'Av Loop Time:' , (tstop-tstart)/ nsimloops
if __name__ == '__main__':
# Create the wx application.
app = wx.PySimpleApp()
# Create a frame with a plot inside it.
pltframe = PlotFigure()
pltframe1 = PlotFigure()
# Initialise the timer - wxPython requires this to be connected to
# the receiving event handler
t = wx.Timer(pltframe, pltframe.timerid)
t.Start(100)
pltframe.Show()
pltframe1.Show()
npoints = 100
nsimloops = 20000
datastep = 2 * np.pi/ npoints
slowloop = .1
#Define and start application thread
thrd = ThreadSimulation(nsimloops, datastep, pltframe,slowloop)
thrd.setDaemon(True)
thrd.start()
pltframe1.axes.plot(np.random.rand(10),np.random.rand(10))
app.MainLoop()