traits_pickle_problem.py
from traits.api import HasTraits, List
import cPickle
class Client(HasTraits):
data = List
class Person(object):
def __init__(self):
self.client = Client()
# dynamic handler
self.client.on_trait_event(self.report,'data_items')
def report(self,obj,name,old,new):
print 'client added-- ' , new.added
if __name__ == '__main__':
p = Person()
p.client.data = [1,2,3]
p.client.data.append(10)
cPickle.dump(p,open('testTraits.pkl','wb'))
The above code reports a dynamic trait. Everything works as expected in this code. However, using a new python process and doing the following:
>>> from traits_pickle_problem import Person, Client
>>> p=cPickle.load(open('testTraits.pkl','rb'))
>>> p.client.data.append(1000)
causes no report of the list append. However, re-establishing the listener separately as follows:
>>> p.client.on_trait_event(p.report,'data_items')
>>> p.client.data.append(1000)
client added-- [1000]
makes it work again.
Am I missing something or does the handler need to be re-established in __setstate__ during the unpickling process.
Any help appreciated. This is for Python 2.7 (32-bit) on windows with traits version 4.30.
Running pickletools.dis(cPickle.dumps(p)), you can see the handler object being referenced:
...
213: c GLOBAL 'traits.trait_handlers TraitListObject'
...
But there's no further information on how it should be wired to the report method. So either the trait_handler doesn't pickle itself out properly, or it's an ephemeral thing like a file handle that can't be pickled in the first place.
In either case, your best option is to overload __setstate__ and re-wire the event handler when the object is re-created. It's not ideal, but at least everything is contained within the object.
class Person(object):
def __init__(self):
self.client = Client()
# dynamic handler
self.client.on_trait_event(self.report, 'data_items')
def __setstate__(self, d):
self.client = d['client']
self.client.on_trait_event(self.report, 'data_items')
def report(self, obj, name, old, new):
print 'client added-- ', new.added
Unpickling the file now correctly registers the event handler:
p=cPickle.load(open('testTraits.pkl','rb'))
p.client.data.append(1000)
>>> client added-- [1000]
You might find this talk Alex Gaynor did at PyCon interesting. It goes into the high points of how pickling work under the hood.
EDIT - initial response used on_trait_change - a typo that appears to work. Changed it back to on_trait_event for clarity.
I had the same problem but came around like this: Imaging I want to pickle only parts of a quiet big class and some of the objects has been set so transient=True so they're not pickled because there is nothing important to save, e.g.
class LineSpectrum(HasTraits):
andor_cam = Instance(ANDORiKonM, transient=True)
In difference to objects which should be saved, e.g.
spectrometer = Instance(SomeNiceSpectrometer)
In my LineSpectrum class, I have a
def __init__(self, f):
super(LineSpectrum, self).__init__()
self.load_spectrum(f)
def __setstate__(self, state): # WORKING!
print("LineSpectrum: __setstate__ with super(...) call")
self.__dict__.update(state)
super(LineSpectrum, self).__init__() # this has to be done, otherwise pickled sliders won't work, also first update __dict__!
self.from_pickle = True # is not needed by traits, need it for myself
self.andor_cam = ANDORiKonM(self.filename)
self.load_spectrum(self.filename)
In my case, this works perfectly - all sliders are working, all values set at the time the object has been pickled are set back.
Hope this works for you or anybody who's having the same problem. Got Anaconda Python 2.7.11, all packages updated.
PS: I know the thread is old, but didn't want to open a new one just for this.
Related
I have classes that contain unpickleable attributes, e.g. COM interfaces. When I pickle the classes I can remove those attributes to make the whole instance of this class pickleable.
But when I unpickle the dumped data again, of course I want to re-establish those previously removed attributes. The easiest for me would be to provide the new values, e.g. the COM interfaces, as additional parameters to __setstate__() but of course this is not possible.
Is there another nice solution to this without setting all those attributes one by another from externally? Maybe something using __getnewargs__(), which I have not really understood yet?
Here is a minimal example:
import win32com.client as COM
import pickle
class Test:
def __init__(self, app):
self.app = app
def __getstate__(self):
attr = self.__dict__.copy()
# Remove the unpicklable entries.
attr['app'] = None
return attr
def __setstate__(self, attr):
self.__dict__.update(attr)
# FIXME: Re-establish COM interfaces here! Would be nice to provide
# app as parameter an do self.app = app here.
# This must be done outside the class since it is only that easy in this
# minimal example.
app = COM.Dispatch('designer.Application')
test = Test(app)
with open('Test.pickle', 'wb') as fid:
pickle.dump(test, fid)
with open('Test.pickle', 'rb') as fid:
test_new = pickle.load(fid)
# Of course now I could do
# test_new.app = app
# But this is cumbersome with a lot of such attributes to be set...
print(test.app, test_new.app)
I have implemented websocket in Django app using Django-channels, now the front-end send some data through the websocket and i want the current running celery task to be able to read it. I tried creating shared memory static object, but not working.
SimulationInputs.add(simulation_id=simulation.id, init_data=init_inputs)
return InteractiveSimulationTask.delay_or_fail(
simulation_id=simulation.id
)
class SimulationData:
data = ''
class SimulationInputs:
data = None
#classmethod
def init_manager(cls, manager):
manager = Manager()
cls.data = manager.dict()
#classmethod
def add(cls, simulation_id, init_data):
cls.data[simulation_id] = init_data
#classmethod
def write(cls, simulation_id, simulation_data):
if cls.data.get(simulation_id):
cls.data[simulation_id] = simulation_data
#classmethod
def read(cls, simulation_id, simulation_data):
simulation_data.data = cls.data.get(simulation_id)
# manage.y
if __name__ == "__main__":
SimulationInputs.init_manager()
class InteractiveSimulationTask(JobtasticTask):
def calculate_result(self, simulation_id, **kwargs):
while True:
SimulationInputs.read(simulation_id=self.simulation.id, simulation_data=simulation_data)
The task always throw exception cls.data.get(simulation_id): NoneObjectType has no method get
I need to share data between the celery task and the main process.
Any hint?
Since you're using celery, you probably have redis or some other memory-store available. Consider using this as your indirection layer, i.e. the read and write methods use the simulation_id as a key to the simulation data
I believe the issue you're facing is due to the lifecycle of the python class. In init_manager when you assign to cls.data you're overwriting the class's property, not the instance's property. This doesn't do what you want it to, as evidenced by the error message: cls.data is going to be None.
What I think you're going for is the "Singleton Pattern". You want to have one and only SimulationInputs object which can read/write the data for each ID. This discussion can help you with implementing a singleton in python
I come up to conclusion that Django and celery should not share the memory, because they are on diff. process and they are diff programs, so they should communicate through socket or messaging system. I solved my problem by using redis Pub/Sub https://redis.io/topics/pubsub.
I'm inheriting a class with an init method already set.
I'd like to add an arg to this method, as well as a few lines of code.
I suspect that I can use super to do this, but am not quite sure how and the examples I've found don't always make sense to me.
Example code AND EXPLANATION:
The bot module has 2 classes: Bot, and BotStreamListener
BotStreamListener inherits from tweepy.StreamListener, which already has init defined
I need to append an arg and some code to this classmethod so that methods in the BotStreamListener class know the instantiated instance of the Bot class. (in the example below, it's mybot).
...
def main():
try:
me = api.me()
print "Starting userstream for %s ( %s )" %(me.name, me.screen_name)
mybot = bot.bot(api)
#What it looks like now
stream = tweepy.Stream(auth, bot.BotStreamListener(api))
#What i would like it to be able to handle
stream = tweepy.Stream(auth, bot.BotStreamListener(api, mybot))
In short, I would like to add a few more args and some code to an inherited init classmethod, without redefining the method.
I'm using python 2.7, any help is apreciated.
I'm not familiar with tweepy other than knowing what it is, but this should get you started:
# bot.py
class BotStreamListener(StreamListener):
def __init__(api, bot):
super(self.__class__, self).__init__(api)
# Do your stuff
self.bot = bot
# Do more of your stuff
mybot = bot.bot(api)
bsl = BotStreamListener(api, mybot)
I am changing some code to spin up VMs in ec2 instead of openstack. Main starts a thread per VM, and then various modules perform tasks on these VM. Each thread controls it's own VM. So, instead of either having to add parameters to all of the downstream modules to look up information, or having to change all of the code to unpickle the class instance that created the vm, I am hoping that I can have the class itself decide whether to start a new VM or return the existing pickle. That way the majority of the code wont need to be altered.
This is the general idea, and closest I have gotten to getting it to work:
import os
import sys
import pickle
if sys.version_info >= (2, 7):
from threading import current_thread
else:
from threading import currentThread as current_thread
class testA:
def __init__(self, var="Foo"):
self.class_pickle_file = "%s.p" % current_thread().ident
if os.path.isfile(self.class_pickle_file):
self.load_self()
else:
self.var = var
pickle.dump(self, open(self.class_pickle_file, "wb"))
def test_method(self):
print self.var
def load_self(self):
return pickle.load(open(self.class_pickle_file, "rb"))
x = testA("Bar")
y = testA()
y.test_method()
But that results in: NameError: global name 'var' is not defined
But, If I do y = pickle.load(open("140355004004096.p", "rb")) it works just fine. So the data IS getting in there by storing self inside the class, it's a problem of getting the class to return the pickle instead of itself...
Any ideas? Thanks.
It looks to me like you create a file named by the current thread's ident, then you instantiate another TestA object using the same thread (!!same ident!!), so it checks for a pickle file (and finds it, that's bad), then self.var never gets set.
In test_method, you check for a variable that was never set.
Run each item in its own thread to get different idents, or ensure you set self.var no matter what.
I am updating some code from using libglade to GtkBuilder, which is supposed to be the way of the future.
With gtk.glade, you could call glade_xml.signal_autoconnect(...) repeatedly to connect signals onto objects of different classes corresponding to different windows in the program. However Builder.connect_signals seems to work only once, and (therefore) to give warnings about any handlers that aren't defined in the first class that's passed in.
I realize I can connect them manually but this seems a bit laborious. (Or for that matter I could use some getattr hackery to let it connect them through a proxy to all the objects...)
Is it a bug there's no function to hook up handlers across multiple objects? Or am I missing something?
Someone else has a similar problem http://www.gtkforums.com/about1514.html which I assume means this can't be done.
Here's what I currently have. Feel free to use it, or to suggest something better:
class HandlerFinder(object):
"""Searches for handler implementations across multiple objects.
"""
# See <http://stackoverflow.com/questions/4637792> for why this is
# necessary.
def __init__(self, backing_objects):
self.backing_objects = backing_objects
def __getattr__(self, name):
for o in self.backing_objects:
if hasattr(o, name):
return getattr(o, name)
else:
raise AttributeError("%r not found on any of %r"
% (name, self.backing_objects))
I have been looking for a solution to this for some time and found that it can be done by passing a dict of all the handlers to connect_signals.
The inspect module can extract methods using
inspect.getmembers(instance, predicate=inspect.ismethod
These can then be concatenated into a dictionary using d.update(d3), watching out for duplicate functions such as on_delete.
Example code:
import inspect
...
handlers = {}
for c in [win2, win3, win4, self]: # self is the main window
methods = inspect.getmembers(c, predicate=inspect.ismethod)
handlers.update(methods)
builder.connect_signals(handlers)
This will not pick up alias method names declared using #alias. For an example of how to do that, see the code for Builder.py, at def dict_from_callback_obj.
I'm only a novice but this is what I do, maybe it can inspire;-)
I instantiate the major components from a 'control' and pass the builder object so that the instantiated object can make use of any of the builder objects (mainwindow in example) or add to the builder (aboutDialog example). I also pass a dictionary (dic) where each component adds "signals" to it.
Then the 'connect_signals(dic)' is executed.
Of course I need to do some manual signal connecting when I need to pass user arguments to the callback method, but those are few.
#modules.control.py
class Control:
def __init__(self):
# Load the builder obj
guibuilder = gtk.Builder()
guibuilder.add_from_file("gui/mainwindow.ui")
# Create a dictionnary to store signal from loaded components
dic = {}
# Instanciate the components...
aboutdialog = modules.aboutdialog.AboutDialog(guibuilder, dic)
mainwin = modules.mainwindow.MainWindow(guibuilder, dic, self)
...
guibuilder.connect_signals(dic)
del dic
#modules/aboutdialog.py
class AboutDialog:
def __init__(self, builder, dic):
dic["on_OpenAboutWindow_activate"] = self.on_OpenAboutWindow_activate
self.builder = builder
def on_OpenAboutWindow_activate(self, menu_item):
self.builder.add_from_file("gui/aboutdialog.ui")
self.aboutdialog = self.builder.get_object("aboutdialog")
self.aboutdialog.run()
self.aboutdialog.destroy()
#modules/mainwindow.py
class MainWindow:
def __init__(self, builder, dic, controller):
self.control = controller
# get gui xml and/or signals
dic["on_file_new_activate"] = self.control.newFile
dic["on_file_open_activate"] = self.control.openFile
dic["on_file_save_activate"] = self.control.saveFile
dic["on_file_close_activate"] = self.control.closeFile
...
# get needed gui objects
self.mainWindow = builder.get_object("mainWindow")
...
Edit: alternative to auto attach signals to callbacks:
Untested code
def start_element(name, attrs):
if name == "signal":
if attrs["handler"]:
handler = attrs["handler"]
#Insert code to verify if handler is part of the collection
#we want.
self.handlerList.append(handler)
def extractSignals(uiFile)
import xml.parsers.expat
p = xml.parsers.expat.ParserCreate()
p.StartElementHandler = self.start_element
p.ParseFile(uiFile)
self.handlerList = []
extractSignals(uiFile)
for handler in handlerList:
dic[handler] = eval(''. join(["self.", handler, "_cb"]))
builder.connect_signals
({
"on_window_destroy" : gtk.main_quit,
"on_buttonQuit_clicked" : gtk.main_quit
})