som = SOM_CLASS() # includes many big difficult data structures
som.hard_work()
som.save_to_disk(filename)
#then later or another program
som = SOM_CLASS()
som.read_from_file(filename)
som.do_anythink_else()
or
som = SOM_CLASS()
save(som)
#...
load(som)
som.work()
what is easiest way to do this?
You can (de)serialize with pickle. It is backward-compatible, i.e. it will support all old protocols in future versions.
import pickle
som = SOM_CLASS()
fileObject = <any file-like object>
pickle.dump(som, fileObject)
#...
som = pickle.load(fileObject)
som.work()
But mind that if you transfer pickled objects to another computer, make sure the connection cannot be tampered with as pickle might be unsecure (this is an article that every pickle user should know).
Another alternative is the older module marshal.
I use this code:
import cPickle
import traceback
class someClass():
def __init__(self):
#set name from variable name. http://stackoverflow.com/questions/1690400/getting-an-instance-name-inside-class-init
(filename,line_number,function_name,text)=traceback.extract_stack()[-2]
def_name = text[:text.find('=')].strip()
self.name = def_name
try:
self.load()
except:
##############
#to demonstrate
self.someAttribute = 'bla'
self.someAttribute2 = ['more']
##############
self.save()
def save(self):
"""save class as self.name.txt"""
file = open(self.name+'.txt','w')
file.write(cPickle.dumps(self.__dict__))
file.close()
def load(self):
"""try load self.name.txt"""
file = open(self.name+'.txt','r')
dataPickle = file.read()
file.close()
self.__dict__ = cPickle.loads(dataPickle)
This code saves and loads the class from its actual class instance name. Code is from my blog http://www.schurpf.com/python-save-a-class/.
Take a look at Python's pickle library.
Use pickle in this way:
import pickle
class Student:
def __init__(self, name, age, grade):
self.name = name
self.age = age
self.grade = grade # 0 - 100
def get_grade(self):
print (self.grade)
s1 = Student("Tim", 19, 95)
#save it
with open(f'test.pickle', 'wb') as file:
pickle.dump(s1, file)
#load it
with open(f'test.pickle', 'rb') as file2:
s1_new = pickle.load(file2)
#check it
s1_new.get_grade()
# it prints 95
Related
I am trying to use Python unittest library to test the following code.
from cv2 import imwrite
import h5py
import lumpy as np
class Myclass:
def __init__(self):
self.data = []
def get_data(self):
return self.data
def load_data(self, path):
count = 10
for i in range(count):
mat_file = path + f'{i}.mat'
with h5py.File(mat_file, 'r') as fin:
data = np.array(fin['data'])
for j in range(len(data)):
filename = path + f'{i}_{j}.jpg'
imwrite(filename, data)
self.data = data
I tried to do the following. But I am getting AttributeError saying Myclass has no attribute 'imwrite'. And I do not know how to mock the nested loops with image writing imwrite.
from mock import MagicMock, patch
def test():
m = MagicMock()
m.__enter__.return_value = data
with patch("h5py.File", return_value=m):
with patch(Myclass.imwrite) as mock_imwrite:
testclass = Myclass()
testclass.load_data(path='test')
assert testclass.get_data() == data
I hope someone can help me out. Any help is very much appreciated
The issue here is that your class Myclass does not have a function imwrite. Only the module (aka the file the class is defined in) is aware of this function.
If you want to patch the imwrite function of the package cv2, you have to write:
(untested code)
with patch('cv2.imwrite') as mock_imwrite:
testclass = Myclass()
...
But the as mock_imwrite part is only needed, if you run an assert on the mock. Otherwise, you may just skip it.
I'm currently working on a text adventure in python (this language just because it's the one I know), and I'm finding that creating and loading savefiles removes some of the mecahnics I've made. I'll include the problematic code here, rather than all of elements that work fine. it's mainly to do with classes and how instances are 'pickled'.
Here are some of the classes I've created:
class Player:
def __init__(self, name):
self.sack = []
self.kit = []
self.equipped = []
self.log = []
self.done = []
class Weapon:
def __init__(self, name, price, minattack, maxattack):
self.name = name
self.price = price
self.minattack = minattack
self.maxattack = maxattack
class Food:
def __init__(self, name, price, healthadd):
self.name = name
self.price = price
self.healthadd = healthadd
class Quest:
def __init__(self, name, requirement, num, gold, npc, place, give_text, prog_text, comp_text, done_text):
self.name = name
self.requirement = requirement
self.num = num
self.score = 0
self.gold = gold
self.npc = npc
self.place = place
self.give_text = give_text
self.prog_text = prog_text
self.comp_text = comp_text
self.done_text = done_text
The instances in the Player class I've included here are just the ones that are appended by other mechanics with Weapons, Food and Quests. The code includes regions where Weapons, Food and Quests are populated (though working on a separate assets file might tidy things up a bit).
Here's how the save/load functions work currently:
def save(lastplace):
clear()
with open('savefile', 'wb') as f:
PlayerID.currentplace = lastplace.name
pickle.dump(PlayerID, f)
print("Game saved:\n")
print(PlayerID.name)
print("(Level %i)" % (PlayerID.level))
print(lastplace.name)
print('')
option = input(">>> ")
goto(lastplace)
def load():
clear()
if os.path.exists("savefile") == True:
with open('savefile', 'rb') as f:
global PlayerID
PlayerID = pickle.load(f)
savedplace = PlayerID.currentplace
savedplace = locations[savedplace]
goto(savedplace)
else:
print("You have no save file for this game.")
option = input('>>> ')
main()
It's worth noting that upon entry to the game, PlayerID (you) becomes a global variable. You might begin to see some of the issues here, or rather the overarching issue. Essentially, the pickling process serialises all of the possible class types stored in lists within the class of Player just get appended by their names, thus removing their class properties when loaded back into the game.
Is there a pythonic way to ensure that class instances are saved for a future load so that they can still behave as classes, particularly when stacked inside the class of Player? I appreciate this is more of an editorial rather than a question by its length, but any help would be hugely appreciated.
I am trying to write python code that will be able to track the created instances of a class, and save it through sessions. I am trying to to this by creating a list inside the class deceleration, which keeps track of instances. My code is as follows:
class test_object:
_tracking = []
def __init__(self, text):
self.name = text
test_object._tracking.insert(0, self)
with open("tst.pkl", mode="rb") as f:
try:
pickles = dill.load(f)
except:
pickles = test_object
logger.warning("Found dill to be empty")
f.close()
My issue is handling when the pickled data is empty. What I'd like to do is in this case simply use the base class. The issue I'm running into is that test_object._tracking ends up being equal to pickles._tracking. Is there a way to make a copy of test_object so that when test_object._tracking gets updates, pickles._tracking stays the same?
You can do the following:
import dill
class test_object:
_tracking = []
def __init__(self, text):
self.name = text
test_object._tracking.insert(0, self)
test_1 = test_object("abc")
print(test_object._tracking)
# result: [<__main__.test_object object at 0x11a8cda50>]
with open("./my_file.txt", mode="rb") as f:
try:
pickles = dill.load(f)
except:
pickles = type('test_object_copy', test_object.__bases__, dict(test_object.__dict__))
pickles._tracking = []
print("Found dill to be empty")
# The above results in "Found dill to be empty"
print(pickles._tracking)
# prints []
it'll set pickles to a copy of the original class. It's tracking attribute would then be empty, and would be different than the original 'tracking'.
I want to create my own logging class: which writes some data to a text file.
For this I have made a class mylog.py
I want to be able to create an instance object of the mylog.py class an pass the instance object as a paremeter to the other classes I have written.
However when I try to access the mylog object using the self notation and without using the self notation I am having issues.
The issue is that when I refer to the mylog object in the startup class and use self.log = logger this doesn't work to use the methods of the mylog class like self.log.write() nor does setting logobj to a variable without self and passing that in.
My mylog.py class
import datetime
import os
class logtextfile(object):
def __init__(self, name):
self.name = name
def __str__(self):
return "{} ".format(self.__class__.__name__)
def write(self,**kwargs):
"""Writes a log message to a user specified file which indicates the action takes and if it was successful"""
self.file = kwargs.get('file',"log.txt")
self.loglevel = kwargs.get('loglevel',"critical")
self.logmessage = kwargs.get('logmessage',"error")
self.success = kwargs.get('success',False)
self.class_name = kwargs.get('class',str("{}".format(self.__class__.__name__)))
self.output = ", ".join([str(datetime.datetime.now().replace(second=0,microsecond=0)),self.class_name,str(self.logmessage),str(self.success),str("\n")])
for key, value in kwargs.items():
setattr(self,key,value)
f = open(str(self.file),"a")
f.write(self.output)
f.close()
def now(self, filename, openas, data):
"""Creates a log file with todays date and time"""
fmt='%Y-%m-%d-%H-%M-%S_{fname}'
fn = datetime.datetime.now().strftime(fmt).format(fname=filename)
f = open(str(fn),openas)
f.write(data + "\n")
f.close()
My startup class
import pandas as pd
import numpy as np
from pandas_datareader import data as web
import datetime
import requests
import lxml
from IPython.display import clear_output
import time
import timeit
from bs4 import BeautifulSoup
import re
import os
import sqlite3
from sqlalchemy import create_engine # database connection
from zenlog import log
class company(object):
def __init__(self, name, logobj):
self.name = name
logger = logobj
def __str__(self):
return "{} ".format(self.__class__.__name__)
def listed(self):
try:
#all companies on asx downloaded from asx website csv
self.function_name = str("{}".format(self.__class__.__name__))
df = pd.read_csv('http://asx.com.au/asx/research/ASXListedCompanies.csv', skiprows=1)
df.columns = ["company","asx_code","industry"]
df["yahoo_code"] = df["asx_code"]+".AX"
message = "succesfully downloaded ASXListedCompanies.csv"
logger.write(file="asx_module_log.txt",logmessage=message,success=True)
return df
except:
message = "ASXListedCompanies.csv could not be retrieved, the website is unavailable"
try:
logger.write(file="asx_module_log.txt",logmessage=message)
except:
log.critical(message)
def valid(self):
try:
df = self.listed()
return df[(df["industry"]!= "Not Applic") & (df["industry"]!="Class Pend")]
except:
message = "Could not retrieve listed companies object with pandas dataframe"
try:
logfile.write(file="asx_module_log.txt",logmessage=message)
except:
log.critical(message)
def invalid(self):
try:
df = self.listed()
return df[(df["industry"]=="Not Applic") | (df["industry"]=="Class Pend")]
except:
message = "Could not retrieve listed companies object with pandas dataframe"
try:
logfile.write(file="asx_module_log.txt",logmessage=message)
except:
log.critical(message)
my code to create an instance of mylog and pass it to the startup class so that it can log to the textfile.
import mylog
import startup
logger = mylog.logtextfile(name="mylogfile")
c = startup.company(name="mycompany",logobj=logger)
df = c.invalid()
df.head()
I can't test your company class: I don't have most of those 3rd-party modules. However, it's generally a bad idea to have "naked" except clauses. Use named exceptions, otherwise you may be catching things that you don't expect.
Anyway, here's a short demo of using your logger in an instance of another class.
import datetime
class LogTextfile(object):
def __init__(self, name):
self.name = name
def __str__(self):
return "{} ".format(self.__class__.__name__)
def write(self, **kwargs):
""" Writes a log message to a user specified file which
indicates the action takes and if it was successful
"""
self.file = kwargs.get('file', "log.txt")
self.loglevel = kwargs.get('loglevel', "critical")
self.logmessage = kwargs.get('logmessage', "error")
self.success = kwargs.get('success', False)
self.class_name = kwargs.get('class', str("{}".format(self.__class__.__name__)))
self.output = ", ".join([str(datetime.datetime.now().replace(second=0, microsecond=0)),
self.class_name, str(self.logmessage), str(self.success), str("\n")])
for key, value in kwargs.items():
setattr(self, key, value)
f = open(str(self.file), "a")
f.write(self.output)
f.close()
def now(self, filename, openas, data):
"""Creates a log file with todays date and time"""
fmt = '%Y-%m-%d-%H-%M-%S_{fname}'
fn = datetime.datetime.now().strftime(fmt).format(fname=filename)
f = open(str(fn), openas)
f.write(data + "\n")
f.close()
class Test(object):
def __init__(self, logger):
self.logger = logger
def logtest(self, message):
self.logger.write(logmessage=message)
logger = LogTextfile(name="mylogfile")
logger.write(logmessage='This is a test')
t = Test(logger)
t.logtest('Message from Test')
contents of "log.txt"
2017-05-04 22:40:00, LogTextfile, This is a test, False,
2017-05-04 22:40:00, LogTextfile, Message from Test, False,
Simply:
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
class A(object):
def __init__(self, x=None):
self.x = x
class B(object):
def __init__(self, a):
if (isinstance(a, A)):
self.x = a.x
else:
self.x = 0
if __name__ == "__main__":
a = A(10)
b = B(a)
I'm trying to create a user system, which uses a setting and Gui module, and when the GUI module requests for the file to load up using pickle, I keep getting an attribute error. this is from the settings module:
import pickle
import hashlib
class User(object):
def __init__(self, fname, lname, dob, gender):
self.firstname = fname
self.lastname = lname
self._dob = dob
self.gender = gender
self.type = 'General'
self._username = ''
self._hashkey = ''
def Report(self):
print("Full Name: {0} {1}\nDate of Birth: {2}\nGender: {3}\nAccess Level: {4}".format(self.firstname,self.lastname, self._dob, self.gender, self.type))
print(self._username)
def Genusername(self):
self._username = str(str(self._dob)[:2] + self.firstname[:2] + self.lastname[:2])
saveUsers(users)
def Genhashkey(self, password):
encoded = password.encode('utf-8','strict')
return hashlib.sha256(encoded).hexdigest()
def Verifypassword(self, password):
if self._hashkey == self.Genhashkey(password):
return True
else:
return False
class SAdmin(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Stock Admin'
class Manager(User):
def __init__(self, fname, lname, dob, gender):
super().__init__(fname, lname, dob, gender)
self.type = 'Manager'
def saveUsers(users):
with open('user_data.pkl', 'wb') as file:
pickle.dump(users, file, -1) # PICKLE HIGHEST LEVEL PROTOCOL
def loadUsers(users):
try:
with open('user_data.pkl', 'rb') as file:
temp = pickle.load(file)
for item in temp:
users.append(item)
except IOError:
saveUsers([])
def userReport(users):
for user in users:
print(user.firstname, user.lastname)
def addUser(users):
fname = input('What is your First Name?\n > ')
lname = input('What is your Last Name?\n > ')
dob = int(input('Please enter your date of birth in the following format, example 12211996\n> '))
gender = input("What is your gender? 'M' or 'F'\n >")
level = input("Enter the access level given to this user 'G', 'A', 'M'\n > ")
password = input("Enter a password:\n > ")
if level == 'G':
usertype = User
if level == 'A':
usertype = SAdmin
if level == 'M':
usertype = Manager
users.append(usertype(fname, lname, dob, gender))
user = users[len(users)-1]
user.Genusername()
user._hashkey = user.Genhashkey(password)
saveUsers(users)
def deleteUser(users):
userReport(users)
delete = input('Please type in the First Name of the user do you wish to delete:\n > ')
for user in users:
if user.firstname == delete:
users.remove(user)
saveUsers(users)
def changePass(users):
userReport(users)
change = input('Please type in the First Name of the user you wish to change the password for :\n > ')
for user in users:
if user.firstname == change:
oldpass = input('Please type in your old password:\n > ')
newpass = input('Please type in your new password:\n > ')
if user.Verifypassword(oldpass):
user._hashkey = user.Genhashkey(newpass)
saveUsers(users)
else:
print('Your old password does not match!')
def verifyUser(username, password):
for user in users:
if user._username == username and user.Verifypassword(password):
return True
else:
return False
if __name__ == '__main__':
users = []
loadUsers(users)
and this is the GUI module:
from PyQt4 import QtGui, QtCore
import Settings
class loginWindow(QtGui.QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.lbl1 = QtGui.QLabel('Username')
self.lbl2 = QtGui.QLabel('Password')
self.username = QtGui.QLineEdit()
self.password = QtGui.QLineEdit()
self.okButton = QtGui.QPushButton("OK")
self.okButton.clicked.connect(self.tryLogin)
self.cancelButton = QtGui.QPushButton("Cancel")
grid = QtGui.QGridLayout()
grid.setSpacing(10)
grid.addWidget(self.lbl1, 1, 0)
grid.addWidget(self.username, 1, 1)
grid.addWidget(self.lbl2, 2, 0)
grid.addWidget(self.password, 2, 1)
grid.addWidget(self.okButton, 3, 1)
grid.addWidget(self.cancelButton, 3, 0)
self.setLayout(grid)
self.setGeometry(300, 300, 2950, 150)
self.setWindowTitle('Login')
self.show()
def tryLogin(self):
print(self.username.text(), self.password.text())
if Settings.verifyUser(self.username.text(),self.password.text()):
print('it Woks')
else:
QtGui.QMessageBox.warning(
self, 'Error', 'Incorrect Username or Password')
class Window(QtGui.QMainWindow):
def __init__(self):
super().__init__()
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
users = []
Settings.loadUsers(users)
if loginWindow().exec_() == QtGui.QDialog.Accepted:
window = Window()
window.show()
sys.exit(app.exec_())
each user is a class and are put into a list and then the list is saved using pickle when I load up just the settings file and verify the login everything works fine but when I open up the GUI module and try to verify it doesn't let me, the error I'm getting:
Traceback (most recent call last):
File "C:\Users`Program\LoginGUI.py", line 53, in <module>
Settings.loadUsers(users)
File "C:\Users\Program\Settings.py", line 51, in loadUsers
temp = pickle.load(file)
AttributeError: Can't get attribute 'Manager' on <module '__main__' (built-in)>
The issue is that you're pickling objects defined in Settings by actually running the 'Settings' module, then you're trying to unpickle the objects from the GUI module.
Remember that pickle doesn't actually store information about how a class/object is constructed, and needs access to the class when unpickling. See wiki on using Pickle for more details.
In the pkl data, you see that the object being referenced is __main__.Manager, as the 'Settings' module was main when you created the pickle file (i.e. you ran the 'Settings' module as the main script to invoke the addUser function).
Then, you try unpickling in 'Gui' - so that module has the name __main__, and you're importing Setting within that module. So of course the Manager class will actually be Settings.Manager. But the pkl file doesn't know this, and looks for the Manager class within __main__, and throws an AttributeError because it doesn't exist (Settings.Manager does, but __main__.Manager doesn't).
Here's a minimal code set to demonstrate.
The class_def.py module:
import pickle
class Foo(object):
def __init__(self, name):
self.name = name
def main():
foo = Foo('a')
with open('test_data.pkl', 'wb') as f:
pickle.dump([foo], f, -1)
if __name__=='__main__':
main()
You run the above to generate the pickle data.
The main_module.py module:
import pickle
import class_def
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
You run the above to attempt to open the pickle file, and this throws roughly the same error that you were seeing. (Slightly different, but I'm guessing that's because I'm on Python 2.7)
The solution is either:
You make the class available within the namespace of the top-level module (i.e. GUI or main_module) through an explicit import, or
You create the pickle file from the same top-level module as the one that you will open it in (i.e. call Settings.addUser from GUI, or class_def.main from main_module). This means that the pkl file will save the objects as Settings.Manager or class_def.Foo, which can then be found in the GUI`main_module` namespace.
Option 1 example:
import pickle
import class_def
from class_def import Foo # Import Foo into main_module's namespace explicitly
if __name__=='__main__':
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Option 2 example:
import pickle
import class_def
if __name__=='__main__':
class_def.main() # Objects are being pickled with main_module as the top-level
with open('test_data.pkl', 'rb') as f:
users = pickle.load(f)
Please first read the answer mentioned by zehnpaard to know the reason for the attribute error. Other than the solution he already provided, in python3 you can use the pickle.Unpickler class and override the find_class method as mentioned below:
import pickle
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
if name == 'Manager':
from settings import Manager
return Manager
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
If you're still getting this error even after importing the appropriate classes in the loading module (zehnpaard's solution #1), then the find_class function of pickle.Unpickler can be overwritten and explicitly directed to look in the current module's namespace.
import pickle
from settings import Manager
class CustomUnpickler(pickle.Unpickler):
def find_class(self, module, name):
try:
return super().find_class(__name__, name)
except AttributeError:
return super().find_class(module, name)
pickle_data = CustomUnpickler(open('file_path.pkl', 'rb')).load()
## No exception trying to get 'Manager'
Note: This method loses the relative-import path information stored in module. So, be careful of namespace collisions in your pickled classes.
If you have a class defined outside the module, whose object is in pickle data,
you have to import the class
from outside_module import DefinedClass1, DefinedClass2, DefinedClass3
with open('pickle_file.pkl', 'rb') as f:
pickle_data = pickle.load(f)
if you use dill dump/load model will work
import dill
from sklearn.preprocessing import FunctionTransformer
sp_clf = FunctionTransformer(lambda X:X.astype('float').fillna(0).applymap(abs))
with open('temp.joblib','wb') as io:
dill.dump(sp_clf,io)
with open('temp.joblib','rb') as io:
dd=dill.load(io)