With Python and JsonPickle, How do I serialize the object with a Certain Casing, eg Camel Case, Pascal, etc? The following answer below does it manually, however looking for a specific Jsonpickle solution, since it can handle complex object types .
JSON serialize a class and change property casing with Python
https://stackoverflow.com/a/8614096/15435022
class HardwareSystem:
def __init__(self, vm_size):
self.vm_size = vm_size
self.some_other_thing = 42
self.a = 'a'
def snake_to_camel(s):
a = s.split('_')
a[0] = a[0].lower()
if len(a) > 1:
a[1:] = [u.title() for u in a[1:]]
return ''.join(a)
def serialise(obj):
return {snake_to_camel(k): v for k, v in obj.__dict__.items()}
hp = HardwareSystem('Large')
print(json.dumps(serialise(hp), indent=4, default=serialise))
Here's my attempt.
from importlib import import_module
import inspect
import json
import jsonpickle
import re
def snake_to_camel(s):
a = s.split('_')
a[0] = a[0].lower()
if len(a) > 1:
a[1:] = [u.title() for u in a[1:]]
return ''.join(a)
def camel_to_snake(s):
snake = []
snake_len = len(s)
for idx, char in enumerate(s):
snake.append(char.lower())
if idx < snake_len - 1:
if char.islower() and s[idx+1].isupper():
snake.append('_')
return ''.join(snake)
def debug_output(obj):
output = '{}({})'
attrs = [attr + '=' + repr(getattr(obj, attr)) for attr in vars(obj)]
return output.format(obj.__class__.__name__, ', '.join(attrs))
class SoftwareSystem:
def __init__(self):
self.software_rating = 'Awesome!'
# Making debug output friendly
def __repr__(self):
return debug_output(self)
class HardwareSystem:
def __init__(self, vm_size):
self.vm_size = vm_size
self.some_other_thing = 42
self.a = 'a'
# Making debug output friendly
def __repr__(self):
return debug_output(self)
#jsonpickle.handlers.register(HardwareSystem, base=True)
#jsonpickle.handlers.register(SoftwareSystem, base=True)
class SystemHandler(jsonpickle.handlers.BaseHandler):
def flatten(self, obj, data):
for k, v in obj.__dict__.items():
data[snake_to_camel(k)] = jsonpickle.encode(v)
return data
def restore(self, obj):
# Gets reference to class
# https://stackoverflow.com/a/55559852/152016
module_path, class_name = obj['py/object'].rsplit('.', 1)
module = import_module(module_path)
class_ = getattr(module, class_name)
# Dealing with __init__ params (except first)
params = inspect.getargs(class_.__init__.__code__)
params = params.args[1:]
# Preparing dict keys
l_obj = {}
for k, v in obj.items():
l_obj[camel_to_snake(k)] = v
# Instantiating constructor params
data = {}
for k, v in l_obj.items():
if k in params:
data[k] = v
result = class_(**data)
# Setting other jsonpickled object attributes
for k, v in l_obj.items():
if not k in params:
setattr(result, k, v)
return result
hw = HardwareSystem(100)
sw = SoftwareSystem()
hw.software_instance = sw
json_str = jsonpickle.encode(hw)
print(json_str)
decoded = jsonpickle.decode(json_str)
print(hw)
This has some assumptions:
Following your original snake_to_camel function, I've put up a camel_to_snake on decoding, that assumes only the first uppercase letter after a lowercase letter will have prepended the _ char (so awesomeABC will translate to awesome_abc, and therefore if you translate it back again it will be incorrectly awesomeAbc)
The above code encodes/decodes properties added after __init__ (see for example hw.software_instance above).
You can nest objects. I've only tried a single object nested.
I've added auxiliary debug_output/__repr__ functions, you may throw these away (or customize :))
This is a follow up question to this post:
How to recursively create a nested dictionary of unknown size in python?
I need to somehow incorporate a value for each element as well.
Starting with the previous solution:
def unpack(obj):
return {str(o): unpack(o) for o in obj.get_items()}
Every item also has a value associated with it besides the name.
I tried the following, but it wouldn't work:
def unpack(obj):
return {{str(o): unpack(o), 'value':o.value} for o in obj.get_items()}
I get a "unhashable type: 'dict'" error.
The resulting dictionary should look something like the following:
example_dict = {'level_1': {'level_2':{},'value':'b'},'value' : 'a'}
Where each nested level would have a value associated with the name of the level.
Ok, so if I understood you correctly you want to put an object tree into a hierarchical dictionary. My first go at this would be
import random
names = ['foo_%s'%i for i in range(10)]
# Make hierarchical test case
class Test:
def __init__(self, depth):
self.name = random.choice(names)
if depth > 1:
self.child = Test(depth-1)
else:
self.child = None
def __str__(self):
return self.name
test = Test(3)
# The meat
def unpack(x):
ret = dict()
for k, v in x.__dict__.items():
try:
ret[k] = unpack(v)
except AttributeError:
ret[k] = v
return ret
print(unpack(test))
‚unpack‘ tries to recursively find all variables of an object and writes them into a dict. If that fails it returns the variable. Note that ‚x.str‘ has to be implemented.
I am creating an list of custom objects from a database. The custom object class and the list creation is shown below. How can I serialize the list holding this custom data?
class MyCustomObject():
"""For displaying the records"""
def __init__(self):
self.rec_id = ""
self.place = ""
rec_list = [] #The List
# Creating a list of MyCustomObject's from rows
for col in rows:
rec = MyCustomObject()
rec.rec_id = col[0]
rec.place = col[1]
rec_list.append(recently_viewed)
How can I serialize this list of MyCustomObjects to JSON.
you could try writing your own serializer as below:
import json
class MyCustomObject():
"""For displaying the records"""
def __init__(self):
self.rec_id = ""
self.place = ""
class mySerializer(json.JSONEncoder):
def default(self, obj):
return obj.__dict__
rec_list = [] #The List
# Creating a list of MyCustomObject's from rows
rows = [[1,2],[3,4]]
for col in rows:
rec = MyCustomObject()
rec.rec_id = col[0]
rec.place = col[1]
rec_list.append(rec)
print [json.dumps(r, cls=mySerializer) for r in rec_list]
output:
['{"place": 2, "rec_id": 1}', '{"place": 4, "rec_id": 3}']
By extending JsonEncoder (https://docs.python.org/2/library/json.html#json.JSONEncoder)
So you will get something like;
import json
class MyJsonEncoder
def default(self, obj):
if isinstance(obj, MyCustomObject):
return {} # dict representation of your object
return super(MyJsonEncoder, self).dumps(obj)
json.dumps(rec_list, encoder=MyJsonEncoder)
I think I am missing something small here. I am testing out Python framework Flask and Flask-MongoAlchemy, and want to convert an entity into JSON output. Here is my code (abstracted):
from flask import Flask
from flaskext.mongoalchemy import MongoAlchemy
try:
from bson.objectid import ObjectId
except:
pass
#a bunch of code to open the mongoDB
class ClassA(db.Document):
title = db.StringField()
field1 = db.StringField()
field2 = db.BoolField()
#app.route('/api/classA', methods=['GET'])
def api_list_all
a = ClassA.query.all()
result = []
for b in a:
result.append(b.wrap())
print result
return json.dumps(result)
Without the json.dumps line, the print statement prompt the right result. But only if I run the json.dumps on result, it yields:
TypeError: ObjectId('...') is not JSON serializable
What am I missing?
The result is a mongo document of some sort that contains ObjectId-type content, which you'll have to tell json how to deserialize. You'll have the same problem with other mongo-specific types, such as ReferenceField(), EmbeddedDocumentField(), etc.
You have to write a deserialization function that you can pass to json. What I use is:
def encode_model(obj, recursive=False):
if obj is None:
return obj
if isinstance(obj, (mongoengine.Document, mongoengine.EmbeddedDocument)):
out = dict(obj._data)
for k,v in out.items():
if isinstance(v, ObjectId):
if k is None:
out['_id'] = str(v)
del(out[k])
else:
# Unlikely that we'll hit this since ObjectId is always NULL key
out[k] = str(v)
else:
out[k] = encode_model(v)
elif isinstance(obj, mongoengine.queryset.QuerySet):
out = encode_model(list(obj))
elif isinstance(obj, ModuleType):
out = None
elif isinstance(obj, groupby):
out = [ (g,list(l)) for g,l in obj ]
elif isinstance(obj, (list)):
out = [encode_model(item) for item in obj]
elif isinstance(obj, (dict)):
out = dict([(k,encode_model(v)) for (k,v) in obj.items()])
elif isinstance(obj, datetime.datetime):
out = str(obj)
elif isinstance(obj, ObjectId):
out = {'ObjectId':str(obj)}
elif isinstance(obj, (str, unicode)):
out = obj
elif isinstance(obj, float):
out = str(obj)
else:
raise TypeError, "Could not JSON-encode type '%s': %s" % (type(obj), str(obj))
return out
Then you'd process the result as:
return json.dumps(result, default=encode_model)
or something to this effect.
You can also use the query.raw_output() method to have that query instance return raw Python dictionaries instead of a Python object. With a dictionary it becomes easy to encode to JSON using json.dumps():
import json
q=db.query(MyObject)
q.raw_output()
json.dumps(q.first())
Reference http://www.mongoalchemy.org/api/expressions/query.html#mongoalchemy.query.Query.raw_output
Combining the previous two answers, you should be able to do something like this:
from bson import json_util
# ...
#app.route('/api/classA', methods=['GET'])
def api_list_all
a = ClassA.query.all()
result = []
for b in a:
result.append(b.wrap())
print result
return json_utils.dumps(result) # Change here.
Ok I'm working on getting better with python, so I'm not sure this is the right way to go about what I'm doing to begin with, but here's my current problem...
I need to get some information via a SOAP method, and only use part of the information now but store the entire result for future uses (we need to use the service as little as possible). Looking up the best way to access the service I figured suds was the way to go, and it was simple and worked like a charm to get the data. But now I want to save the result somehow, preferably serialized / in a database so I can pull it out later and use it the same.
What's the best way to do this, it looks like pickle/json isn't an option? Thanks!
Update
Reading the top answer at How can I pickle suds results? gives me a better idea of why this isn't an option, I guess I'm stuck recreating a basic object w/ the information I need?
I have been using following approach to convert Suds object into JSON:
from suds.sudsobject import asdict
def recursive_asdict(d):
"""Convert Suds object into serializable format."""
out = {}
for k, v in asdict(d).items():
if hasattr(v, '__keylist__'):
out[k] = recursive_asdict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(recursive_asdict(item))
else:
out[k].append(item)
else:
out[k] = v
return out
def suds_to_json(data):
return json.dumps(recursive_asdict(data))
Yep, I confirm the explanation I gave in the answer you refer to -- dynamically generated classes are not easily picklable (nor otherwise easily serializable), you need to extract all the state information, pickle that state, and reconstruct the tricky sudsobject on retrieval if you really insist on using it;-).
Here is what I came up with before researching and finding this answer. This actually works well for me on complex suds responses and also on other objects such as __builtins__ since the solution is suds agnostic:
import datetime
def object_to_dict(obj):
if isinstance(obj, (str, unicode, bool, int, long, float, datetime.datetime, datetime.date, datetime.time)):
return obj
data_dict = {}
try:
all_keys = obj.__dict__.keys() # vars(obj).keys()
except AttributeError:
return obj
fields = [k for k in all_keys if not k.startswith('_')]
for field in fields:
val = getattr(obj, field)
if isinstance(val, (list, tuple)):
data_dict[field] = []
for item in val:
data_dict[field].append(object_to_dict(item))
else:
data_dict[field] = object_to_dict(val)
return data_dict
This solution works and is actually faster. It also works on objects that don't have the __keylist__ attribute.
I ran a benchmark 100 times on a complex suds output object, this solutions run time was 0.04 to .052 seconds (0.045724287 average). While recursive_asdict solution above ran in .082 to 0.102 seconds so nearly double (0.0829765582 average).
I then went back to the drawing board and re-did the function to get more performance out of it, and it does not need the datetime import. I leveraged in using the __keylist__ attribute, so this will not work on other objects such as __builtins__ but works nicely for suds object output:
def fastest_object_to_dict(obj):
if not hasattr(obj, '__keylist__'):
return obj
data = {}
fields = obj.__keylist__
for field in fields:
val = getattr(obj, field)
if isinstance(val, list): # tuple not used
data[field] = []
for item in val:
data[field].append(fastest_object_to_dict(item))
else:
data[field] = fastest_object_to_dict(val)
return data
The run time was 0.18 - 0.033 seconds (0.0260889721 average), so nearly 4x as faster than the recursive_asdict solution.
I made an implementation of a dummy class for Object intance of suds, and then being able to serialize. The FakeSudsInstance behaves like an original Suds Object instance, see below:
from suds.sudsobject import Object as SudsObject
class FakeSudsNode(SudsObject):
def __init__(self, data):
SudsObject.__init__(self)
self.__keylist__ = data.keys()
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, FakeSudsNode(value))
elif isinstance(value, list):
l = []
for v in value:
if isinstance(v, list) or isinstance(v, dict):
l.append(FakeSudsNode(v))
else:
l.append(v)
setattr(self, key, l)
else:
setattr(self, key, value)
class FakeSudsInstance(SudsObject):
def __init__(self, data):
SudsObject.__init__(self)
self.__keylist__ = data.keys()
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, FakeSudsNode(value))
else:
setattr(self, key, value)
#classmethod
def build_instance(cls, instance):
suds_data = {}
def node_to_dict(node, node_data):
if hasattr(node, '__keylist__'):
keys = node.__keylist__
for key in keys:
if isinstance(node[key], list):
lkey = key.replace('[]', '')
node_data[lkey] = node_to_dict(node[key], [])
elif hasattr(node[key], '__keylist__'):
node_data[key] = node_to_dict(node[key], {})
else:
if isinstance(node_data, list):
node_data.append(node[key])
else:
node_data[key] = node[key]
return node_data
else:
if isinstance(node, list):
for lnode in node:
node_data.append(node_to_dict(lnode, {}))
return node_data
else:
return node
node_to_dict(instance, suds_data)
return cls(suds_data)
Now, after a suds call, for example below:
# Now, after a suds call, for example below
>>> import cPickle as pickle
>>> suds_intance = client.service.SomeCall(account, param)
>>> fake_suds = FakeSudsInstance.build_instance(suds_intance)
>>> dumped = pickle.dumps(fake_suds)
>>> loaded = pickle.loads(dumped)
I hope it helps.
The solutions suggesed above lose valuable information about class names - it can be of value in some libraries like DFP client https://github.com/googleads/googleads-python-lib where entity types might be encoded in dynamically generated class names (i.e. TemplateCreative/ImageCreative)
Here's the solution I used that preserves class names and restores dict-serialized objects without data loss (except suds.sax.text.Text which would be converted into regular unicode objects and maybe some other types I haven't run into)
from suds.sudsobject import asdict, Factory as SudsFactory
def suds2dict(d):
"""
Suds object serializer
Borrowed from https://stackoverflow.com/questions/2412486/serializing-a-suds-object-in-python/15678861#15678861
"""
out = {'__class__': d.__class__.__name__}
for k, v in asdict(d).iteritems():
if hasattr(v, '__keylist__'):
out[k] = suds2dict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(suds2dict(item))
else:
out[k].append(item)
else:
out[k] = v
return out
def dict2suds(d):
"""
Suds object deserializer
"""
out = {}
for k, v in d.iteritems():
if isinstance(v, dict):
out[k] = dict2suds(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if isinstance(item, dict):
out[k].append(dict2suds(item))
else:
out[k].append(item)
else:
out[k] = v
return SudsFactory.object(out.pop('__class__'), out)
I updated the recursive_asdict example above to be compatible with python3 (items instead of iteritems).
from suds.sudsobject import asdict
from suds.sax.text import Text
def recursive_asdict(d):
"""
Recursively convert Suds object into dict.
We convert the keys to lowercase, and convert sax.Text
instances to Unicode.
Taken from:
https://stackoverflow.com/a/15678861/202168
Let's create a suds object from scratch with some lists and stuff
>>> from suds.sudsobject import Object as SudsObject
>>> sudsobject = SudsObject()
>>> sudsobject.Title = "My title"
>>> sudsobject.JustAList = [1, 2, 3]
>>> sudsobject.Child = SudsObject()
>>> sudsobject.Child.Title = "Child title"
>>> sudsobject.Child.AnotherList = ["4", "5", "6"]
>>> childobject = SudsObject()
>>> childobject.Title = "Another child title"
>>> sudsobject.Child.SudObjectList = [childobject]
Now see if this works:
>>> result = recursive_asdict(sudsobject)
>>> result['title']
'My title'
>>> result['child']['anotherlist']
['4', '5', '6']
"""
out = {}
for k, v in asdict(d).items():
k = k.lower()
if hasattr(v, '__keylist__'):
out[k] = recursive_asdict(v)
elif isinstance(v, list):
out[k] = []
for item in v:
if hasattr(item, '__keylist__'):
out[k].append(recursive_asdict(item))
else:
out[k].append(
item.title() if isinstance(item, Text) else item)
else:
out[k] = v.title() if isinstance(v, Text) else v
return out
I like this way. We don't do the iteration ourselves, it is python that iterates when converting it to string
class Ob:
def __init__(self, J) -> None:
self.J = J
def __str__(self):
if hasattr(self.J, "__keylist__"):
self.J = {key: Ob(value) for key, value in dict(self.J).items()}
if hasattr(self.J, "append"):
self.J = [Ob(data) for data in sefl.J]
return str(self.J)
result = Ob(result_soap)