How do I serialise a Python Enum member to JSON, so that I can deserialise the resulting JSON back into a Python object?
For example, this code:
from enum import Enum
import json
class Status(Enum):
success = 0
json.dumps(Status.success)
results in the error:
TypeError: <Status.success: 0> is not JSON serializable
How can I avoid that?
I know this is old but I feel this will help people. I just went through this exact problem and discovered if you're using string enums, declaring your enums as a subclass of str works well for almost all situations:
import json
from enum import Enum
class LogLevel(str, Enum):
DEBUG = 'DEBUG'
INFO = 'INFO'
print(LogLevel.DEBUG)
print(json.dumps(LogLevel.DEBUG))
print(json.loads('"DEBUG"'))
print(LogLevel('DEBUG'))
Will output:
LogLevel.DEBUG
"DEBUG"
DEBUG
LogLevel.DEBUG
As you can see, loading the JSON outputs the string DEBUG but it is easily castable back into a LogLevel object. A good option if you don't want to create a custom JSONEncoder.
The correct answer depends on what you intend to do with the serialized version.
If you are going to unserialize back into Python, see Zero's answer.
If your serialized version is going to another language then you probably want to use an IntEnum instead, which is automatically serialized as the corresponding integer:
from enum import IntEnum
import json
class Status(IntEnum):
success = 0
failure = 1
json.dumps(Status.success)
and this returns:
'0'
If you want to encode an arbitrary enum.Enum member to JSON and then decode
it as the same enum member (rather than simply the enum member's value attribute), you can do so by writing a custom JSONEncoder class, and a decoding function to pass as the object_hook argument to json.load() or json.loads():
PUBLIC_ENUMS = {
'Status': Status,
# ...
}
class EnumEncoder(json.JSONEncoder):
def default(self, obj):
if type(obj) in PUBLIC_ENUMS.values():
return {"__enum__": str(obj)}
return json.JSONEncoder.default(self, obj)
def as_enum(d):
if "__enum__" in d:
name, member = d["__enum__"].split(".")
return getattr(PUBLIC_ENUMS[name], member)
else:
return d
The as_enum function relies on the JSON having been encoded using EnumEncoder, or something which behaves identically to it.
The restriction to members of PUBLIC_ENUMS is necessary to avoid a maliciously crafted text being used to, for example, trick calling code into saving private information (e.g. a secret key used by the application) to an unrelated database field, from where it could then be exposed (see https://chat.stackoverflow.com/transcript/message/35999686#35999686).
Example usage:
>>> data = {
... "action": "frobnicate",
... "status": Status.success
... }
>>> text = json.dumps(data, cls=EnumEncoder)
>>> text
'{"status": {"__enum__": "Status.success"}, "action": "frobnicate"}'
>>> json.loads(text, object_hook=as_enum)
{'status': <Status.success: 0>, 'action': 'frobnicate'}
In Python >= 3.7, can just use
json.dumps(enum_obj, default=str)
If you want to use the enum value, you can do
json.dumps(enum_obj, default=lambda x: x.value)
or if you want to use the enum name,
json.dumps(enum_obj, default=lambda x: x.name)
You just need to inherit from str or int class:
from enum import Enum, unique
#unique
class StatusEnum(int, Enum):
pending: int = 11
approved: int = 15
declined: int = 266
That's it, it will be serialised using any JSON encoder.
I liked Zero Piraeus' answer, but modified it slightly for working with the API for Amazon Web Services (AWS) known as Boto.
class EnumEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Enum):
return obj.name
return json.JSONEncoder.default(self, obj)
I then added this method to my data model:
def ToJson(self) -> str:
return json.dumps(self.__dict__, cls=EnumEncoder, indent=1, sort_keys=True)
I hope this helps someone.
If you are using jsonpickle the easiest way should look as below.
from enum import Enum
import jsonpickle
#jsonpickle.handlers.register(Enum, base=True)
class EnumHandler(jsonpickle.handlers.BaseHandler):
def flatten(self, obj, data):
return obj.value # Convert to json friendly format
if __name__ == '__main__':
class Status(Enum):
success = 0
error = 1
class SimpleClass:
pass
simple_class = SimpleClass()
simple_class.status = Status.success
json = jsonpickle.encode(simple_class, unpicklable=False)
print(json)
After Json serialization you will have as expected {"status": 0} instead of
{"status": {"__objclass__": {"py/type": "__main__.Status"}, "_name_": "success", "_value_": 0}}
You can even combine the solutions mentioned above with the automatic value creation for Enums. I use this in combination with Pydantic and FastAPI to provide lower case names for a REST API:
from enum import Enum, auto
import json
class StrEnum(str, Enum):
pass
# this creates nice lowercase and JSON serializable names
# https://docs.python.org/3/library/enum.html#using-automatic-values
class AutoNameLower(StrEnum):
def _generate_next_value_(name, start, count, last_values):
return name.lower()
class AutoNameLowerStrEnum(AutoNameLower):
pass
class MyActualEnum(AutoNameLowerStrEnum):
THIS = auto()
THAT = auto()
FOO = auto()
BAR = auto()
print(MyActualEnum.THIS)
print(json.dumps(MyActualEnum.THIS))
print(list(MyActualEnum))
Console:
>>> MyActualEnum.THIS
>>> "this"
>>> [<MyActualEnum.THIS: 'this'>, <MyActualEnum.THAT: 'that'>, <MyActualEnum.FOO: 'foo'>, <MyActualEnum.BAR: 'bar'>]
This worked for me:
class Status(Enum):
success = 0
def __json__(self):
return self.value
Didn't have to change anything else. Obviously, you'll only get the value out of this and will need to do some other work if you want to convert the serialized value back into the enum later.
Related
I have a JSON object that reads:
j = {"id": 1, "label": "x"}
I have two types:
class BaseModel:
def __init__(self, uuid):
self.uuid = uuid
class Entity(BaseModel):
def __init__(self, id, label):
super().__init__(id)
self.name = name
Note how id is stored as uuid in the BaseModel.
I can load Entity from the JSON object as:
entity = Entity(**j)
I want to re-write my model leveraging dataclass:
#dataclass
class BaseModel:
uuid = str
#dataclass
class Entity:
name = str
Since my JSON object does not have the uuid, entity = Entitye(**j) on the dataclass-based model will throw the following error:
TypeError: __init__() got an unexpected keyword argument 'id'
The "ugly" solutions I can think of:
Rename id to uuid in JSON before initialization:
j["uuid"] = j.pop("id")
Define both id and uuid:
#dataclass
class BaseModel:
uuid = str
#dataclass
class Entity:
id = str
name = str
# either use:
uuid = id
# or use this method
def __post_init__(self):
super().uuid = id
Is there any cleaner solution for this kind of object initialization in the dataclass realm?
might be ruining the idea of removing the original __init__ but how about writing a function to initialize the data class?
def init_entity(j):
j["uuid"] = j.pop("id")
return Entity(**j)
and in your code entity = initEntity(j)
I think the answer here might be to define a classmethod that acts as an alternative constructor to the dataclass.
from dataclasses import dataclass
from typing import TypeVar, Any
#dataclass
class BaseModel:
uuid: str
E = TypeVar('E', bound='Entity')
#dataclass
class Entity(BaseModel):
name: str
#classmethod
def from_json(cls: type[E], **kwargs: Any) -> E:
return cls(kwargs['id'], kwargs['label']
(For the from_json type annotation, you'll need to use typing.Type[E] instead of type[E] if you're on python <= 3.8.)
Note that you need to use colons for your type-annotations within the main body of a dataclass, rather than the = operator, as you were doing.
Example usage in the interactive REPL:
>>> my_json_dict = {'id': 1, 'label': 'x'}
>>> Entity.from_json(**my_json_dict)
Entity(uuid=1, name='x')
It's again questionable how much boilerplate code this saves, however. If you find yourself doing this much work to replicate the behaviour of a non-dataclass class, it's often better just to use a non-dataclass class. Dataclasses are not the perfect solution to every problem, nor do they try to be.
Simplest solution seems to be to use an efficient JSON serialization library that supports key remappings. There are actually tons of them that support this, but dataclass-wizard is one example of a (newer) library that supports this particular use case.
Here's an approach using an alias to dataclasses.field() which should be IDE friendly enough:
from dataclasses import dataclass
from dataclass_wizard import json_field, fromdict, asdict
#dataclass
class BaseModel:
uuid: int = json_field('id', all=True)
#dataclass
class Entity(BaseModel):
name: str = json_field('label', all=True)
j = {"id": 1, "label": "x"}
# De-serialize the dictionary object into an `Entity` instance.
e = fromdict(Entity, j)
repr(e)
# Entity(uuid=1, name='x')
# Assert we get the same object when serializing the instance back to a
# JSON-serializable dict.
assert asdict(e) == j
Lambda execution failed with status 200 due to customer function error: Object of type 'Decimal' is not JSON serializable
I went through all the existing solutions in the following link but nothing worked for me. What am I doing wrong?:
Python JSON serialize a Decimal object
import json
import boto3
import decimal
client = boto3.resource('dynamodb')
table = client.Table('table')
def lambda_handler(event, context):
method = event["httpMethod"]
print(event)
if method=="POST":
return POST(event)
elif method=="DELETE":
return DELETE(event)
elif method=="GET":
return GET(event)
#the respons format
def send_respons(responseBody, statusCode):
response = {
"statusCode": statusCode,
"headers": {
"my_header": "my_value"
},
"body": json.dumps(responseBody),
"isBase64Encoded": 'false'
}
return response
def GET(event):
tab = table.scan()['Items']
ids = []
for item in tab:
ids.append({"id":item["id"], "decimalOBJ":decimal.Decimal(item["decimalOBJ"]}))
return send_respons(ids, 201)
Here is an example of extending the JSONEncoder to handle Decimal type also specified in the json docs
from decimal import Decimal
class DecimalEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Decimal):
return str(obj)
return json.JSONEncoder.default(self, obj)
Call it using
json.dumps(some_object, cls=DecimalEncoder)
By converting to a str a good degree of precision will be maintained, without relying on external packages.
It seems you have two options:
Probably easiest, you can serialize the int/float value of a Decimal object:
""" assume d is your decimal object """
serializable_d = int(d) # or float(d)
d_json = json.dumps(d)
You can add simplejson to your requirements.txt, which now has support for serializing Decimals. It's a drop-in replacement for the included json module.
import simplejson as json # instead of import json
The rest of your code will work the same. If you need further assistance, kindly leave a comment.
Create a function to handle the TypeError and use default argument of json.dumps to set this. This avoid TypeError: Object of type Decimal is not JSON serializable.
https://docs.python.org/3/library/json.html
If specified, default should be a function that gets called for
objects that can’t otherwise be serialized. It should return a JSON
encodable version of the object or raise a TypeError. If not
specified, TypeError is raised.
json_utils.py:
import decimal
import json
def dumps(item: dict) -> str:
return json.dumps(item, default=default_type_error_handler)
def default_type_error_handler(obj):
if isinstance(obj, decimal.Decimal):
return int(obj)
raise TypeError
test_json_utils.py:
import json
from decimal import Decimal
from commons import json_utils
def test_different_data_types():
# Prepare data
item = {
"a-string": "lorem",
"a-boolean": True,
"a-number": 4711,
"a-decimal-object": Decimal(4711) # Used by dynamoDb boto3 client
}
# Execute
item_as_json = json_utils.dumps(item)
# Assert
item_back_to_dict = json.loads(item_as_json)
assert item_back_to_dict == item
i am trying convert json string to model
then it is easy to get value with .
i have checked another question
but different, my json sting looks like,
{
"id":"123",
"name":"name",
"key":{
"id":"345",
"des":"des"
},
}
i prefer to use 2 class like,
class A:
id = ''
name = ''
key = new B()
class B:
id = ''
des = ''
There are few libraries that might help:
marshmallow is nice
colander from Pylons
schematics
For easier cases you can also use something from standard library like
named tuples and one from collections which is available also in py2
SimpleNamespace
In order to do that you should provide your custom callback as an object_hook argument to the json.loads function.
object_hook is an optional function that will be called with the
result of any object literal decode (a dict). The return value of
object_hook will be used instead of the dict. This feature
can be used to implement custom decoders (e.g. JSON-RPC class hinting).
Consider using collections.namestuple subclasses:
json_str = '''
{
"id":"123",
"name":"name",
"key":{
"id":"345",
"des":"des"
}
}'''
B = collections.namedtuple('B', 'id des')
A = collections.namedtuple('A', 'id name key')
def make_models(o):
if 'key' in o:
return A(o['id'], o['name'], B(id=o['key']['id'], des=o['key']['des']))
else:
return o
result = json.loads(json_str, object_hook=make_models)
print(type(result)) # outputs: <class '__main__.A'>
print(result.id) # outputs: 123
print(result.key.id) # outputs: 345
My problem could be summarised by the following example:
from enum import Enum
import json
class FooBarType(Enum):
standard = 0
foo = 1
bar = 2
dict = {'name': 'test', 'value': 'test', 'type': FooBarType.foo}
json.dumps(dict)
TypeError: <FooBarType.foo: 1> is not JSON serializable
I get a type error, because enums are not JSON serializable.
I primarily though of implementing a JsonEncoder and adding it to the json.dumps() call but I cannot change the line where json.dumps() call is made.
So, my question is :
Is it possible to dump an enum in json without passing an encoder to json.dumps(), but instead, by adding class method(s) in FooBarType enum ?
I expect to extract the following json:
{'name': 'test', 'value': 'test', 'type': 'foo'}
or
{'name': 'test', 'value': 'test', 'type': 1}
Try:
from enum import Enum
# class StrEnum(str, Enum):
# """Enum where members are also (and must be) strs"""
class Color(str, Enum):
RED = 'red'
GREEN = 'green'
BLUE = 'blue'
data = [
{
'name': 'car',
'color': Color.RED,
},
{
'name': 'dog',
'color': Color.BLUE,
},
]
import json
print(json.dumps(data))
Result:
[
{
"name": "car",
"color": "red"
},
{
"name": "dog",
"color": "blue"
}
]
Sadly, there is no direct support for Enum in JSON.
The closest automatic support is to use IntEnum (which enum34 also supports), and then json will treat your enums as ints; of course, decoding them will give you an int back, but that is as good it gets without specifying your encoder/decoder.
Just adding method(s) to the FooBarType enum won't do what you want.
As I mentioned in my comment, you can however use part of my answer to the question Making object JSON serializable with regular encoder to monkey-patch the json module so it will return the name (or value) of Enum members. I'm assuming you're using the enums34 module by Ethan Furman et al, which was backported to Python 2.7 since that version doesn't come with it built-in — it became part of the standard library in Python 3.4.
Note this will work even though you can't change the line where the json.dumps() call occurs as long as that happens after the patch is applied. This is because Python normally caches imported modules in sys.modules, i.e. they aren't reloaded everytime they are used in separate scripts — so any changes made this to them are "sticky" and remain in effect.
So for what you want to do, first create your own module to make the patch. For example: make_enum_json_serializable.py.
""" Module that monkey-patches the json module when it's imported so
JSONEncoder.default() automatically checks to see if the object being encoded
is an instance of an Enum type and, if so, returns its name.
"""
from enum import Enum
from json import JSONEncoder
_saved_default = JSONEncoder().default # Save default method.
def _new_default(self, obj):
if isinstance(obj, Enum):
return obj.name # Could also be obj.value
else:
return _saved_default
JSONEncoder.default = _new_default # Set new default method.
Then, in your own script, all you need to do is essentially add one line:
from enum import Enum
import json
import make_enum_json_serializable # ADDED
class FooBarType(Enum):
standard = 0
foo = 1
bar = 2
a_dict = {'name': 'spam', 'value': 42, 'type': FooBarType.foo}
print(json.dumps(a_dict))
Output:
{"type": "foo", "name": "spam", "value": 42}
UPDATE: Please read the answer from #gil9red, I think it's better than mine!
I don't think there is a great way for this and you will lose features of the Enum.
Simplest option: Don't subclass Enum:
class FooBarType:
standard = 0
foo = 1
bar = 2
dict = {'type': FooBarType.foo}
json.dumps(dict)
What you could also do:
class EnumIntValue(int):
def __new__(cls, name, value):
c = int.__new__(cls, int(value))
c.name = name
return c
def __repr__(self):
return self.name
def __str__(self):
return self.name
class FooBarType:
standard = EnumIntValue('standard',0)
foo = EnumIntValue('foo',0)
bar = EnumIntValue('bar',2)
dict = {'type': FooBarType.foo}
json.dumps(dict)
This will actually give you
{"type": foo}
And therefore not really be valid json, but you can play around with it to fit your needs!
I've recently bumped into a situation where I had to serialize an object that has a couple of Enum types as members.
Basically, I've just added a helper function that maps enum types to their name.
from enum import Enum, auto
from json import dumps
class Status(Enum):
OK = auto()
NOT_OK = auto()
class MyObject:
def __init__(self, status):
self.status = status
obja = MyObject(Status.OK)
objb = MyObject(Status.NOT_OK)
print(dumps(obja))
print(dumps(objb))
This of course fails with the error TypeError: Object of type MyObject is not JSON serializable, as the status member of the MyObject instances is not serializable.
from enum import Enum, auto
from json import dumps
def _prepare_for_serialization(obj):
serialized_dict = dict()
for k, v in obj.__dict__.items():
serialized_dict[k] = v.name if isinstance(v, Enum) else v
return serialized_dict
class Status(Enum):
OK = auto()
NOT_OK = auto()
class MyObject:
def __init__(self, status):
self.status = status
obja = MyObject(Status.OK)
objb = MyObject(Status.NOT_OK)
print(dumps(_prepare_for_serialization(obja)))
print(dumps(_prepare_for_serialization(objb)))
This prints:
{"status": "OK"}
{"status": "NOT_OK"}
Later on, I've used the same helper function to cherry-pick keys for the serialized dict.
You can use a metaclass instead of an enum, and instead of multiple-inheritance without these side effects.
https://gist.github.com/earonesty/81e6c29fa4c54e9b67d9979ddbd8489d
For example:
class FooBarType(metaclass=TypedEnum):
standard = 0
foo = 1
bar = 2
That way every instance is an integer and is also a FooBarType.
Metaclass below.
class TypedEnum(type):
"""This metaclass creates an enumeration that preserves isinstance(element, type)."""
def __new__(mcs, cls, bases, classdict):
"""Discover the enum members by removing all intrinsics and specials."""
object_attrs = set(dir(type(cls, (object,), {})))
member_names = set(classdict.keys()) - object_attrs
member_names = member_names - set(name for name in member_names if name.startswith("_") and name.endswith("_"))
new_class = None
base = None
for attr in member_names:
value = classdict[attr]
if new_class is None:
# base class for all members is the type of the value
base = type(classdict[attr])
ext_bases = (*bases, base)
new_class = super().__new__(mcs, cls, ext_bases, classdict)
setattr(new_class, "__member_names__", member_names)
else:
if not base == type(classdict[attr]): # noqa
raise SyntaxError("Cannot mix types in TypedEnum")
new_val = new_class.__new__(new_class, value)
setattr(new_class, attr, new_val)
for parent in bases:
new_names = getattr(parent, "__member_names__", set())
member_names |= new_names
for attr in new_names:
value = getattr(parent, attr)
if not isinstance(value, base):
raise SyntaxError("Cannot mix inherited types in TypedEnum: %s from %s" % (attr, parent))
# convert all inherited values to the new class
setattr(new_class, attr, new_class(value))
return new_class
def __call__(cls, arg):
for name in cls.__member_names__:
if arg == getattr(cls, name):
return type.__call__(cls, arg)
raise ValueError("Invalid value '%s' for %s" % (arg, cls.__name__))
#property
def __members__(cls):
"""Sufficient to make the #unique decorator work."""
class FakeEnum: # pylint: disable=too-few-public-methods
"""Object that looks a bit like an Enum instance."""
def __init__(self, name, value):
self.name = name
self.value = value
return {name: FakeEnum(name, getattr(cls, name)) for name in cls.__member_names__}
def __iter__(cls):
"""List all enum values."""
return (getattr(cls, name) for name in cls.__member_names__)
def __len__(cls):
"""Get number of enum values."""
return len(cls.__member_names__)
If you have a class model instead a dict, you can convert to json with this:
from enum import Enum
import json
class FooBarType(str, Enum):
standard = 0
foo = 1
bar = 2
class ModelExample():
def __init__(self, name: str, type: FooBarType) -> None:
self.name = name
self.type = type
# instantiate a class with your values
model_example = ModelExample(name= 'test', type= FooBarType.foo)
# vars -> get a dict of the class
json.loads(json.dumps(vars(model_example)))
Result:
{'name': 'test', 'type': '1'}
For a custom object I am able to encode into json using JSONEncoder.
class CustomEncoder(JSONEncoder):
def encode(self, custom):
prop_dict = {}
for prop in Custom.all_properties_names():
if custom.__getattribute__(prop) is not None:
if prop is 'created_timestamp':
prop_dict.update({prop: custom.__getattribute__(
prop).isoformat()})
else:
prop_dict.update({prop: custom.__getattribute__(prop)})
return prop_dict
To generate json, I am using json.dumps(custom, cls=CustomEncoder, indent=True)
Now I have a list of Custom class objects. How do convert the list to json?
custom_list = //get custom object list from service
How do I convert the whole list to json? Do I need to iterate and capture json of each custom object and append to a list with comma separated? I feel like there should be something straightforward I am missing here.
The custom encoder is called only when needed. If you have a custom thing that the JSON library thinks it can encode, like a string or dictionary, the custom encoder won't be called. The following example shows that encoding an object, or a list including an object, works with a single custom encoder:
import json
class Custom(object):
pass
class CustomEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, Custom):
return 'TASTY'
return CustomEncoder(self, o)
print json.dumps( Custom(), cls=CustomEncoder )
print json.dumps( [1, [2,'three'], Custom()], cls=CustomEncoder )
Output:
"TASTY"
[1, [2, "three"], "TASTY"]
In my way, I convert object to dict then using json.dumps list of dict:
def custom_to_dict(custom):
return {
'att1': custom.att1,
'att2': custom.att2,
...
}
#custom_list is your list of customs
out = json.dumps([custom_to_dict(custom) for custom in custom_list])
It might be helpful