I've written a storage interface that can use different storage backends. As a demonstration I've written an implementation that stores objects as a key/value pair in a standard Python dictionary.
class MyStorageInterface(object):
def store(self, key, value):
raise NotImplementedError("Class must be subclassed")
def get(self, key):
raise NotImplementedError("Class must be subclassed")
# Example implementation
class DictStorage(MyStorageInterface):
def __init__(self):
self._cache = dict()
def store(self, key, value):
self._cache[key] = value
def get(self, key):
return self._cache[key]
Now, I want to write some unit tests for my DictStorage implementation. My question is whether or not something like:
storage = DictStorage()
value = 8
key = 'foo'
storage.store(key, value)
actual = storage.get(key)
assert actual == 8, "cache value for %s is %s, expected %s" % (key, str(actual), str(value))
is a suitable unit test for the get method or whether or not there is a Pythonic pattern for testing classes that implement collection type objects.
I don't know if there's anything "collection-specific" but the structure of your test looks good to me. The only difference is that I'd make use of the unittest package and define it as test case:
import unittest
class MyStorageInterface(object):
def store(self, key, value):
raise NotImplementedError("Class must be subclassed")
def get(self, key):
raise NotImplementedError("Class must be subclassed")
class DictStorage(MyStorageInterface):
def __init__(self):
self._cache = dict()
def store(self, key, value):
self._cache[key] = value
def get(self, key):
return self._cache[key]
class DictStorageTests(unittest.TestCase):
def setUp(self):
self._sut = DictStorage()
def test_storing_and_retrieving_value(self):
value_in = 8
key = 'foo'
self._sut.store(key, value_in)
value_out = self._sut.get(key)
self.assertEqual(value_in, value_out)
if __name__ == '__main__':
unittest.main()
Output
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK
SQLAlchemy offers the PickleType and offers mutation tracking for any type that is mutable (like a dict).
The SQLAlchemy documentation mentions that this is the way to implement a mutable PickleType but it does not state exactly how to proceed with it.
Note: I want to store a dict in the PickleType.
How do you implement this?
While the documentation mentions some examples, it is not sufficient in my eyes, so I will add my implementation here that can be used to implement a mutable dict that is pickled and stored in the database.
Use the MutableDict example from the docs:
class MutableDict(Mutable, dict):
#classmethod
def coerce(cls, key, value):
if not isinstance(value, MutableDict):
if isinstance(value, dict):
return MutableDict(value)
return Mutable.coerce(key, value)
else:
return value
def __delitem(self, key):
dict.__delitem__(self, key)
self.changed()
def __setitem__(self, key, value):
dict.__setitem__(self, key, value)
self.changed()
def __getstate__(self):
return dict(self)
def __setstate__(self, state):
self.update(self)
Now create a column to be tracked:
class MyModel(Base):
data = Column(MutableDict.as_mutable(PickleType))
I would like to see some other examples that are maybe more advanced or possibly use different data structures. What would a generic approach for pickle look like? Is there one (I suppose not, or SQLAlchemy would have one).
Here's a solution I came up with. It wraps any type and detects any attribute sets and calls Mutable.changed(). It also wraps function calls and detects changes by taking a snapshot of the object before and after and comparing. Should work for Pickleable types...
from sqlalchemy.ext.mutable import Mutable
class MutableTypeWrapper(Mutable):
top_attributes = ['_underlying_object',
'_underlying_type',
'_last_state',
'_snapshot_update',
'_snapshot_changed',
'_notify_if_changed',
'changed',
'__getstate__',
'__setstate__',
'coerce']
#classmethod
def coerce(cls, key, value):
if not isinstance(value, MutableTypeWrapper):
try:
return MutableTypeWrapper(value)
except:
return Mutable.coerce(key, value)
else:
return value
def __getstate__(self):
return self._underlying_object
def __setstate__(self, state):
self._underlying_type = type(state)
self._underlying_object = state
def __init__(self, underlying_object, underlying_type=None):
if (underlying_object is None and underlying_type is None):
print('Both underlying object and type are none.')
raise RuntimeError('Unable to create MutableTypeWrapper with no underlying object or type.')
if (underlying_object is not None):
self._underlying_object = underlying_object
else:
self._underlying_object = underlying_type()
if (underlying_type is not None):
self._underlying_type = underlying_type
else:
self._underlying_type = type(underlying_object)
def __getattr__(self, attr):
if (attr in MutableTypeWrapper.top_attributes):
return object.__getattribute__(self, attr)
orig_attr = self._underlying_object.__getattribute__(attr)
if callable(orig_attr):
def hooked(*args, **kwargs):
self._snapshot_update()
result = orig_attr(*args, **kwargs)
self._notify_if_changed()
# prevent underlying from becoming unwrapped
if result == self._underlying_object:
return self
return result
return hooked
else:
return orig_attr
def __setattr__(self, attr, value):
if (attr in MutableTypeWrapper.top_attributes):
object.__setattr__(self, attr, value)
return
self._underlying_object.__setattr__(attr, value)
self.changed()
def _snapshot_update(self):
self._last_state = pickle.dumps(self._underlying_object,
pickle.HIGHEST_PROTOCOL)
def _snapshot_changed(self):
return self._last_state != pickle.dumps(self._underlying_object,
pickle.HIGHEST_PROTOCOL)
def _notify_if_changed(self):
if (self._snapshot_changed()):
self.changed()
And then use it with PickleType as follows:
class TestModel(Base):
__tablename__ = 'testtable'
id = Column(Integer, primary_key=True)
obj = Column(MutableTypeWrapper.as_mutable(PickleType))
The disadvantage here is the underlying class is snapshotted before every function call, and then changes are compared after in order to verify if the underlying object has changed. This will have a significant performance impact.
The other way to ensure that your PickleType objects are updated when you modify them is to copy and assign them before committing changes.
New to python...
I have the following class Key, that extends dict:
class Key( dict ):
def __init__( self ):
self = { some dictionary stuff... }
def __getstate__(self):
state = self.__dict__.copy()
return state
def __setstate__(self, state):
self.__dict__.update( state )
I want to save an instance of the class with its data using pickle.dump and then retrieve the data using pickle.load. I understand that I am supposed to somehow change the getstate and the setstate, however, am not entirely clear on how I am supposed to do that... any help would be greatly appreciated!
I wrote a subclass of dict that does this here it is.
class AttrDict(dict):
"""A dictionary with attribute-style access. It maps attribute access to
the real dictionary. """
def __init__(self, *args, **kwargs):
dict.__init__(self, *args, **kwargs)
def __getstate__(self):
return self.__dict__.items()
def __setstate__(self, items):
for key, val in items:
self.__dict__[key] = val
def __repr__(self):
return "%s(%s)" % (self.__class__.__name__, dict.__repr__(self))
def __setitem__(self, key, value):
return super(AttrDict, self).__setitem__(key, value)
def __getitem__(self, name):
return super(AttrDict, self).__getitem__(name)
def __delitem__(self, name):
return super(AttrDict, self).__delitem__(name)
__getattr__ = __getitem__
__setattr__ = __setitem__
def copy(self):
return AttrDict(self)
It basically converts the state to a basic tuple, and takes that back again to unpickle.
But be aware that you have to have to original source file available to unpickle. The pickling does not actually save the class itself, only the instance state. Python will need the original class definition to re-create from.
I'm storing JSON down as blob/text in a column using MySQL. Is there a simple way to convert this into a dict using python/SQLAlchemy?
You can very easily create your own type with SQLAlchemy
For SQLAlchemy versions >= 0.7, check out Yogesh's answer below
import jsonpickle
import sqlalchemy.types as types
class JsonType(types.MutableType, types.TypeDecorator):
impl = types.Unicode
def process_bind_param(self, value, engine):
return unicode(jsonpickle.encode(value))
def process_result_value(self, value, engine):
if value:
return jsonpickle.decode(value)
else:
# default can also be a list
return {}
This can be used when you are defining your tables (example uses elixir):
from elixir import *
class MyTable(Entity):
using_options(tablename='my_table')
foo = Field(String, primary_key=True)
content = Field(JsonType())
active = Field(Boolean, default=True)
You can also use a different json serialiser to jsonpickle.
sqlalchemy.types.MutableType has been deprecated (v0.7 onward), the documentation recommends using sqlalchemy.ext.mutable instead.
I found a Git gist by dbarnett that I have tested for my usage. It has worked well so far, for both dictionary and lists.
Pasting below for posterity:
import simplejson
import sqlalchemy
from sqlalchemy import String
from sqlalchemy.ext.mutable import Mutable
class JSONEncodedObj(sqlalchemy.types.TypeDecorator):
"""Represents an immutable structure as a json-encoded string."""
impl = String
def process_bind_param(self, value, dialect):
if value is not None:
value = simplejson.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = simplejson.loads(value)
return value
class MutationObj(Mutable):
#classmethod
def coerce(cls, key, value):
if isinstance(value, dict) and not isinstance(value, MutationDict):
return MutationDict.coerce(key, value)
if isinstance(value, list) and not isinstance(value, MutationList):
return MutationList.coerce(key, value)
return value
#classmethod
def _listen_on_attribute(cls, attribute, coerce, parent_cls):
key = attribute.key
if parent_cls is not attribute.class_:
return
# rely on "propagate" here
parent_cls = attribute.class_
def load(state, *args):
val = state.dict.get(key, None)
if coerce:
val = cls.coerce(key, val)
state.dict[key] = val
if isinstance(val, cls):
val._parents[state.obj()] = key
def set(target, value, oldvalue, initiator):
if not isinstance(value, cls):
value = cls.coerce(key, value)
if isinstance(value, cls):
value._parents[target.obj()] = key
if isinstance(oldvalue, cls):
oldvalue._parents.pop(target.obj(), None)
return value
def pickle(state, state_dict):
val = state.dict.get(key, None)
if isinstance(val, cls):
if 'ext.mutable.values' not in state_dict:
state_dict['ext.mutable.values'] = []
state_dict['ext.mutable.values'].append(val)
def unpickle(state, state_dict):
if 'ext.mutable.values' in state_dict:
for val in state_dict['ext.mutable.values']:
val._parents[state.obj()] = key
sqlalchemy.event.listen(parent_cls, 'load', load, raw=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True, propagate=True)
sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True, propagate=True)
sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True, propagate=True)
class MutationDict(MutationObj, dict):
#classmethod
def coerce(cls, key, value):
"""Convert plain dictionary to MutationDict"""
self = MutationDict((k,MutationObj.coerce(key,v)) for (k,v) in value.items())
self._key = key
return self
def __setitem__(self, key, value):
dict.__setitem__(self, key, MutationObj.coerce(self._key, value))
self.changed()
def __delitem__(self, key):
dict.__delitem__(self, key)
self.changed()
class MutationList(MutationObj, list):
#classmethod
def coerce(cls, key, value):
"""Convert plain list to MutationList"""
self = MutationList((MutationObj.coerce(key, v) for v in value))
self._key = key
return self
def __setitem__(self, idx, value):
list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
self.changed()
def __setslice__(self, start, stop, values):
list.__setslice__(self, start, stop, (MutationObj.coerce(self._key, v) for v in values))
self.changed()
def __delitem__(self, idx):
list.__delitem__(self, idx)
self.changed()
def __delslice__(self, start, stop):
list.__delslice__(self, start, stop)
self.changed()
def append(self, value):
list.append(self, MutationObj.coerce(self._key, value))
self.changed()
def insert(self, idx, value):
list.insert(self, idx, MutationObj.coerce(self._key, value))
self.changed()
def extend(self, values):
list.extend(self, (MutationObj.coerce(self._key, v) for v in values))
self.changed()
def pop(self, *args, **kw):
value = list.pop(self, *args, **kw)
self.changed()
return value
def remove(self, value):
list.remove(self, value)
self.changed()
def JSONAlchemy(sqltype):
"""A type to encode/decode JSON on the fly
sqltype is the string type for the underlying DB column.
You can use it like:
Column(JSONAlchemy(Text(600)))
"""
class _JSONEncodedObj(JSONEncodedObj):
impl = sqltype
return MutationObj.as_mutable(_JSONEncodedObj)
I think the JSON example from the SQLAlchemy docs is also worth mentioning:
https://docs.sqlalchemy.org/en/13/core/custom_types.html#marshal-json-strings
However, I think it can be improved to be less strict regarding NULL and empty strings:
class JSONEncodedDict(TypeDecorator):
impl = VARCHAR
def process_bind_param(self, value, dialect):
if value is None:
return None
return json.dumps(value, use_decimal=True)
def process_result_value(self, value, dialect):
if not value:
return None
return json.loads(value, use_decimal=True)
There is a recipe for this in the official documentation:
from sqlalchemy.types import TypeDecorator, VARCHAR
import json
class JSONEncodedDict(TypeDecorator):
"""Represents an immutable structure as a json-encoded string.
Usage::
JSONEncodedDict(255)
"""
impl = VARCHAR
def process_bind_param(self, value, dialect):
if value is not None:
value = json.dumps(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = json.loads(value)
return value
How about json.loads()?
>>> d= {"foo":1, "bar":[2,3]}
>>> s='{"foo":1, "bar":[2,3]}'
>>> import json
>>> json.loads(s) == d
True
Based on #snapshoe answer and to answer #Timmy's comment:
You can do it by using properties. Here is an example of a table:
class Providers(Base):
__tablename__ = "providers"
id = Column(
Integer,
Sequence('providers_id', optional=True),
primary_key=True
)
name = Column(Unicode(40), index=True)
_config = Column("config", Unicode(2048))
#property
def config(self):
if not self._config:
return {}
return json.loads(self._config)
#config.setter
def config(self, value):
self._config = json.dumps(value)
def set_config(self, field, value):
config = self.config
config[field] = value
self.config = config
def get_config(self):
if not self._config:
return {}
return json.loads(self._config)
def unset_config(self, field):
config = self.get_config()
if field in config:
del config[field]
self.config = config
Now you can use it on a Providers() object:
>>> p = Providers()
>>> p.set_config("foo", "bar")
>>> p.get_config()
{"foo": "bar"}
>>> a.config
{u'foo': u'bar'}
I know this is an old Question maybe even dead, but I hope this could help someone.
This is what I came up with based on the two answers above.
import json
class JsonType(types.TypeDecorator):
impl = types.Unicode
def process_bind_param(self, value, dialect):
if value :
return unicode(json.dumps(value))
else:
return {}
def process_result_value(self, value, dialect):
if value:
return json.loads(value)
else:
return {}
As an update to the previous responses, which we've used with success so far. As of MySQL 5.7 and SQLAlchemy 1.1 you can use the native MySQL JSON data type, which gives you better performance and a whole range of operators for free.
It lets you to create virtual secondary indexes on JSON elements too.
But of course you will lock yourself into running your app on MySQL only when moving the logic into the database itself.