I am trying to define simple getter/setter methods for a mixin class that I intend to use in my database schema:
from sqlalchemy import Column, Integer, create_engine
from sqlalchemy.orm import synonym, scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base, declared_attr
engine = create_engine('sqlite:///')
Base = declarative_base(bind=engine)
Session = scoped_session(sessionmaker(bind=engine))
class Mixin(object):
_attr = Column('attr', Integer)
#property
def attr(self):
return self._attr
#attr.setter
def attr(self, value):
self._attr = value
attr = synonym('_attr', descriptor=attr)
class DummyClass(Base, Mixin):
__tablename__ = 'dummy'
id = Column(Integer, primary_key=True)
Base.metadata.create_all()
if __name__ == '__main__':
session = Session()
testobj = DummyClass()
session.add(testobj)
testobj.attr = 42
assert testobj.attr == 42
When I try to run this example, I get the following error:
sqlalchemy.exc.InvalidRequestError: Mapper properties (i.e. deferred,column_property(), relationship(), etc.) must be declared as #declared_attr callables on declarative mixin classes.`
My code is almost a 1:1 copy from the SQLAlchemy Declarative Tutorial, with the only difference that the property/synonym is declared in a Mixin class. Appending or prepending the "#declared_attr" decorator to the existing decorators does not change anything.
How do I resolve this situation?
Perhaps create the attr() as an #declared_attr-decorated class method which returns the synonym?
class Mixin(object):
_attr = Column('attr', Integer)
def get_attr(self):
return self._attr
def set_attr(self, value):
self._attr = value
#declared_attr
def attr(cls):
return synonym('_attr', descriptor=property(cls.get_attr, cls.set_attr))
#contact.setter
def contact_id(self, contact_id):
if (contact_id is None):
raise ValueError('contact_id can only be a uuid string')
if not isinstance(contact_id, str):
raise TypeError('contact_id is not a string')
self._contact_id = contact_id
With a setter the code above can throw a much more clearer error
than what PyAlchemy can throw at you.
You might also choose to gracefully handle the error in production if possible.
Related
I'm trying to override __getattr__ in SQLAlchemy so I can access certain dictionary fields using object.field syntax rather than having to do object.fields["field"].
However, SQLAlchemy isn't recognizing changes to the field if I use the dot notation syntax. Accessing the field using normal object.fields["field"] however works as expected.
I've provided a code sample below.
#!/usr/bin/env python3
import sqlalchemy
from sqlalchemy import Column, Integer, JSON
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.orm import sessionmaker
Session = sessionmaker()
Base = declarative_base()
class Track(Base): # noqa: WPS230
__tablename__ = "track"
id = Column(Integer, primary_key=True)
_fields = Column(MutableDict.as_mutable(JSON), default="{}")
def __init__(self, id):
self.id = id
self._fields = {}
self._fields["custom"] = "field"
def __getattr__(self, name: str):
"""See if ``name`` is a custom field."""
try:
return self.__dict__["_fields"][name]
except KeyError:
raise AttributeError from None
def __setattr__(self, name, value):
"""Set custom fields if a valid key."""
if "_fields" in self.__dict__ and name in self.__dict__["_fields"]:
self._fields[name] = value
else:
super().__setattr__(name, value)
engine = sqlalchemy.create_engine("sqlite:///:memory:")
Session.configure(bind=engine)
Base.metadata.create_all(engine) # creates tables
session = Session()
track = Track(id=1)
session.add(track)
session.commit()
assert track.custom
track.custom = "new"
assert track in session.dirty
assert track.custom == "new"
I believe it has something to do with how sqlalchemy loads __dict__, but not sure how to work around it to get my desired behavior.
With the help of #CaseIIT on github, I came up with the following solution using a list FIELDS of fields that can be set:
class Track(Base): # noqa: WPS230
__tablename__ = "track"
FIELDS = ["custom"]
id = Column(Integer, primary_key=True)
_fields = Column(MutableDict.as_mutable(JSON), default="{}")
def __init__(self):
self._fields = {}
self._fields["custom"] = "field"
def __getattr__(self, name: str):
"""See if ``name`` is a custom field."""
if name in self.FIELDS:
return self._fields[name]
else:
raise AttributeError from None
def __setattr__(self, name, value):
"""Set custom custom_fields if a valid key."""
if name in self.FIELDS:
self._fields[name] = value
else:
super().__setattr__(name, value)
It feels wrong to have a duplicate list of accessible fields between _fields.keys() and FIELDS, but I can't seem to get around the issues that come with trying to inspect _fields.
Backstory
I have a questionnaire that asks sensitive questions most of which are true/false. The majority of the time the values are false which poses a challenge when keeping the data private at rest. When encrypting each question into a separate column, it is really easy to tell which value is true and which is false with a bit of guessing. To combat this, the questions and answers are put into a dictionary object with some salt (nonsense that changes randomly) then encrypted. Making it impossible without the key to know what the answers were.
Method
Below is an example of the model used to encrypt the data with salt at rest making it impossible to look at the data and know the contents.
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy_utils.types import JSONType
from sqlalchemy_utils.types.encrypted.encrypted_type import StringEncryptedType, AesEngine
Base = declarative_base()
class SensitiveQuestionnaire(Base):
user_id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
_data = data: dict = sa.Column(StringEncryptedType(JSONType, 'secret', AesEngine, 'pkcs5'),
nullable=False, default=lambda: {'_salt': salt_shaker()})
# values are viewed using a python property to look into the `_data` dict
#property
def sensitive_question(self) -> Optional[float]:
return self._data.get('sensitive_question')
# values are set into the `_data` dict
#sensitive_question.setter
def sensitive_question(self, value: bool) -> None:
self._data['sensitive_question'] = value
# in a real example there would be 20+ properties that map to questions
def __init__(self, **kwargs):
# Sqlalchemy does not use the __init__ method so we are free to set object defaults here
self._data = {'_salt': salt_shaker()}
for key in kwargs:
setattr(self, key, kwargs[key])
#property
def _salt(self) -> str:
return self._data['_salt']
def salt_shaker():
return ''.join([random.choice('hldjs..' for i in range(50)])
The Problem
After the SensitiveQuestionnaire object is initialized none of the changes are persisted in the database.
# GIVEN a questionnaire
questionnaire = model.SensitiveQuestionnaire(user_id=1)
db.session.add()
db.session.commit()
# WHEN updating the questionnaire and saving it to the database
questionnaire.sensitive_question= True
db.session.commit()
# THEN we get the questionnaire from the database
db_questionnaire = model.SensitiveQuestionnaire.query\
.filter(model.SensitiveQuestionnaire.user_id == 1).first()
# THEN the sensitive_question value is persisted
assert db_questionnaire.sensitive_question is True
Value from the db_questionnaire.sensitive_question is None when it should be True.
After spending the better part of the day to figure this out, the cause of the issue is how Sqlalchemy knows when there is a change. The short version is sqlalchemy uses python's __setitem__ to hook in sqlalchemy's change() method letting it know there was a change. More info can be found in sqlalchemy's docs.
The answer is to wrap the StringEncryptedType in a MultableDict Type
Mutation Tracking
Provide support for tracking of in-place changes to scalar values, which are propagated into ORM change events on owning parent objects.
From SqlAlchemy's docs: https://docs.sqlalchemy.org/en/13/orm/extensions/mutable.html
Solution
Condensed version... wrapping the StringEncryptedType in a MutableDict
_data = data: dict = sa.Column(
MutableDict.as_mutable(StringEncryptedType(JSONType, 'secret', AesEngine, 'pkcs5')),
nullable=False, default=lambda: {'_salt': salt_shaker()})
Full version from the question above
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy_utils.types import JSONType
from sqlalchemy_utils.types.encrypted.encrypted_type import StringEncryptedType, AesEngine
Base = declarative_base()
class SensitiveQuestionnaire(Base):
user_id: int = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
# The MutableDict.as_mutable below is what changed!
_data = data: dict = sa.Column(
MutableDict.as_mutable(StringEncryptedType(JSONType, 'secret', AesEngine, 'pkcs5')),
nullable=False, default=lambda: {'_salt': salt_shaker()})
#property
def sensitive_question(self) -> Optional[float]:
return self._data.get('sensitive_question')
# values are set into the `_data` dict
#sensitive_question.setter
def sensitive_question(self, value: bool) -> None:
self._data['sensitive_question'] = value
# in a real example there would be 20+ properties that map to questions
def __init__(self, **kwargs):
self._data = {'_salt': salt_shaker()}
for key in kwargs:
setattr(self, key, kwargs[key])
#property
def _salt(self) -> str:
return self._data['_salt']
def salt_shaker():
return ''.join([random.choice('hldjs..' for i in range(50)])
I know there is a way to instrument SQLAlchemy to prepend common prefix to all columns, is it possible to add a common prefix to all table names derived from single declarative_base?
Use declared_attr and a customized Base along the lines of:
from sqlalchemy.ext.declarative import declared_attr
class PrefixerBase(Base):
__abstract__ = True
_the_prefix = 'someprefix_'
#declared_attr
def __tablename__(cls):
return cls._the_prefix + cls.__incomplete_tablename__
class SomeModel(PrefixerBase):
__incomplete_tablename__ = 'sometable'
...
A class marked with __abstract__ will not get a table or mapper created and can act as the extended declarative base.
You could also make it even more implicit using a custom metaclass (originally described here):
from sqlalchemy.ext.declarative.api import DeclarativeMeta
class PrefixerMeta(DeclarativeMeta):
def __init__(cls, name, bases, dict_):
if '__tablename__' in dict_:
cls.__tablename__ = dict_['__tablename__'] = \
'someprefix_' + dict_['__tablename__']
return super().__init__(name, bases, dict_)
Base = declarative_base(metaclass=PrefixerMeta)
class SomeModel(Base):
__tablename__ = 'sometable'
...
I have a class inheritance scheme as layed out in http://docs.sqlalchemy.org/en/latest/orm/inheritance.html#joined-table-inheritance
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
type = Column(String)
__mapper_args__ = {'polymorphic_on': type}
class Child(Parent):
__tablename__ = 'child'
id = Column(Integer, ForeignKey('parent.id'), primary_key=True)
__mapper_args__ = {'polymorphic_identity': 'child'}
I'd like to be able to create an instance of Child using the constructor of Parent (like Parent(type='child')) but it doesn't work. When I fire up IPython...
In [1]: from stackoverflow.question import Parent, Child
In [2]: from sqlalchemy import create_engine
In [3]: from sqlalchemy.orm import sessionmaker
In [4]: session = sessionmaker(bind=create_engine(...), autocommit=True)()
In [5]: with session.begin():
p = Parent(type='child')
session.add(p)
...:
/.../lib/python3.4/site-packages/sqlalchemy/orm/persistence.py:155: SAWarning: Flushing object <Parent at 0x7fe498378e10> with incompatible polymorphic identity 'child'; the object may not refresh and/or load correctly
mapper._validate_polymorphic_identity(mapper, state, dict_)
In [6]: session.query(Parent).all()
Out[6]: [<stackoverflow.question.Parent at 0x7fe498378e10>]
In [7]: session.query(Child).all()
Out[7]: []
Is this possible? Is it a good idea?
Definitely not a good idea. Instead of using a constructor to do some hack, you could just have a separate helper function (a factory):
# create this manually
OBJ_TYPE_MAP = {
# #note: using both None and 'parent', but should settle on one
None: Parent, 'parent': Parent,
'child': Child,
}
# ... or even automatically from the mappings:
OBJ_TYPE_MAP = {
x.polymorphic_identity: x.class_
for x in Parent.__mapper__.self_and_descendants
}
print(OBJ_TYPE_MAP)
def createNewObject(type_name, **kwargs):
typ = OBJ_TYPE_MAP.get(type_name)
assert typ, "Unknown type: {}".format(type_name)
return typ(**kwargs)
a_parent = createNewObject(None, p_field1='parent_name1')
a_child = createNewObject(
'child', p_field1='child_name1', c_field2='child_desc')
session.add_all([a_child, a_parent])
Another note: for the Parent i would define a value for {'polymorphic_identity': 'parent'}. It makes it much cleaner than having None .
EDIT-1: using Constructor
Not that i recommend it, or that I really know what I am doing here, but if you add the __new__ as defined below to the Parent class:
def __new__(cls, *args, **kwargs):
typ = kwargs.get('type') # or .pop(...)
if typ and not kwargs.pop('_my_hack', None):
# print("Special handling for {}...".format(typ))
if typ == 'parent':
# here we can *properly* call the next in line
return super(Parent, cls).__new__(cls, *args, **kwargs)
elif typ == 'child':
# #note: need this to avoid endless recursion
kwargs["_my_hack"] = True
# here we need to cheat somewhat
return Child.__new__(Child, *args, **kwargs)
else:
raise Exception("nono")
else:
x = super(Parent, cls).__new__(cls, *args, **kwargs)
return x
you will be able to use both the old way (when no type=xxx is passed to __init__), or do what you ask for by providing a parameter:
old_parent = Parent(field1=xxx, ...)
old_child = Child(field1=xxx, ...)
new_child = Parent(type='child', field1=xxx, ...)
Again, I am not sure of all implications, especially because sqlalchemy also overrides the creation routines and uses its own meta classes.
The thing is that when using sqlalchemy declarative mappings a mapper is generated for each class.
What you are trying to do is to make an instance of Parent that will behave as an instance of Child, which is something you can't do, at least without resorting to hacks.
By that fact (that you have to go through hoops) it's not a good idea. Maybe you don't need inheritance at all ?
EDIT
If you don't want to have conditional logic or lookups and you have to select a class based on user input you could do something like this
cls = getattr(module_containing_the_classes, "<user_input>")
cls(**kw)
So say I have some classes X, Y and Z using SQLAlchemy declarative syntax to define some simple columns and relationships
Requirements:
At the class level, (X|Y|Z).primary_keys returns a collection of
the respective class' primary keys' (InstrumentedAttribute
objects) I also want (X|Y|Z).relations to reference the class'
relations in the same way
At the instance level, I would like the same attributes to reference
those attributes' instantiated values, whether they've been
populated using my own constructors, individual attributes
setters, or whatever SQLAlchemy does when it retrieves rows from
the db.
So far I have the following.
import collections
import sqlalchemy
import sqlalchemy.ext.declarative
from sqlalchemy import MetaData, Column, Table, ForeignKey, Integer, String, Date, Text
from sqlalchemy.orm import relationship, backref
class IndexedMeta(sqlalchemy.ext.declarative.DeclarativeMeta):
"""Metaclass to initialize some class-level collections on models"""
def __new__(cls, name, bases, defaultdict):
cls.pk_columns = set()
cls.relations = collections.namedtuple('RelationshipItem', 'one many')( set(), set())
return super().__new__(cls, name, bases, defaultdict)
Base = sqlalchemy.ext.declarative.declarative_base(metaclass=IndexedMeta)
def build_class_lens(cls, key, inst):
"""Populates the 'indexes' of primary key and relationship attributes with the attributes' names. Additionally, separates "x to many" relationships from "x to one" relationships and associates "x to one" relathionships with the local-side foreign key column"""
if isinstance(inst.property, sqlalchemy.orm.properties.ColumnProperty):
if inst.property.columns[0].primary_key:
cls.pk_columns.add(inst.key)
elif isinstance(inst.property, sqlalchemy.orm.properties.RelationshipProperty):
if inst.property.direction.name == ('MANYTOONE' or 'ONETOONE'):
local_column = cls.__mapper__.get_property_by_column(inst.property.local_side[0]).key
cls.relations.one.add( (local_column, inst.key) )
else:
cls.relations.many.add(inst.key)
sqlalchemy.event.listen(Base, 'attribute_instrument', build_class_lens)
class Meeting(Base):
__tablename__ = 'meetings'
def __init__(self, memo):
self.memo = memo
id = Column(Integer, primary_key=True)
date = Column(Date)
memo = Column('note', String(60), nullable=True)
category_name = Column('category', String(60), ForeignKey('categories.name'))
category = relationship("Category", backref=backref('meetings'))
topics = relationship("Topic",
secondary=meetings_topics,
backref="meetings")
...
...
Ok, so that gets me by on the class level, though I feel like I am doing silly things with metaclasses, and I get some strange intermittent errors where the 'sqlalchemy' module allegedly isn't recognized in build_class_lens and evals to Nonetype.
I am not quite sure how I should proceed at the instance level.
I've looked into the events interface. I see the ORM event init, but it seems to run prior to the __init__ function defined on my models, meaning the instance attributes haven't yet been populated at that time, so I can't build my 'lens' on them.
I also wonder if the Attribute event set might be of help. That is my next try, though i still wonder if it is the most appropriate way.
All in all I really wonder if I am missing some really elegant way to approach this problem.
I think the metaclass thing with declarative goes by the old XML saying, "if you have a problem, and use XML, now you have two problems". The metaclass in Python is useful pretty much as a hook to detect the construction of new classes, and that's about it. We now have enough events that there shouldn't be any need to use a metaclass beyond what declarative already does.
In this case I'd go a little further and say that the approach of trying to actively build up these collections is not really worth it - it's much easier to generate them lazily, as below:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
import collections
from sqlalchemy.orm.properties import RelationshipProperty
class memoized_classproperty(object):
"""A decorator that evaluates once at the class level,
assigns the new value to the class.
"""
def __init__(self, fget, doc=None):
self.fget = fget
self.__doc__ = doc or fget.__doc__
self.__name__ = fget.__name__
def __get__(desc, self, cls):
result = desc.fget(cls)
setattr(cls, desc.__name__, result)
return result
class Lens(object):
#memoized_classproperty
def pk_columns(cls):
return class_mapper(cls).primary_key
#memoized_classproperty
def relations(cls):
props = collections.namedtuple('RelationshipItem', 'one many')(set(), set())
# 0.8 will have "inspect(cls).relationships" here
mapper = class_mapper(cls)
for item in mapper.iterate_properties:
if isinstance(item, RelationshipProperty):
if item.direction.name == ('MANYTOONE' or 'ONETOONE'):
local_column = mapper.get_property_by_column(item.local_side[0]).key
props.one.add((local_column, item.key))
else:
props.many.add(item.key)
return props
Base= declarative_base(cls=Lens)
meetings_topics = Table("meetings_topics", Base.metadata,
Column('topic_id', Integer, ForeignKey('topic.id')),
Column('meetings_id', Integer, ForeignKey('meetings.id')),
)
class Meeting(Base):
__tablename__ = 'meetings'
def __init__(self, memo):
self.memo = memo
id = Column(Integer, primary_key=True)
date = Column(Date)
memo = Column('note', String(60), nullable=True)
category_name = Column('category', String(60), ForeignKey('categories.name'))
category = relationship("Category", backref=backref('meetings'))
topics = relationship("Topic",
secondary=meetings_topics,
backref="meetings")
class Category(Base):
__tablename__ = 'categories'
name = Column(String(50), primary_key=True)
class Topic(Base):
__tablename__ = 'topic'
id = Column(Integer, primary_key=True)
print Meeting.pk_columns
print Meeting.relations.one
# assignment is OK, since prop is memoized
Meeting.relations.one.add("FOO")
print Meeting.relations.one