Develop Reference

python - loading #hybrid_property of SQLAlchemy ORM to pandas DataFrame

TLDR DataFrame.from_records() ignores #hybrid_property of sqlalchemy ORM
Hello, I want to be able to populate pandas.DataFrame from sqlalchemy objects.
I have a class which is defined like that:
class Cat_24(Base):
__tablename__ = "cat_24"
ad_id = Column(Integer, primary_key=True, unique=True)
title = Column(String(128))
description = Column(String(8192))
offer_type = Column(String(16))
...
prices = relationship("AdPrice", backref=backref("cat_24"))
...
#hybrid_property
def last_price(self):
if self.prices:
return self.prices[-1].price
else:
return None
#last_price.expression
def last_price(cls):
# return cls.prices[-1].price
return (
select([AdPrice.price])
.where(cls.ad_id == AdPrice.ad_id)
.order_by(AdPrice.price.desc())
.limit(1)
.as_scalar()
)
When I tries to select rows and load them:
tmp = session.query(Cat_24).filter_by(offer_type='Продам').all()
df = pd.DataFrame.from_records( tmp )
I can see only regular Columns in dataframe, last_price will not be loaded.
So question, how to load hybrid_property in DataFrame?
Update ugly solution which I figured out:
def __iter__(self):
for column in self.__table__.columns:
yield column.name, getattr(self, column.name)
if self.aditional_attrs:
for attr in self.aditional_attrs:
yield attr, getattr(self, attr)
self.aditional_attrs is a tuple with hybrid_property names which I want to get, after that tmp query from the above can be converted into list of dictionaries:
tmp = list(map(dict,tmp))

How to select different type of columns dynamically to use in where clause?

I have a following table in sqlalchemy:
class FieldType(enum.Enum):
INT_FIELD = 0
FLOAT_FIELD = 1
STRING_FIELD = 2
class EAVTable(Base):
__tablename__ = 'EAVTable'
field_name = Column(Stirng, primary_key=True)
field_type = Column(Enum(FieldType))
int_field = Column(Integer)
float_field = Column(Float)
string_field = Column(String)
This is to model the EAV model which fits my business purpose.
Now to use it easily in the code I have the following hybrid_property.
#hybrid_propderty
def value(self):
if self.field_type == FieldType.INT_FIELD:
return self.int_field
...
#value.setter
def value(self, value):
if type(value) == int:
self.field_type = FieldType.INT_FIELD
self.int_field = value
...
This works fine when I try to get and set the fields in Python code. But I still have a problem:
session.query(EAVTable).filter(EAVTable.value == 123)
This does not work out of the box but I had an idea of using hybrid.expression where we use a case statement:
#value.expression
def value(cls):
return case(
[
(cls.field_type == FieldType.INT_FIELD, cls.int_field),
(cls.field_type == FieldType.FLOAT_FIELD, cls.float_field),
...
]
)
This in theory works, for example, the SQL generated for query session.query(EAVTable.value = 123 looks like:
select * from where case
when field_type = INT_FIELD then int_field
when field_type = FLOAT_FIELD then float_field
when field_type = STRING_FIELD then string_field
end = 123;
Which semantically looks like what I like, but later I find that the case expression requires all the cases have the same type, or they are cast into the same type.
I understand this is a requirement from the SQL language and has nothing to do with sqlachemy, but for more seasoned sqlalchemy user, is there any easy way to do what I want to achieve? Is there a way to walk around this constraint?

You could move the comparison inside the CASE expression using a custom comparator:
from sqlalchemy.ext.hybrid import Comparator
class PolymorphicComparator(Comparator):
def __init__(self, cls):
self.cls = cls
def __clause_element__(self):
# Since SQL doesn't allow polymorphism here, don't bother trying.
raise NotImplementedError(
f"{type(self).__name__} cannot be used as a clause")
def operate(self, op, other):
cls = self.cls
return case(
[
(cls.field_type == field_type, op(field, other))
for field_type, field in [
(FieldType.INT_FIELD, cls.int_field),
(FieldType.FLOAT_FIELD, cls.float_field),
(FieldType.STRING_FIELD, cls.string_field),
]
],
else_=False
)
class EAVTable(Base):
...
# This replaces #value.expression
#value.comparator
def value(cls):
return PolymorphicComparator(cls)
This way the common type is just boolean.

sqlalchemy: how to return list of objects joining 3 tables?

#hybrid_method
# #paginate
def investors(self, **kwargs):
"""All investors for a given Custodian"""
ind_inv_type_id = InvestorType.where(description="Individual").first().id
inv_query = Investor.with_joined(InvestorAddress, InvestmentAddress, CustodianAddress) \
.filter_by(custodians_id=self.id) \
.with_joined(Investment) \
.filter_by(investor_types_id=ind_inv_type_id)
investors = Investor.where(None, False, inv_query, **kwargs)
temp_inv_query = Investor.with_joined(CustodianInvestor, Custodian)\
.filter_by(Custodian.id==self.id)
temp_investors = Investor.where(None, False, temp_inv_query, **kwargs)
return list(set(investors + temp_investors))
# end def investors
# #auth.access_controlled
class InvestorAddress(db.Model, EntityAddressMixin):
# Metadata
__tablename__ = 'investor_addresses'
# Database Columns
investors_id = db.Column(db.ForeignKey("investors.investors_id"),
nullable=False)
investor = db.relationship("Investor", foreign_keys=[investors_id],
backref=db.backref("InvestorAddress"))
# end class InvestorAddress
class InvestmentAddress(db.Model):
"""This model differs from other EntityAddress Models because it links to either an investor_address or an custodian_address."""
# Metadata
__tablename__ = 'investment_addresses'
# Database Columns
address_types_id = db.Column(
db.ForeignKey("address_types.address_types_id"),
nullable=False)
address_type = db.relationship("AddressType",
foreign_keys=[address_types_id],
backref=db.backref("InvestmentAddress"))
investments_id = db.Column(db.ForeignKey("investments.investments_id"),
nullable=False)
investment = db.relationship("Investment",
foreign_keys=[investments_id],
backref=db.backref("InvestmentAddress"))
investor_addresses_id = db.Column(db.ForeignKey(
"investor_addresses.investor_addresses_id"))
investor_address = db.relationship("InvestorAddress",
foreign_keys=[investor_addresses_id],
backref=db.backref("InvestmentAddress"))
custodian_addresses_id = db.Column(db.ForeignKey(
"custodian_addresses.custodian_addresses_id"))
custodian_address = db.relationship("CustodianAddress",
foreign_keys=[custodian_addresses_id],
backref=db.backref("InvestmentAddress")
)
# end class InvestmentAddress
class CustodianAddress(db.Model, EntityAddressMixin):
"""Defines the relationship between a Custodian and their addresses."""
# Metadata
__tablename__ = 'custodian_addresses'
# Database Columns
custodians_id = db.Column(db.ForeignKey(
"custodians.custodians_id"), nullable=False)
custodian = db.relationship("Custodian", foreign_keys=[custodians_id],
backref=db.backref("CustodianAddress"))
# end CustodianAddress
i have an application and this function is supposed to return a list of 'investors' for a given 'Custodian'. Now when it executes i get an error: "sqlalchemy.exc.ArgumentError: mapper option expects string key or list of attributes". The error comes from the 'join' in the 'inv_query'.
I have included my 3 models that im using for the Join.

As described in the documentation provided by you. here
You should provide string arguments(table names) in with_joined. Given you have defined the relationship
Investor.with_joined('investorAddressTable', 'investmentAddressTable, 'custodianAddressTable')
In case you can use session then you can query the ORM classes directly like
session.query(Investor).join(InvestorAddress).join(InvestmentAddress).join(CustodianAddress).all() # will assume you have set the foreign key properly

Can polymorhpic_on be used over more than one column?

I have a table (and I am not allowed to change it's schema) which contains rows that are represented by three classes.
The values in columns a and b determine which class corresponds to a row.
if row.a == 'X': return X
elif row.b == 'Y': return Y
else: return Z
Obviously there is not good column for __polymorhpic_on__. Is there a way to achieve this?

polymorphic_on can be a column or SQL expression, so you just need to figure out an appropriate expression that will return an identity value.
In this case, the expression can be a case statement returning constants that will identify the subclasses. a == 'X' maps to 1, b == 'Y' to 2, etc. Note that the expression doesn't return an instance, it produces a SQL expression used for mapping the eventual instance.
case(((a == 'X', 1), (b == 'Y', 2)), else_=3)
The following is a full runnable example of this solution. The case statement has been modified to use literal_columns. It's more verbose, but doesn't require sending 4 bind params each time.
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
engine = sa.create_engine('sqlite://', echo=True)
session = Session(bind=engine)
Base = declarative_base(bind=engine)
class ThreeWay(Base):
__tablename__ = 'three_way'
id = sa.Column(sa.Integer, primary_key=True)
a = sa.Column(sa.String)
b = sa.Column(sa.String)
__mapper_args__ = {
'polymorphic_on': sa.case(
(
(a == sa.literal_column("'X'"), sa.literal_column('1')),
(b == sa.literal_column("'Y'"), sa.literal_column('2'))
),
else_=sa.literal_column('3')
)
}
class X(ThreeWay):
__mapper_args__ = {
'polymorphic_identity': 1
}
def __init__(self, **kwargs):
kwargs['a'] = 'X'
super(X, self).__init__(**kwargs)
class Y(ThreeWay):
__mapper_args__ = {
'polymorphic_identity': 2
}
def __init__(self, **kwargs):
kwargs['b'] = 'Y'
super(Y, self).__init__(**kwargs)
class Z(ThreeWay):
__mapper_args__ = {
'polymorphic_identity': 3
}
Base.metadata.create_all()
session.add_all((X(), Y(), Z()))
session.commit()
print(session.query(ThreeWay).count()) # inserted 3 generic ThreeWay rows
print(session.query(Y).count()) # 1 of them is specifically a Y row

SQLAlchemy Linear One-to-One Relationship

I have two classes mapped to two tables respectively.
Ex:
Obj 1: ID (PK), KEY (String(20))
Obj 2: ID (PK), obj_1_id (FK), value (String(20))
I would like to be able to perform obj_1.value = *val*, whereby val is stored on the secon'd table's respective column, instead of obj_1.value.value = val`.
How can I create such relationship, spread/mapped to two tables' columns?
What I want is not one-to-one (object HAS object) but rather map a column of an object to a different table.
Following is what I have tried (following the docs) and it does not work as it creates obj1.value.value = .. instead of direct column mapping
What I have tried:
class Obj1(Base):
__tablename__ == ...
id = ..
key = ..
value = relationship("Obj2", uselist=False, backref="obj1")
class Obj2(Base):
__tablename__ == ...
id = .. # PK
obj_1_id = .. # FK
value = ...

Why not just wrap the python property:
class Obj1(Base):
__tablename__ = 'obj1'
id = Column(Integer, primary_key=True)
key = Column(String(20))
_value_rel = relationship("Obj2", uselist=False, backref="obj1")
#property
def value(self):
return self._value_rel and self._value_rel.value
#value.setter
def value(self, value):
if value is None:
self._value_rel = None
elif self._value_rel is None:
self._value_rel = Obj2(value=value)
else:
self._value_rel.value = value