Develop Reference

SQLAlchemy ORM autoloading with relationships

I have made a table from one service (not using sqlalchemy), and now i want to query the tables using sqlalchemy
i used a syntax like this
from app import db, engine, Base
from sqlalchemy import Table, Column, String, Integer, TIMESTAMP, Float
class Post(Base):
__table__ = Table('posts', Base.metadata,
autoload=True, autoload_with=engine)
And this is working, i can query my objects.
The annoyance is, that i can't get the properties before runtime, because (obviously), there is nothing before it compiles. Is there a way to add the types when using something like this, or do i have to assume, that they are there?

For existing tables we can reflect the table-specific information (columns, FKs, etc.) and still add our own extra ORM-specific attributes like relationships:
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
print(sa.__version__) # 1.3.20
connection_uri = "mssql+pyodbc://#mssqlLocal64"
engine = sa.create_engine(connection_uri, echo=False)
# show existing sample data
with engine.connect() as conn:
results = conn.execute(sa.text("SELECT * FROM users"))
print([dict(r) for r in results])
# [{'id': 1, 'name': 'Gord'}]
results = conn.execute(sa.text("SELECT * FROM posts"))
print([dict(r) for r in results])
# [{'id': 1, 'user_id': 1, 'post': 'Hello world!'}]
Base = declarative_base()
meta = Base.metadata
class User(Base):
__table__ = sa.Table("users", meta, autoload_with=engine)
class Post(Base):
__table__ = sa.Table("posts", meta, autoload_with=engine)
author = sa.orm.relationship("User")
def __repr__(self):
return f"<Post(id={self.id}, author_name='{self.author.name}', post='{self.post}')>"
Session = sessionmaker(bind=engine)
session = Session()
my_post = session.query(Post).get(1)
print(my_post) # <Post(id=1, author_name='Gord', post='Hello world!')>

How can I create column_properties that use a groupby?

I have this sql query:
select
rooms.*,
COUNT(DISTINCT(o.resident_id)) as resident_count,
COUNT(reviews.id) as review_count,
COUNT(photos.id) as photo_count,
AVG(reviews.rating) as mean_review
from
t_rooms rooms
JOIN
t_room_listings listings on listings.room_id = rooms.id
JOIN
t_occupancies o on o.listing_id = listings.id
LEFT JOIN
t_reviews reviews on reviews.occupancy_id = o.id
LEFT JOIN
t_photos photos on photos.occupancy_id = o.id
GROUP BY rooms.id
Which I know I can write in ORM query form as:
q = (session
.query(
Room,
func.count(func.distinct(Occupancy.resident_id)).label('resident_count'),
func.count(Review.id).label('review_count'),
func.count(Photo.id).label('photo_count'),
(
(3 + func.avg(Review.rating)) / (1 + func.count(Review.rating))
).label('bayesian_rating')
)
.select_from(
join(Room, RoomListing).join(Occupancy).outerjoin(Review).outerjoin(Photo)
)
.group_by(Room.id)
)
for room, res_ct, rev_ct, p_ct in q:
wish_that_I_could_write(room.res_ct, room.rev_ct, room.p_ct, room.score)
But how can I declare resident_count, review_count etc as column_propertys in my Room class, so that I don't need to construct this query each time?

You may achieve this result with mapping query to object like so:
class ExtendedRoom(object):
pass
# q is your query
mapper(ExtendedRoom, q.statement.alias())
for room in session.query(ExtendedRoom).all():
# now room have review_count and other attributes
print(room.review_count)
Here simplified example with column_property.
from sqlalchemy import create_engine, Column, Integer, MetaData, Table, String, func
from sqlalchemy.sql import select
from sqlalchemy.orm import sessionmaker, mapper, column_property
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=engine)
Base = declarative_base()
session = Session()
metadata = MetaData()
room = Table('room', metadata,
Column('id', Integer, primary_key=True),
Column('name', String),
Column('num', Integer, default=0),
)
metadata.create_all(engine)
statement = select([room]).group_by(room.c.id).alias()
class Room(object):
pass
mapper(Room, statement, properties={
'count': column_property(func.count(statement.c.num)),
})
print(session.query(Room).all())

SQLAlchemy generates a different query for counts than expected with a polymorphic_identity

I have a query that I create, it looks like
items = Session.query(Widgets.id).filter_by(
state=WidgetStates.NEW
)
when I look at the str representation of it I see this as the planned query
str(items)
'SELECT widgets.id AS widgets_guid \nFROM widgets \nWHERE widgets.state = %(state_1)s'
However, when I execute the query to get a count with echo=True I see a different query being exected:
items.count()
2014-08-09 11:59:48,875 INFO sqlalchemy.engine.base.Engine SELECT count(*) AS count_1
FROM widgets, (SELECT widgets.id AS widgets_id
FROM widgets
WHERE widgets.state = %(state_1)s) AS anon_1
WHERE widgets.type IN (%(type_1)s)
The problem is that it's going to count the entire widgets table where type equals "FOO_WIDGET". But it's not going to filter the count by state it as I would have expected it to.
I think the issue relates to the Widget model having a polymorphic_identity discriminator applied to it:
class Widget(Model):
class types(object):
FOO_WIDGET = 'foo'
__mapper_args__ = {
'polymorphic_identity': Widget.types.FOO_WIDGET
}
But the issue is it's not using the items query to count, it's using two different tables to get the selected count and one of them does not have any filtering on it. How do I get this query to work as expected?
Runnable Example
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Table, Column, Unicode, Integer, create_engine, MetaData, func
from sqlalchemy.orm import scoped_session, sessionmaker
metadata = MetaData()
Base = declarative_base(metadata=metadata)
widgets = Table(
'widgets', metadata,
Column('id', Integer, primary_key=True),
Column('type', Unicode),
Column('state', Unicode)
)
class Widget(Base):
__table__ = widgets
class types(object):
FOO_WIDGET = 'foo'
BAR_WIDGET = 'bar'
__mapper_args__ = {
'polymorphic_on': widgets.c.type,
}
class FooWidget(Widget):
__mapper_args__ = {
'polymorphic_identity': Widget.types.FOO_WIDGET
}
db_engine = create_engine('sqlite:///:memory:', echo=True)
Session = scoped_session(sessionmaker())
Session.configure(bind=db_engine)
metadata.create_all(db_engine)
items = Session.query(FooWidget.id).filter_by(
state='new'
)
print str(items)
print 'i expect the next statement to print something approximating:'
print '''
select count(*) from widgets where type = 'foo' and state = 'new'
'''
print items.count()
# What this actually prints
'''
2014-08-28 09:55:15,055 INFO sqlalchemy.engine.base.Engine SELECT count(*) AS count_1
FROM widgets, (SELECT widgets.id AS widgets_id
FROM widgets
WHERE widgets.state = ?) AS anon_1
WHERE widgets.type IN (?)
'''
To run this example you need SQLAlchemy (Tested here with SQLA 0.9.7, in my actual app it's 0.7.x, bug exists in both versions)

A Deduplication DB schema with sqlalchemy - How to represent a group with ORM semantics?

I'm trying to create a simple representation for an entity deduplication schema using mysql, and using sqlalchemy for programmatic access.
I'm trying to achieve a specific effect which I think is kind of a self-referential query but i'm not sure:
Essentially I have an 'entities' table (with unique entity_id) and an associated Entity object,
and then an entity_groups table which (for simplicity) has a 'group_id' and 'entity_id' columns, so that I 'register' an entity with a group by creating a row for that relation.
this table too is associated with an ORM object - EntityGroup.
Question is, how do i get the EntityGroup object reference all entities in the group?
I expect I need to write something like:
mapper(EntityGroup, entity_groups_table,
properties={
'entities': relationship(
Entity,
.... ?
)
},
and i'm alittle fuzzy on the details. Basically I need all the rows in entity_groups that have the same group_id as the row represented by the object. And then I need to materialize
all the Entity objects associated those rows' entity_id column. This sounds like something achievable by a more verbose Query() operation in sqlalchemy, but i'm not sure how to combine that with the relationship() construct (if at all - perhaps go manual? )
Any help will be useful, I hope I was clear and to the point

You really should not do it using a Query, as if you configure the relationships properly you will get this automatically. Assuming that you use entity_group table solely to store the relationship and nothing else, you should just configure many-to-many relationship as documented. Fully working example should help:
from sqlalchemy import create_engine, Column, Integer, String, MetaData, ForeignKey, Table
from sqlalchemy.orm import relationship, mapper, scoped_session, sessionmaker, backref
from sqlalchemy.ext.associationproxy import association_proxy
# Configure test DB
engine = create_engine(u'sqlite:///:memory:', echo=False)
session = scoped_session(sessionmaker(bind=engine, autoflush=False))
metadata = MetaData()
# tables
entities_table = Table('entities', metadata,
Column('entity_id', Integer, primary_key=True),
)
groups_table = Table('groups', metadata,
Column('group_id', Integer, primary_key=True),
)
entity_groups_table = Table('entity_groups', metadata,
Column('entity_id', Integer, ForeignKey('entities.entity_id'), primary_key=True),
Column('group_id', Integer, ForeignKey('groups.group_id'), primary_key=True),
)
# object model
class Group(object):
def __repr__(self): return "<Group: %d>" % (self.group_id,)
class Entity(object):
def __repr__(self): return "<Entity: %d>" % (self.entity_id,)
# mappers
mapper(Group, groups_table)
mapper(Entity, entities_table,
properties={'groups': relationship(Group, secondary=entity_groups_table, backref='entities')},
)
# create db schema
metadata.create_all(engine)
# == TESTS
# create entities
e1 = Entity()
e2 = Entity()
g1 = Group()
g2 = Group()
g3 = Group()
g1.entities.append(e1)
g2.entities.append(e2)
g3.entities.append(e1)
g3.entities.append(e2)
session.add(e1)
session.add(e2)
session.commit()
# query...
session.expunge_all()
# check Peter
for g in session.query(Group).all():
print "group: ", g, " has ", g.entities
should produce something like:
group: <Group: 1> has [<Entity: 1>]
group: <Group: 2> has [<Entity: 1>, <Entity: 2>]
group: <Group: 3> has [<Entity: 2>]

Setting up relations/mappings for a SQLAlchemy many-to-many database

I'm new to SQLAlchemy and relational databases, and I'm trying to set up a model for an annotated lexicon. I want to support an arbitrary number of key-value annotations for the words which can be added or removed at runtime. Since there will be a lot of repetition in the names of the keys, I don't want to use this solution directly, although the code is similar.
My design has word objects and property objects. The words and properties are stored in separate tables with a property_values table that links the two. Here's the code:
from sqlalchemy import Column, Integer, String, Table, create_engine
from sqlalchemy import MetaData, ForeignKey
from sqlalchemy.orm import relation, mapper, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine('sqlite:///test.db', echo=True)
meta = MetaData(bind=engine)
property_values = Table('property_values', meta,
Column('word_id', Integer, ForeignKey('words.id')),
Column('property_id', Integer, ForeignKey('properties.id')),
Column('value', String(20))
)
words = Table('words', meta,
Column('id', Integer, primary_key=True),
Column('name', String(20)),
Column('freq', Integer)
)
properties = Table('properties', meta,
Column('id', Integer, primary_key=True),
Column('name', String(20), nullable=False, unique=True)
)
meta.create_all()
class Word(object):
def __init__(self, name, freq=1):
self.name = name
self.freq = freq
class Property(object):
def __init__(self, name):
self.name = name
mapper(Property, properties)
Now I'd like to be able to do the following:
Session = sessionmaker(bind=engine)
s = Session()
word = Word('foo', 42)
word['bar'] = 'yes' # or word.bar = 'yes' ?
s.add(word)
s.commit()
Ideally this should add 1|foo|42 to the words table, add 1|bar to the properties table, and add 1|1|yes to the property_values table. However, I don't have the right mappings and relations in place to make this happen. I get the sense from reading the documentation at http://www.sqlalchemy.org/docs/05/mappers.html#association-pattern that I want to use an association proxy or something of that sort here, but the syntax is unclear to me. I experimented with this:
mapper(Word, words, properties={
'properties': relation(Property, secondary=property_values)
})
but this mapper only fills in the foreign key values, and I need to fill in the other value as well. Any assistance would be greatly appreciated.

Simply use Dictionary-Based Collections mapping mapping - out of the box solution to your question. Extract from the link:
from sqlalchemy.orm.collections import column_mapped_collection, attribute_mapped_collection, mapped_collection
mapper(Item, items_table, properties={
# key by column
'notes': relation(Note, collection_class=column_mapped_collection(notes_table.c.keyword)),
# or named attribute
'notes2': relation(Note, collection_class=attribute_mapped_collection('keyword')),
# or any callable
'notes3': relation(Note, collection_class=mapped_collection(lambda entity: entity.a + entity.b))
})
# ...
item = Item()
item.notes['color'] = Note('color', 'blue')
print item.notes['color']
Or try the solution for Inserting data in Many to Many relationship in SQLAlchemy. Obviously you have to replace the list logic with the dict one.
Ask question author to post hist final code with associationproxy, which he mentioned he used in the end.

There is very similar question with slight interface difference. But it's easy to fix it by defining __getitem__, __setitem__ and __delitem__ methods.

Comment for Brent, above:
You can use session.flush() instead of commit() to get an id on your model instances. flush() will execute the necessary SQL, but will not commit, so you can rollback later if needed.

I ended up combining Denis and van's posts together to form the solution:
from sqlalchemy import Column, Integer, String, Table, create_engine
from sqlalchemy import MetaData, ForeignKey
from sqlalchemy.orm import relation, mapper, sessionmaker
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
meta = MetaData()
Base = declarative_base(metadata=meta, name='Base')
class PropertyValue(Base):
__tablename__ = 'property_values'
WordID = Column(Integer, ForeignKey('words.id'), primary_key=True)
PropID = Column(Integer, ForeignKey('properties.id'), primary_key=True)
Value = Column(String(20))
def _property_for_name(prop_name):
return s.query(Property).filter_by(name=prop_name).first()
def _create_propval(prop_name, prop_val):
p = _property_for_name(prop_name)
if not p:
p = Property(prop_name)
s.add(p)
s.commit()
return PropertyValue(PropID=p.id, Value=prop_val)
class Word(Base):
__tablename__ = 'words'
id = Column(Integer, primary_key=True)
string = Column(String(20), nullable=False)
freq = Column(Integer)
_props = relation(PropertyValue, collection_class=attribute_mapped_collection('PropID'), cascade='all, delete-orphan')
props = association_proxy('_props', 'Value', creator=_create_propval)
def __init__(self, string, freq=1):
self.string = string
self.freq = freq
def __getitem__(self, prop):
p = _property_for_name(prop)
if p:
return self.props[p.id]
else:
return None
def __setitem__(self, prop, val):
self.props[prop] = val
def __delitem__(self, prop):
p = _property_for_name(prop)
if p:
del self.props[prop]
class Property(Base):
__tablename__ = 'properties'
id = Column(Integer, primary_key=True)
name = Column(String(20), nullable=False, unique=True)
def __init__(self, name):
self.name = name
engine = create_engine('sqlite:///test.db', echo=False)
Session = sessionmaker(bind=engine)
s = Session()
meta.create_all(engine)
The test code is as follows:
word = Word('foo', 42)
word['bar'] = "yes"
word['baz'] = "certainly"
s.add(word)
word2 = Word('quux', 20)
word2['bar'] = "nope"
word2['groink'] = "nope"
s.add(word2)
word2['groink'] = "uh-uh"
del word2['bar']
s.commit()
word = s.query(Word).filter_by(string="foo").first()
print word.freq, word['baz']
# prints 42 certainly
The contents of the databases are:
$ sqlite3 test.db "select * from property_values"
1|2|certainly
1|1|yes
2|3|uh-uh
$ sqlite3 test.db "select * from words"
1|foo|42
2|quux|20
$ sqlite3 test.db "select * from properties"
1|bar
2|baz
3|groink