I have two tables bound by a M2M relationship. Books and Writers, writers can have many books and books can have many writers.
I want to have a count property on both books and writers so I could sort them by, for example, the writer who wrote the most books.
# many to many association table
book_writer_association_table = Table('book_writer_association',Base.metadata,
Column('book_id',ForeignKey('book.id'), primary_key=True),
Column('Writer',ForeignKey('writer.id'), primary_key=True)
)
class Book(Base):
__tablename__ = 'base'
id = Column(Integer, primary_key=True)
name = Column(String)
writers = relationship(Writer,secondary=book_writer_association_table,back_populates="books")
class Writer(Base):
__tablename__ = 'writer'
id = Column(Integer, primary_key=True)
name = Column(String)
books = relationship(Book,secondery=book_writer_association_table,back_populates="writers")
#hybrid_property
def book_count(self):
return len(self.books)
#book_count.expression
def book_count(cls):
#what goes here?
I tried various approaches like detailed here:
class Foo(Base):
__tablename__ = 'foo'
id = Column(Integer, primary_key=True)
bar_id = Column(Integer, ForeignKey('bar.id'))
bar = relationship('Bar')
class Bar(Base):
__tablename__ = 'bar'
id = Column(Integer, primary_key=True)
#hybrid_property
def foo_count(self):
return object_session(self).query(Foo).filter(Foo.bar==self).count()
#foo_count.expression
def foo_count(cls):
return select([func.count(Foo.id)]).where(Foo.bar_id == cls.id).label('foo_count')
However, in this example, there are only two tables and I'm unsure how to achieve a more complicated join here. Another user suggested using column_property but I run into exactly the same problem there. I'm unsure how to further add tables to the join.
You can customize idea from here to M2M case. For this you should mention association_table in hybrid_property instead of Book table. So, you eliminate join with Book table and simplify your case to One-to-Many relation.
I came up with this solution.
from typing import List
from sqlalchemy import Column, ForeignKey, Integer, String, select, func, create_engine, Table
from sqlalchemy.ext.declarative import as_declarative
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, object_session, sessionmaker, Session
# Declare models
#as_declarative()
class Base:
pass
book_writer_association_table = Table('book_writer_association',Base.metadata,
Column('book_id',ForeignKey('book.id'), primary_key=True),
Column('writer_id',ForeignKey('writer.id'), primary_key=True)
)
class Book(Base):
__tablename__ = 'book'
id = Column(Integer, primary_key=True)
name = Column(String)
writers = relationship("Writer", secondary=book_writer_association_table, back_populates="books")
class Writer(Base):
__tablename__ = 'writer'
id = Column(Integer, primary_key=True)
name = Column(String)
books = relationship("Book", secondary=book_writer_association_table, back_populates="writers")
#hybrid_property
def book_count(self):
return object_session(self).query(book_writer_association_table).filter(book_writer_association_table.c.writer_id == self.id).count()
#book_count.expression
def book_count(cls):
return select([func.count(book_writer_association_table.c.book_id)]).where(book_writer_association_table.c.writer_id == cls.id).label('book_count')
# Load DB schema
engine = create_engine('sqlite:///sqlite3.db')
Base.metadata.create_all(engine)
SessionLocal = sessionmaker(autocommit=True, bind=engine)
db: Session = SessionLocal()
# Creating test instances
b1 = Book(name="Book 1")
b2 = Book(name="Book 2")
db.add(b1)
db.add(b2)
w1 = Writer(name="Writer 1")
w2 = Writer(name="Writer 2")
db.add(w1)
db.add(w2)
b1.writers.append(w1)
b1.writers.append(w2)
b2.writers.append(w1)
query = db.query(Writer, Writer.book_count)
print(str(query)) # checking query
print()
writers: List[Writer] = query.all() # testing query
for writer, book_count in writers:
print(f"{writer.name}: {book_count}")
Result:
> Writer 1: 2
> Writer 2: 1
I'm unsure how to further add tables to the join.
SQL from here db.query(Writer, Writer.book_count) looks clean, without any joins. So, I think you shouldn't have any problems with subsequent joins.
> SELECT writer.id AS writer_id, writer.name AS writer_name, (SELECT count(book_writer_association.book_id) AS count_1
> FROM book_writer_association
> WHERE book_writer_association.writer_id = writer.id) AS book_count
> FROM writer
Edit: If you need join Book table to provide additional filtering you can do it like this. Here I filtered book with price less than 150
from typing import List
from sqlalchemy import Column, ForeignKey, Integer, String, select, func, create_engine, Table
from sqlalchemy.ext.declarative import as_declarative
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, object_session, sessionmaker, Session
# Declare models
#as_declarative()
class Base:
pass
book_writer_association_table = Table('book_writer_association',Base.metadata,
Column('book_id',ForeignKey('book.id'), primary_key=True),
Column('writer_id',ForeignKey('writer.id'), primary_key=True)
)
class Book(Base):
__tablename__ = 'book'
id = Column(Integer, primary_key=True)
name = Column(String)
price = Column(Integer)
writers = relationship("Writer", secondary=book_writer_association_table, back_populates="books")
class Writer(Base):
__tablename__ = 'writer'
id = Column(Integer, primary_key=True)
name = Column(String)
books = relationship("Book", secondary=book_writer_association_table, back_populates="writers")
#hybrid_property
def book_count(self):
return (
object_session(self)
.query(book_writer_association_table)
.join(Book, Book.id == book_writer_association_table.c.book_id)
.filter(book_writer_association_table.c.writer_id == self.id)
.filter(Book.price > 150)
.count()
)
#book_count.expression
def book_count(cls):
# return select([func.count(book_writer_association_table.c.book_id)]).where(book_writer_association_table.c.writer_id == cls.id).label('book_count')
#
return (
select([func.count(book_writer_association_table.c.book_id)])
.join(Book, Book.id == book_writer_association_table.c.book_id)
.where(book_writer_association_table.c.writer_id == cls.id)
.filter(Book.price > 150)
.label('book_count')
)
# Load DB schema
engine = create_engine('sqlite:///sqlite3.db')
Base.metadata.create_all(engine)
SessionLocal = sessionmaker(autocommit=True, bind=engine)
db: Session = SessionLocal()
# Creating test instances
b1 = Book(name="Book 1", price=100)
b2 = Book(name="Book 2", price=200)
db.add(b1)
db.add(b2)
w1 = Writer(name="Writer 1")
w2 = Writer(name="Writer 2")
db.add(w1)
db.add(w2)
b1.writers.append(w1)
b1.writers.append(w2)
b2.writers.append(w1)
query = db.query(Writer, Writer.book_count)
print(str(query)) # checking query
print()
writers: List[Writer] = query.all() # testing query
for writer, book_count in writers:
print(f"{writer.name}: {book_count}")
query:
SELECT writer.id AS writer_id,
writer.name AS writer_name,
(SELECT count(book_writer_association.book_id) AS count_1
FROM book_writer_association
JOIN book ON book.id = book_writer_association.book_id
WHERE book_writer_association.writer_id = writer.id
AND book.price > ?) AS book_count
FROM writer
Related
For example, suppose I have three models: Book, Author, and BookAuthor where a book can have many authors and an author can have many books.
class BookAuthor(Base):
__tablename__ = 'book_authors'
author_id = Column(ForeignKey('authors.id'), primary_key=True)
book_id = Column(ForeignKey('books.id'), primary_key=True)
blurb = Column(String(50))
class Author(Base):
__tablename__ = 'authors'
id = Column(Integer, primary_key=True)
class Book(Base):
__tablename__ = 'books'
id = Column(Integer, primary_key=True)
I would like to create an authors attribute of Book which returns every author for the book and the corresponding blurb about each author. Something like this
class Book(Base):
__tablename__ = 'books'
id = Column(Integer, primary_key=True)
#authors.expression
def authors(cls):
strSQL = "my custom SQL query"
return execute(strSQL)
Demo
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import declarative_base, Session
# Make the engine
engine = create_engine("sqlite+pysqlite:///:memory:", future=True, echo=False)
# Make the DeclarativeMeta
Base = declarative_base()
class BookAuthor(Base):
__tablename__ = 'book_authors'
author_id = Column(ForeignKey('authors.id'), primary_key=True)
book_id = Column(ForeignKey('books.id'), primary_key=True)
blurb = Column(String(50))
class Author(Base):
__tablename__ = 'authors'
id = Column(Integer, primary_key=True)
class Book(Base):
__tablename__ = 'books'
id = Column(Integer, primary_key=True)
# Create the tables in the database
Base.metadata.create_all(engine)
# Make data
with Session(bind=engine) as session:
# add parents
a1 = Author()
session.add(a1)
a2 = Author()
session.add(a2)
session.commit()
# add children
b1 = Book()
session.add(b1)
b2 = Book()
session.add(b2)
session.commit()
# map books to authors
ba1 = BookAuthor(author_id=a1.id, book_id=b1.id, blurb='foo')
ba2 = BookAuthor(author_id=a1.id, book_id=b2.id, blurb='bar')
ba3 = BookAuthor(author_id=a2.id, book_id=b2.id, blurb='baz')
session.add(ba1)
session.add(ba2)
session.add(ba3)
session.commit()
# Get the authors for book with id 2
with Session(bind=engine) as session:
s = """
SELECT foo.* FROM (
SELECT
authors.*,
book_authors.blurb,
book_authors.book_id
FROM authors INNER JOIN book_authors ON authors.id = book_authors.author_id
) AS foo
INNER JOIN books ON foo.book_id = books.id
WHERE books.id = :bookid
"""
result = session.execute(s, params={'bookid':2}).fetchall()
print(result)
See that semi-nasty query at the end? It successfully returns the authors for book 2, including the corresponding blurb about each author. I would like to create a .authors attribute of my Book model that executes this query.
Figured it out. The trick was to key was to use a plain descriptor with object_session()
class Book(Base):
__tablename__ = 'books'
id = Column(Integer, primary_key=True)
#property
def authors(self):
s = """
SELECT foo.* FROM (
SELECT
authors.*,
book_authors.blurb,
book_authors.book_id
FROM authors INNER JOIN book_authors ON authors.id = book_authors.author_id
) AS foo
INNER JOIN books ON foo.book_id = books.id
WHERE books.id = :bookid
"""
result = object_session(self).execute(s, params={'bookid': self.id}).fetchall()
return result
I have three different classes I want to integrate with my database using SQLAlchemy. The relation I have is like Class2 object are main classes, each Class2 can have multiple Class1 objects which Class1 should have one to one relation with Class2. Each Class3 has two one to one relations with two different classes of type Class1.
import sqlalchemy as db
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class Class1(Base):
__tablename__ = 'Class1s'
id = Column(Integer, primary_key=True)
owner = relationship("Class2", back_populates="Class1s")
crypto_type = Column(String)
balance = Column(Integer, primary_key=True, default=0)
class Class2(Base):
__tablename__ = 'Class2s'
id = Column(Integer, primary_key=True)
class Class3(Base):
__tablename__ = 'Class3'
id = Column(Integer, primary_key=True)
amount = Column(Integer)
source = relationship("Class1")
destination = relationship("Class1")
class Database:
def __init__(self):
self.engine = db.create_engine("sqlite:///database.db", echo=True)
session = sessionmaker()
session.configure(bind=self.engine)
Base.metadata.create_all(self.engine)
if __name__ == '__main__':
db = Database()
print('Database Initialized Successfully!')
user = Class2()
This raises the following error:
sqlalchemy.exc.NoForeignKeysError: Could not determine join condition between parent/child tables on relationship Class1.owner - there are no foreign keys linking these tables. Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or specify a 'primaryjoin' expression.
Please read Basic Relationship Patters
You indeed need to define how the tables underlying your classes are linked, which is done using the ForeignKey
class Class1(Base):
__tablename__ = 'Class1s'
id = Column(Integer, primary_key=True)
owner = relationship("Class2", back_populates="Class1s")
crypto_type = Column(String)
balance = Column(Integer, primary_key=True, default=0)
class Class2(Base):
__tablename__ = 'Class2s'
id = Column(Integer, primary_key=True)
class1_id = Column(ForeignKey('Class1s.id')) # <!-- NEW **
# Class1s = relationship("Class2", back_populates="owner")
In your case you will also need to be explicit when defining the relationship between the Class3 and Class1 (I guess, Transaction and TransactionLeg).
See Handling Multiple Join Paths for more information.
class Class3(Base):
__tablename__ = 'Class3'
id = Column(Integer, primary_key=True)
amount = Column(Integer)
source_id = Column(ForeignKey('Class2s.id')) # <!-- NEW **
destination_id = Column(ForeignKey('Class2s.id')) # <!-- NEW **
source = relationship("Class1", foreign_keys=[source_id]) # <!-- MODIFIED **
destination = relationship("Class1", foreign_keys=[destination_id]) # <!-- MODIFIED **
Please note that nothing above is tested and might contain typos.
Please could someone help me with this error? I have actually been really struggling to find solid, simple examples for SQLAlchemy. Whilst there are plenty of Model examples of there is not much examples of how to use these Models.
The Error:
sqlalchemy.exc.NoForeignKeysError:
Could not determine join condition between parent/child tables on relationship Species.sc_genus
- there are no foreign keys linking these tables.
Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or specify a 'primaryjoin' expression.
The Code
from sqlalchemy import Integer, Column, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relation
Base = declarative_base()
class Genus(Base):
__tablename__ = 'genus'
id = Column(Integer, primary_key=True)
common_name = Column(String)
scientific_name = Column(String)
sc_sub_family = "sc_sub_family"
def __repr__(self):
return "<Genus(common_name='%s')>" % (self.scientific_name)
# Species is a child of Genus
class Species(Base):
__tablename__ = 'species'
id = Column(Integer, primary_key=True)
common_name = Column(String)
scientific_name = Column(String)
sc_genus = relation("Genus", backref="species")
def __repr__(self):
return "<Species(common_name='%s')>" % (self.scientific_name)
def addSpecies(session):
species = Species()
species.common_name = "House Cat"
species.scientific_name = "Felis catus"
genus = Genus()
genus.scientific_name = "Felis"
session.add(genus)
species.sc_genus = genus
session.add(species)
session.commit()
if __name__ == "__main__":
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
## A bunch of stuff to make the connection to the database work.
engine = create_engine('sqlite:///foos.db', echo=True)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
addSpecies(session)
I needed to specify a foreign key for the relationship.
class Genus(Base):
__tablename__ = 'genus'
id = Column(Integer, primary_key=True)
scientific_name = Column(String)
# sc_sub_family = "sc_sub_family"
def __repr__(self):
return "<Genus(common_name='%s')>" % (self.scientific_name)
# Species is a child of Genus
class Species(Base):
__tablename__ = 'species'
id = Column(Integer, primary_key=True)
common_name = Column(String)
scientific_name = Column(String)
sc_genus = relation("Genus", backref="species")
sc_genus_id = Column(Integer, ForeignKey('genus.id'))
def __repr__(self):
return "<Species(common_name='%s')>" % (self.scientific_name)
I have some models that I am trying to search, so I am looking to show return a result of all Parent objects, where either the Parent name is "foo" or the Child name is "foo".
I have the query:
parents = Session.query(Parent).\
join(Child_s3).\
filter(Parent.name.ilike("%foo%")).\
filter(Child_s3.name.ilike("%foo%")).\
order_by(asc(Product.name))
And the models:
class Parent(BaseSO):
__tablename__ = 'parents'
id = Column(Integer, primary_key=True)
name = Column(Unicode(100), nullable=False, unique=True)
colours = relationship('Child_s3', secondary=Parent_images, backref='Parentc')
class Child_s3(BaseSO):
__tablename__ = 'children'
id = Column(Integer, primary_key=True)
name = Column(Unicode)
Parent_images = Table(
'Parent_images', BaseSO.metadata,
Column('parent_id', Integer, ForeignKey('parents.id')),
Column('child_id', Integer, ForeignKey('children.id'))
)
The query I have shows parents with the name of "foo" but does not show any parent objects, that also have children called "foo", can anyone help build this query to search both tables for the corresponding parent objects?
This code shows how to get the result using either an explicit join or a subquery:
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
Parent_images = sa.Table(
'Parent_images', Base.metadata,
sa.Column('parent_id', sa.Integer, sa.ForeignKey('parents.id')),
sa.Column('child_id', sa.Integer, sa.ForeignKey('children.id'))
)
class Parent(Base):
__tablename__ = 'parents'
id = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.Unicode(100), nullable=False, unique=True)
colours = orm.relationship('Child_s3', secondary=Parent_images, backref='parents')
def __repr__(self):
return 'Parent(name=%s)' % self.name
__str__ = __repr__
class Child_s3(Base):
__tablename__ = 'children'
id = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.Unicode)
def __repr__(self):
return 'Child_s3(name=%s)' % self.name
__str__ = __repr__
if __name__ == '__main__':
engine = sa.create_engine('sqlite:///')
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
Session = orm.sessionmaker(bind=engine)
session = Session()
for parent, child in [('boofoo', 'spam'), ('baz', 'foobar'), ('bar', 'quux')]:
p1 = Parent(name=parent)
session.add(p1)
p1.colours.append(Child_s3(name=child))
session.commit()
print('Join')
session = Session()
q = (session.query(Parent)
.join(Child_s3, Parent.colours)
.filter(sa.or_(Parent.name.ilike('%foo%'),
Child_s3.name.ilike('%foo%'))))
for p in q.all():
print(p, p.colours)
session.commit()
print()
print('Subquery')
session = Session()
q = (session.query(Parent)
.filter(sa.or_(Parent.name.ilike('%foo%'),
Parent.colours.any(Child_s3.name.ilike('%foo%')))))
for p in q.all():
print(p, p.colours)
session.commit()
print()
The join query
q = (session.query(Parent)
.join(Child_s3, Parent.colours)
.filter(sa.or_(Parent.name.ilike('%foo%'),
Child_s3.name.ilike('%foo%'))))
generates this SQL
SELECT parents.id AS parents_id, parents.name AS parents_name
FROM parents JOIN "Parent_images" AS "Parent_images_1" ON parents.id = "Parent_images_1".parent_id JOIN children ON children.id = "Parent_images_1".child_id
WHERE lower(parents.name) LIKE lower(?) OR lower(children.name) LIKE lower(?)
The subquery
q = (session.query(Parent)
.filter(sa.or_(Parent.name.ilike('%foo%'),
Parent.colours.any(Child_s3.name.ilike('%foo%')))))
generates this SQL:
SELECT parents.id AS parents_id, parents.name AS parents_name
FROM parents
WHERE lower(parents.name) LIKE lower(?) OR (EXISTS (SELECT 1
FROM "Parent_images", children
WHERE parents.id = "Parent_images".parent_id AND children.id = "Parent_images".child_id AND lower(children.name) LIKE lower(?)))
The script produces this output from the sample data:
Join
Parent(name=baz) [Child_s3(name=foobar)]
Parent(name=boofoo) [Child_s3(name=spam)]
Subquery
Parent(name=boofoo) [Child_s3(name=spam)]
Parent(name=baz) [Child_s3(name=foobar)]
I have an insert action that is kinda complex, it boils to:
input: list of tags, book name
INSERT INTO books (book_name) VALUES (book name)
for each tag:
if tag does not exist:
INSERT INTO tags (tag_name) VALUES (tag name)
INSERT INTO books_tags (book_id, tag_id) VALUES (book id, tag id)
Can I do it somehow easier than just writing those in SQLALchemy syntax?
Short Answer: Use custom creator in the definition of association_proxy:
def _tag_find_or_create(tag_name):
tag = Tag.query.filter_by(tag_name=tag_name).first()
return tag or Tag(tag_name=tag_name)
class Book(Base):
__tablename__ = 'book'
id = Column(Integer, primary_key=True)
book_name = Column(String)
# relationship
_tags = relationship('Tag', secondary='book_tag')
tags = association_proxy('_tags', 'tag_name', creator=_tag_find_or_create)
Long Answer: The code below is a self-contained working example. Few words on the implementation:
We use Association Proxy in order to simplify many-to-many relationship. Give this whole page a good read.
In particular, we redefine the default creator function, which will first query the database/session to check for the Tag, and create one if not found.
Code:
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy import UniqueConstraint, ForeignKey
from sqlalchemy.orm import relationship, scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.associationproxy import association_proxy
# Configure test data SA
engine = create_engine('sqlite:///:memory:', echo=True)
session = scoped_session(sessionmaker(bind=engine))
Base = declarative_base(engine)
Base.query = session.query_property()
def _tag_find_or_create(tag_name):
tag = Tag.query.filter_by(tag_name=tag_name).first()
return tag or Tag(tag_name=tag_name)
class Book(Base):
__tablename__ = 'book'
id = Column(Integer, primary_key=True)
book_name = Column(String)
# relationship
_tags = relationship('Tag', secondary='book_tag')
tags = association_proxy('_tags', 'tag_name', creator=_tag_find_or_create)
class BookTag(Base):
__tablename__ = 'book_tag'
__tableargs__ = (UniqueConstraint('book_id', 'tag_id', name='book_tag_uc'),)
id = Column(Integer, primary_key=True)
book_id = Column(Integer, ForeignKey('book.id'))
tag_id = Column(Integer, ForeignKey('tag.id'))
class Tag(Base):
__tablename__ = 'tag'
id = Column(Integer, primary_key=True)
tag_name = Column(String, unique=True)
# CREATE SCHEMA
Base.metadata.create_all()
def _insert_test_data():
book = Book(book_name="book-1")
book.tags.append("fiction")
book.tags.append("history")
session.add(book)
session.commit()
assert 1 == len(Book.query.all())
assert 2 == len(Tag.query.all())
book2 = Book(book_name="book-2")
book2.tags.append("history")
book2.tags.append("crime")
session.add(book2)
session.commit()
assert 2 == len(Book.query.all())
assert 3 == len(Tag.query.all())
def _add_new_book(book_name, tags):
book = Book.query.filter(Book.book_name == book_name).first()
assert not(book), "Book with name '{}' already found [{}]".format(
book_name, book)
book = Book(book_name=book_name)
for tag in tags:
book.tags.append(tag)
session.add(book)
session.commit()
if __name__ == '__main__':
_insert_test_data()
_add_new_book('SuperMan', ['fiction', 'romance'])