I've began learning sqlalchemy and the script below shows how far I've gotten. I've created a class to start the database, a class that creates a table containing games and their ID's and a class that is used to create tables for each individual external data source.
I realized I need to incorporate foreign keys but upon doing so I get the error below. It's very confusing because I'm fairly certain the MLBGamelist table was created. Any help on this wuld be greatly appreciated.
sqlalchemy.exc.NoReferencedTableError: Foreign key associated with column 'CaesarsGamelist.game_ref' could not find table 'MLBGamelist' with which to generate a foreign key to target column 'game_ref'
class DBControl:
def __init__(self,mem):
print('>>>> [MAIN]: INITIALIZING MAIN DATABASE CONNECTION')
self.engine = create_engine(memory[mem], echo=False)
self.inspector = inspect(self.engine)
self.db_connection = self.engine.connect()
self.create_session = sessionmaker(bind=self.engine)
class GamelistMLBControl:
def __init__(self,book,db_control):
self.table_name = f'{book}Gamelist'
self.db_control = db_control
def commit_entry(self,site_data):
write_session = scoped_session(self.db_control.create_session)
insert_stmt = insert(self.check_table()).values(site_data)
write_session.execute(insert_stmt)
write_session.commit()
write_session.remove()
def check_table(self):
metadata = MetaData(bind=self.db_control.engine)
if self.table_name not in self.db_control.inspector.get_table_names():
table_name = Table(
str(self.table_name),
metadata,
Column("event_id", Integer, primary_key=True),
Column("game_ref", String),
Column("game_datetime", Integer),
Column("book", String),
)
metadata.create_all(self.db_control.db_connection)
print(f'> [{self.table_name}]: Table created')
else:
metadata.reflect(self.db_control.engine)
print(f'> [{self.table_name}]: Table exists')
return Table(table_name, metadata, autoload=True)
class GamelistControl:
def __init__(self,book,db_control):
self.table_name = f'{book}Gamelist'
self.db_control = db_control
def commit_entry(self,site_data):
write_session = scoped_session(self.db_control.create_session)
insert_stmt = insert(self.check_table()).values(site_data)
write_session.execute(insert_stmt)
write_session.commit()
write_session.remove()
def check_table(self):
metadata = MetaData(bind=self.db_control.engine)
if self.table_name not in self.db_control.inspector.get_table_names():
table_name = Table(
str(self.table_name),
metadata,
Column("event_id", Integer, primary_key=True),
Column("game_ref", String, ForeignKey('MLBGamelist.game_ref')),
Column("game_datetime", Integer, ForeignKey('MLBGamelist.game_datetime')),
Column("book", String),
)
metadata.create_all(self.db_control.db_connection)
print(f'> [{self.table_name}]: Table created')
else:
metadata.reflect(self.db_control.engine)
print(f'> [{self.table_name}]: Table exists')
return Table(table_name, metadata, autoload=True)
SQLAlchemy stores details of tables in MetaData objects when a table is created or reflected. Binding an engine to a MetaData object does not in itself cause any table details to be stored. In the code in the question, each class creates a separate MetaData instance, hence the error occurs because the Metadata instance created in GamelistControl doesn't have details of the table created in GamelistMLBControl.
To solve the problem, either call metadata.reflect() after creating the MetaData instance in GamelistControl or create a single MetaData as an attribute of DBControl and use that to create tables in both classes.
Related
I've been reading various examples from SQLAlchemy documentation for cascade deletes, but nothing I try seems to work. Below is some sample code adapted from that documentation, but using back_populates instead of backref, as I understand that backref is being deprecated.
In the "main" section below, I would expect that deleting the order that "contains" the items would delete the items as well, but that does not happen. Obviously I don't understand something about how to configure these tables... what is it?
# third party imports
from sqlalchemy import Column, ForeignKey, Integer, String, create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy_utils import create_database, database_exists
Base = declarative_base()
class Order(Base):
__tablename__ = "business_order"
id = Column(Integer, primary_key=True)
name = Column(String(32))
items = relationship(
"Item", back_populates="order", cascade="all, delete, delete-orphan"
)
class Item(Base):
__tablename__ = "business_item"
id = Column(Integer, primary_key=True)
name = Column(String(32))
order_id = Column(Integer, ForeignKey("business_order.id"))
order = relationship("Order", back_populates="items")
def get_session(url="sqlite:///:memory:", create_db=True):
"""Get a SQLAlchemy Session instance for input database URL.
:param url:
SQLAlchemy URL for database, described here:
http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls.
:param create_db:
Boolean indicating whether to create database from scratch.
:returns:
Sqlalchemy Session instance.
"""
# Create a sqlite in-memory database engine
if not database_exists(url):
if create_db:
create_database(url)
else:
msg = (
"Database does not exist, will not create without "
"create_db turned on."
)
print(msg)
return None
connect_args = {}
engine = create_engine(url, echo=False, connect_args=connect_args)
Base.metadata.create_all(engine)
# create a session object that we can use to insert and
# extract information from the database
Session = sessionmaker(bind=engine, autoflush=False)
session = Session()
return session
if __name__ == "__main__":
sqlite_url = "sqlite:///test_sqlite.db"
session = get_session(sqlite_url)
order = Order(name="order1")
session.add(order)
item = Item(order_id=order.id, name="item1")
session.add(item)
session.commit()
session.delete(order) # should delete items too, right?
session.commit()
orders = session.query(Order).all()
print(len(orders)) # this returns 0 as expected
items = session.query(Item).all()
print(len(items)) # this returns 1, why?
Order has an (implicit) autoincrement PK. When you do
order = Order(name="order1")
session.add(order)
order.id is None. Therefore, when you do
item = Item(order_id=order.id, name="item1")
item.order_id will also be None, so item is actually not associated with order. Therefore, the delete doesn't cascade.
order doesn't get its id until .flush() (or .commit()) is called. So you could either do
order = Order(name="order1")
session.add(order)
session.flush() # !
item = Item(order_id=order.id, name="item1")
session.add(item)
session.commit()
or do
order = Order(name="order1", items=[Item(name="item1")])
session.add(order)
session.commit()
session.delete(order) # should delete items too, right?
session.commit()
orders = session.query(Order).all()
print(len(orders)) # this returns 0 as expected
items = session.query(Item).all()
print(len(items)) # this also returns 0 as expected
Is there a way to generate db model dynamically based on the columns in the database table for Flask SQLAlchemy?
I have an application to display data in from a database table, but the column name would change at times and break my app. I would like to have a way to generate the data model dynamically based on the actual column names in the database.
I currently explicitly declare all the columns as below.
class MyDbModel(db.Model):
__tablename__ = 'my_table'
id = db.Column('id', db.NVARCHAR(length=300), primary_key=True)
name= db.Column('name', db.NVARCHAR(length=300),
nullable=True)
I'm not very familiar with sqlalchamy, I have tried the following but is getting an error as below, and I'm not sure if this is the right way to do it.
could not assemble any primary key columns for mapped table
from sqlalchemy import Table, Column
from sqlalchemy.orm import mapper
db = SQLAlchemy()
class MyDbModel(db.Model):
__tablename__ = 'my_table'
def __init__(self):
#helper function to get column headers of a db table
table_cols = get_headers_or_columns(
'my_table'
)
t = Table(
'my_table', db.metadata,
Column('id', db.NVARCHAR(length=300), primary_key=True),
*(Column(table_col, db.NVARCHAR(length=300)) for table_col in table_cols)
)
mapper(self, t)
Any help is appreciated!
I figured this out, in case anyone needed. It's really simple, use a helper function to get the column headers of your table, then just use setattr to set the attributes of the class.
from app import db
class MyDbModel(db.Model):
pass
def map_model_attrs(model, table):
"""
:param model: your db Model Class
:param table your db table name
"""
table_cols = get_headers_or_columns(table)
for col in table_cols:
setattr(
model, col, db.Column(col, db.NVARCHAR(length=300),
nullable=True)
)
map_mode_attrs(MyDbModel, 'my_table')
I can create a temporary table this way:
session.execute("CREATE TABLE temptable SELECT existingtable.id, "
"existingtable.column2 FROM existingtable WHERE existingtable.id<100000")
but the new table is unreadable because it says it has no primary key. existingtable.id is the primary key of exisitingtable, so I expected it to get the same treatment in the temp table.
However, I would rather find some ORM way of doing this anyway. Given:
temp_table = Table('temptable', metadata,
Column('id', Integer, primary_key=True),
Column('column2', Integer),
useexisting=True )
class TempTable(object):
pass
mapper(TempTable, temp_table)
temp_table.create(bind=session.bind, checkfirst=True)
if session.query(TempTable).delete(): #make sure it's empty
session.commit()
How can I populate temp_table with some selected contents of existingtable without doing 100000 session.query.add(TempTable(...)) commands? Or is there a way of creating the table from a query similar to the plain SQL version above?
It's not exactly ORM, but to create the table initially, I'd clone the table structure (see cloneTable in the example below). For copying the data, I then would use the InsertFromSelect example.
Edit: Since version 0.8.3, SqlAlchemy supports Insert.from_select() out of the box. Hence the InsertFromSelect class and the respective visitor in the example below can be directly replaced and are no longer needed. I leave the original example unchanged for historic reasons.
Here is a working example
from sqlalchemy import Table
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.expression import UpdateBase
class InsertFromSelect(UpdateBase):
def __init__(self, table, select):
self.table = table
self.select = select
#compiles(InsertFromSelect)
def visit_insert_from_select(element, compiler, **kw):
return "INSERT INTO %s %s" % (
compiler.process(element.table, asfrom=True),
compiler.process(element.select)
)
def cloneTable(name, table, metadata):
cols = [c.copy() for c in table.columns]
constraints = [c.copy() for c in table.constraints]
return Table(name, metadata, *(cols + constraints))
# test data
from sqlalchemy import MetaData, Column, Integer
from sqlalchemy.engine import create_engine
e = create_engine('sqlite://')
m = MetaData(e)
t = Table('t', m, Column('id', Integer, primary_key=True),
Column('number', Integer))
t.create()
e.execute(t.insert().values(id=1, number=3))
e.execute(t.insert().values(id=9, number=-3))
# create temp table
temp = cloneTable('temp', t, m)
temp.create()
# copy data
ins = InsertFromSelect(temp, t.select().where(t.c.id>5))
e.execute(ins)
# print result
for r in e.execute(temp.select()):
print(r)
I am developing an extension to an existing app which uses sqlalchemy 0.6.
The app has sqlalchemy tables created the non-declarative way. I am trying to create in my extension a new table with a foreign key column pointing at the primary key of the main table in the application database and I am creating it declaratively.
This all works fine, with the table created once the extension is loaded, and with no complaints at all. My table prints out and demonstrates that new rows have been added ok.
What I want and think is possible (but don't know as I have never used sql or any other database) is for the corresponding row in my table to be deleted when the row in the app's main table with the corresponding foreign key is deleted.
So far, and with many permutations having been tried, nothing has worked. I thought that with a backref set and with a relation defined with delete being cascaded, there shouldn't be a problem. Because the new table is defined in an extension which should just plugin, I don't want to edit the code in the main app at all, at least that is my goal. One of the problems that I have, though, is that the main app table that I want to reference, has no member variables defined in its class, does not declare its primary key in its mapper and only has the primary key declared in the table. This makes it difficult to create a relation(ship) clause, the first argument of which must be to a class or mapper (in this case neither of which have the primary key declared).
Is there any way of achieving this?
ps - here is some of the code that I am using. LocalFile is the declarative class. All the connection details are taken care of by the main application.
if not self.LocalFile.__table__.exists(bind=Engine):
self.LocalFile__table__.create(bind=Engine)
Here is the LocalFile class - Base is a declarative base class with bind=Engine passed in the constructor:
class LocalFile(Base):
__tablename__ = 'local_file'
_id = Column(Integer, Sequence('local_file_sequence', start=1, increment=1), primary_key=True)
_filename = Column(String(50), nullable=False)
_filepath = Column(String(128), nullable=False)
_movieid = Column(Integer, ForeignKey(db.tables.movies.c.movie_id, onupdate='CASCADE', ondelete='CASCADE'))
#movies = relation(db.Movie, backref="local_file", cascade="all")
#property
def filename(self):
return self._filename
#filename.setter
def filename(self, filename):
self._filename = filename
#property
def filepath(self):
return self._filepath
#filepath.setter
def filepath(self, filepath):
self._filepath = filepath
#property
def movieid(self):
return self._movieid
#movieid.setter
def movieid(self, movieid):
self._movieid = movieid
#property
def id(self):
return self._id
#id.setter
def id(self, id):
self._id = id
filename = synonym('_filename', descriptor=filename)
movieid = synonym('_movieid', descriptor=movieid)
filepath = synonym('_filepath', descriptor=filepath)
id = synonym('_id', descriptor=id)
def __init__(self, filename, filepath, movieid):
self._filename = filename
self._filepath = filepath
self._movieid = movieid
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.filename, self.filepath, self.movieid)
Edit:
The backend is sqlite3. Below is the code from the creation of the table produced by using the echo command (thanks for pointing that out, it's very useful - already I suspect that the existing application is generating far more sql than is necessary).
Following the reported sql table creation is the code generated when a row is removed. I personally can't see any statement that references the possible deletion of a row in the local file table, but I know very little sql currently. Thanks.
2011-12-29 16:29:18,530 INFO sqlalchemy.engine.base.Engine.0x...0650
CREATE TABLE local_file (
_id INTEGER NOT NULL,
_filename VARCHAR(50) NOT NULL,
_filepath VARCHAR(128) NOT NULL,
_movieid INTEGER,
PRIMARY KEY (_id),
FOREIGN KEY(_movieid) REFERENCES movies (movie_id) ON DELETE CASCADE ON UPDATE CASCADE
)
2011-12-29T16:29:18: I: sqlalchemy.engine.base.Engine.0x...0650(base:1387):
CREATE TABLE local_file (
_id INTEGER NOT NULL,
_filename VARCHAR(50) NOT NULL,
_filepath VARCHAR(128) NOT NULL,
_movieid INTEGER,
PRIMARY KEY (_id),
FOREIGN KEY(_movieid) REFERENCES movies (movie_id) ON DELETE CASCADE ON UPDATE CASCADE
)
2011-12-29 16:29:18,534 INFO sqlalchemy.engine.base.Engine.0x...0650 ()
2011-12-29T16:29:18: I: sqlalchemy.engine.base.Engine.0x...0650(base:1388): ()
2011-12-29 16:29:18,643 INFO sqlalchemy.engine.base.Engine.0x...0650 COMMIT
2011-12-29T16:29:18: I: sqlalchemy.engine.base.Engine.0x...0650(base:1095): COMMIT
for row in table produces the following for the two tables:
the local file table:
(, u' 310 To Yuma')
(, u' Ravenous')
the movie table in the existing app:
(, u'IMDb - 3:10 to Yuma')
(, u'Ravenous')
The code when deleting a row is so long that I cannot include it here (200 lines or so - isn't that a little too many for deleting one row?), but it makes no reference to deleting a row in the localfile table. There are statements like:
2011-12-29 17:09:17,141 INFO sqlalchemy.engine.base.Engine.0x...0650 UPDATE movies SET poster_md5=?, updated=? WHERE movies.movie_id = ?
2011-12-29T17:09:17: I: sqlalchemy.engine.base.Engine.0x...0650(base:1387): UPDATE movies SET poster_md5=?, updated=? WHERE movies.movie_id = ?
2011-12-29 17:09:17,142 INFO sqlalchemy.engine.base.Engine.0x...0650 (None, '2011-12-29 17:09:17.141019', 2)
2011-12-29T17:09:17: I: sqlalchemy.engine.base.Engine.0x...0650(base:1388): (None, '2011-12-29 17:09:17.141019', 2)
2011-12-29 17:09:17,150 INFO sqlalchemy.engine.base.Engine.0x...0650 DELETE FROM posters WHERE posters.md5sum = ?
2011-12-29T17:09:17: I: sqlalchemy.engine.base.Engine.0x...0650(base:1387): DELETE FROM posters WHERE posters.md5sum = ?
2011-12-29 17:09:17,157 INFO sqlalchemy.engine.base.Engine.0x...0650 (u'083841e14b8bb9ea166ea4b2b976f03d',)
In SQLite you must turn on support for foreign keys explicitly or it just ignores any SQL related to foreign keys.
engine = create_engine(database_url)
def on_connect(conn, record):
conn.execute('pragma foreign_keys=ON')
from sqlalchemy import event
event.listen(engine, 'connect', on_connect)
I am trying to store some simulation measurements (times and values) using sqlalchemy. Here are the relevant table definitions. If there is a more sensible table definition, I'd love to see it.
from sqlalchemy import create_engine, schema, orm
engine = create_engine('sqlite:///:memory:', echo=True)
metadata = schema.MetaData(bind=engine)
container_table = schema.Table('containers', metadata,
schema.Column('id', schema.types.Integer, primary_key=True))
measurement_table = schema.Table('measurements', metadata,
schema.Column('id', schema.types.Integer, primary_key=True),
schema.Column('container_id', schema.types.Integer,
schema.ForeignKey('containers.id')),
schema.Column('time', schema.types.Float),
schema.Column('value', schema.types.Float))
metadata.create_all()
The times will be unique for each container, and the below properties should be ordered by time.
I would like to be able to both read and assign these properties:
c = Container()
times = range(10)
values = [t**2 for t in times]
c.times = times
c.values = values
But I don't know how to do the mapping. I assume that if it's possible, it will look something like this:
class Container(object):
times = some_sort_of_proxy()
values = some_sort_of_proxy()
orm.mapper(Container, container_table, properties={
# Magic
})
How do I go about doing this? Is this a reasonable mapping, or do I need to have a different underlying table structure?
class EmailAddress(object):
#property
def email(self):
return self._email
#email.setter
def email(self, email):
self._email = email
mapper(EmailAddress, addresses_table, properties={
'_email': addresses_table.c.email
})