I am trying create a view with SQLAlchemy with Postgresql as the underlying DB. The separate select query to create the view works well and returns results but when I use it in the create view, I get the error sqlalchemy.exc.NoSuchTableError: popular which means the view is not being selected. I am getting the error when I try to select from the view. Creating the view does not throw any error but it does not create the view. Here is my code:
from sqlalchemy import *
import sqlalchemy as db
from sqlalchemy import func
from sqlalchemy import desc
from sqlalchemy import Table
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.expression import Executable, ClauseElement
try:
engine = db.create_engine('postgresql://user:pass#localhost:5432/db_name')
connection = engine.connect()
except:
print('Error establishing DB connection')
# Import metadata
metadata = db.MetaData()
# Import articles, authors and log tables
art = db.Table('articles', metadata, autoload=True, autoload_with=engine)
aut = db.Table('authors', metadata, autoload=True, autoload_with=engine)
log = db.Table('log', metadata, autoload=True, autoload_with=engine)
class CreateView(Executable, ClauseElement):
def __init__(self, name, select):
self.name = name
self.select = select
#compiles(CreateView)
def visit_create_view(element, compiler, **kw):
return "CREATE VIEW %s AS %s" % (
element.name,
compiler.process(element.select, literal_binds=True)
)
# Method to create view with top three articles
def view_top_three():
top_three_view = CreateView('popular', db.select([art.columns.title, func.count(log.columns.path)]) \
.where(func.concat('/article/', art.columns.slug) == log.columns.path) \
.where(log.columns.path != "/") \
.group_by(log.columns.path, art.columns.title) \
.order_by(desc(func.count(log.columns.path))) \
.limit(3))
engine.execute(top_three_view)
v = Table('popular', metadata, autoload=True, autoload_with=engine)
for r in engine.execute(v.select()):
print(r)
# Call the method which creates view and selects from view
view_top_three()
Any help will be appreciated.
Since your CreateView inherits from Executable, and ClauseElement, it is not considered a data changing operation. In other words
engine.execute(top_three_view)
executes the CREATE VIEW statement and then implicitly rollbacks, when the connection is returned to the pool.
Instead it should be a subclass of DDLElement, as shown in the usage recipes wiki. Simply changing the baseclass will allow SQLAlchemy autocommit to work properly.
I found the solution. The issue was to do with autocommit. Setting autocommit to true when creating the engine solved the issue as follows:
engine = db.create_engine('postgresql://user:pass#localhost:5432/db_name').execution_options(autocommit=True)
Special mention to #ilja-everilä
Related
I have a web application that is on top of mysql. When I started it, I built the mysql database from scratch, and connected to it using pymysql.
Fast forward...
I've rewritten everything using sqlalchemy to connect to the db (non-declarative?) I can connect to the db, read the db, update, etc. I also use MySQLWorkbench to view the database in a graphic way.
My webapp has a few tables that will poll the database for changes, and update the table. For instance, I have a job queue which will update the percentage-done. Here's the kicker:
When I send a job through, I will watch the database in MySQLWorkbench. I can verify that the job is going through (I see the percentage climbing). This confirms that the webapp is writing to the database. But! On the other end, where the table is asking for the status, it is not recieving the updated information (I've done print outs in the models, as well as in the flask app, as well as the FE js).
What is going on here? Even though I can see it in MySQLWorkbench is the connection not 'releasing' or something? Here is some of my code:
#models.py
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker, Query
import os
import json
Base = automap_base()
engine = create_engine("mysql+pymysql://user:asdas758ef10d2364d54d7e8#localhost:{}/metadatacontroller".format(3306))
# reflect the tables
Base.prepare(engine, reflect=True)
EncodeQueue = Base.classes.encode_queue
db_session = scoped_session(sessionmaker(bind=engine))
class DataStore:
def __init__(self):
self.db = db_session()
#receive status
def queue_rendering_items(self):
query = self.db.query(EncodeQueue).filter(EncodeQueue.status.in_(['queue', 'rendering', 'rendering thumbnails'])).all()
return_query = []
for item in query:
row = {'artist_id': item.artist_id,
'art_id': item.art_id,
'status': item.status,
'progress': item.errors,
'artwork_title': item.artwork_title,
'artist_name': item.artist_name,
}
return_query.append(row)
return return_query
#update status
def update_render_progress(self, job_id, progress):
self.db.query(EncodeQueue).filter_by(job_id=job_id).update\({'errors': json.dumps(progress)})
self.db.commit()
(ignore the back slash in update_render_progress ... weird SO formatting)
This is how 'status' code is being called on a flask end point:
#app.route('/api/jobs', methods=["GET"])
def get_all_jobs():
db = models.DataStore()
all_art = db.queue_rendering_items()
print(all_art)
return jsonify({'data':all_art})
Lastly, the status is being updated by a different file, on a different machine:
import models
db = models.DataStore()
db.update_render_progress(job_id, {'percentage': percentage_complete, 'time_remain': render.render_estimated_seconds_remaining})
What the heck is happening?
I'm a new learner on Python and SQLAlchemy, and I met a curious problem as below.
user = Table('users', meta, autoload=True, autoload_with=engine)
then I
print(user.columns)
it works fine, the output are user.ID, user.Name, etc. But then:
Session = sessionmaker(bind=engine)
session = Session()
session.query(user).order_by(user.id)
shows error:
AttributeError: 'Table' object has no attribute 'id'
I change the "id" to "Name", it's the same error.
I also tried the filter_by method, the same error.
Why this happened?
You could use:
session.query(user).order_by(user.c.id)
Since this is the top answer when you search for this error, I will drop my solution for this as well here.
For my problem, I had to reference the class instead of the table in my models remote_side
so it had to be "Transaction.id" instead of transactions.id
class Transaction(Base):
__tablename__ = "transactions"
...
offset_transaction = relationship(
...
remote_side="Transaction.id",
)
...
For people tumbling into this after many things have changed, this is a solution that works now.
I'm also a new to this, and for me all of those answers failed to work, but using select instead of query worked. Sorry for different table and column names, but this was from an actual query.
Using MSSQL database with ODCB driver.
from sqlalchemy import select, create_engine, MetaData, Table, URL
from sqlalchemy.orm import Session
connect_url = URL.create(
'mssql+pyodbc',
host = 'server',
port = 'port',
database = 'database',
query = dict(driver = 'SQL Server Native Client 11.0'))
engine = create_engine(connect_url)
connection = engine.connect()
metadata = MetaData()
reqgroups = Table('ReqGroup', metadata, autoload_with=engine)
# Print the column names. This works and shows REQGROUPID as one of the columns
print(reqgroups.columns.keys())
with Session(engine) as session:
# Filter with columns REQGROUPID
stmt = select(reqgroups).filter_by(REQGROUPID = "Tarve_ts")
ResultSet = session.execute(stmt).fetchone()
print(ResultSet)
I'm trying to get the number of rows from a SQL Server which consists of many tables by looping through it to see how much data they contain. However, I'm not sure what will go into select_from() function. As I currently supply Unicode for table names and it raised
NoInspectionAvailable: No inspection system is available for object of type <type 'unicode'>
The code that I used was
from sqlalchemy import create_engine
import urllib
from sqlalchemy import inspect
import sqlalchemy
from sqlalchemy import select, func, Integer, Table, Column, MetaData
from sqlalchemy.orm import sessionmaker
connection_string = "DRIVER={SQL Server}; *hidden*"
connection_string = urllib.quote_plus(connection_string)
connection_string = "mssql+pyodbc:///?odbc_connect=%s" % connection_string
engine = sqlalchemy.create_engine(connection_string)
Session = sessionmaker()
Session.configure(bind=engine)
session = Session()
connection = engine.connect()
inspector = inspect(engine)
for table_name in inspector.get_table_names():
print session.query(func.count('*')).select_from(table_name).scalar()
Typically, it's a class name that refers to a class that describes the database table.
In the sqlalchemy docs, http://docs.sqlalchemy.org/en/latest/orm/tutorial.html, they have you create a base class using declarative base and then create child classes for each table you want to query. You would then pass that class name into the select_from function unquoted.
The Flask framework provides a built-in base class that is ready for use called db.Model and Django has one called models.Model.
Alternatively, you can also pass queries. I use the Flask framework typically for python so I usually initiate queries like this:
my_qry = db.session.query(Cust).filter(Cust, Cust.cust == 'lolz')
results = my_qry.all()
On a side note, if you decide to look at .NET they also have nice ORMs. Personally, I favor Entity Framework, but Linq to SQL is out there, too.
I wrote a module which is to create an empty database file
def create_database():
engine = create_engine("sqlite:///myexample.db", echo=True)
metadata = MetaData(engine)
metadata.create_all()
But in another function, I want to open myexample.db database, and create tables to it if it doesn't already have that table.
EG of the first, subsequent table I would create would be:
Table(Variable_TableName, metadata,
Column('Id', Integer, primary_key=True, nullable=False),
Column('Date', Date),
Column('Volume', Float))
(Since it is initially an empty database, it will have no tables in it, but subsequently, I can add more tables to it. Thats what i'm trying to say.)
Any suggestions?
I've managed to figure out what I intended to do. I used engine.dialect.has_table(engine, Variable_tableName) to check if the database has the table inside. IF it doesn't, then it will proceed to create a table in the database.
Sample code:
engine = create_engine("sqlite:///myexample.db") # Access the DB Engine
if not engine.dialect.has_table(engine, Variable_tableName): # If table don't exist, Create.
metadata = MetaData(engine)
# Create a table with the appropriate Columns
Table(Variable_tableName, metadata,
Column('Id', Integer, primary_key=True, nullable=False),
Column('Date', Date), Column('Country', String),
Column('Brand', String), Column('Price', Float),
# Implement the creation
metadata.create_all()
This seems to be giving me what i'm looking for.
Note that in 'Base.metadata' documentation it states about create_all:
Conditional by default, will not attempt to recreate tables already
present in the target database.
And if you can see that create_all takes these arguments: create_all(self, bind=None, tables=None, checkfirst=True), and according to documentation:
Defaults to True, don't issue CREATEs for tables already present in
the target database.
So if I understand your question correctly, you can just skip the condition.
The accepted answer prints a warning that engine.dialect.has_table() is only for internal use and not part of the public API. The message suggests this as an alternative, which works for me:
import os
import sqlalchemy
# Set up a connection to a SQLite3 DB
test_db = os.getcwd() + "/test.sqlite"
db_connection_string = "sqlite:///" + test_db
engine = create_engine(db_connection_string)
# The recommended way to check for existence
sqlalchemy.inspect(engine).has_table("BOOKS")
See also the SQL Alchemy docs.
For those who define the table first in some models.table file, among other tables.
This is a code snippet for finding the class that represents the table we want to create ( so later we can use the same code to just query it )
But together with the if written above, I still run the code with checkfirst=True
ORMTable.__table__.create(bind=engine, checkfirst=True)
models.table
class TableA(Base):
class TableB(Base):
class NewTableC(Base):
id = Column('id', Text)
name = Column('name', Text)
form
Then in the form action file:
engine = create_engine("sqlite:///myexample.db")
if not engine.dialect.has_table(engine, table_name):
# Added to models.tables the new table I needed ( format Table as written above )
table_models = importlib.import_module('models.tables')
# Grab the class that represents the new table
# table_name = 'NewTableC'
ORMTable = getattr(table_models, table_name)
# checkfirst=True to make sure it doesn't exists
ORMTable.__table__.create(bind=engine, checkfirst=True)
engine.dialect.has_table does not work for me on cx_oracle.
I am getting AttributeError: 'OracleDialect_cx_oracle' object has no attribute 'default_schema_name'
I wrote a workaround function:
from sqlalchemy.engine.base import Engine
def orcl_tab_or_view_exists(in_engine: Engine, in_object: str, in_object_name: str,)-> bool:
"""Checks if Oracle table exists in current in_engine connection
in_object: 'table' | 'view'
in_object_name: table_name | view_name
"""
obj_query = """SELECT {o}_name FROM all_{o}s WHERE owner = SYS_CONTEXT ('userenv', 'current_schema') AND {o}_name = '{on}'
""".format(o=in_object, on=in_object_name.upper())
with in_engine.connect() as connection:
result = connection.execute(obj_query)
return len(list(result)) > 0
This is the code working for me to create all tables of all model classes defined with Base class
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
class YourTable(Base):
__tablename__ = 'your_table'
id = Column(Integer, primary_key = True)
DB_URL="mysql+mysqldb://<user>:<password>#<host>:<port>/<db_name>"
scoped_engine = create_engine(DB_URL)
Base = declarative_base()
Base.metadata.create_all(scoped_engine)
I'm pretty new to sqlalchemy and oracle. I'm trying to perform a query on a pre existing table(schema or whatever they call it) in an oracle database. However, I get the following error:
Couldn't find any column information for table prefix.tablename
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, create_session
#----------------------------------------------------------------------
class NcLiVals(object):
pass
def loadSession():
""""""
db = create_engine('oracle://username:password#host:port/SID')
metadata = MetaData(db)
netc = Table('prefix.tablename, metadata, autoload=True)
netc_mapper = mapper(NcLiVals, netc)
session = create_session()
vals = session.query(NcLiVals).all()
return vals
use schema=prefix as an argument to Table
e.g.
netc = Table(tablename, metadata, autoload=True, schema=prefix)