I am using parametrized query utilizing Text object in SQL alchemy and are getting different result.
Working example:
import sqlalchemy as sqlal
from sqlalchemy.sql import text
db_table = 'Cars'
id_cars = 8
query = text("""SELECT *
FROM Cars
WHERE idCars = :p2
""")
self.engine.execute(query, {'p2': id_cars})
Example that produces sqlalchemy.exc.ProgrammingError: (pymysql.err.ProgrammingError) (1064, "You have an error in your SQL syntax)
import sqlalchemy as sqlal
from sqlalchemy.sql import text
db_table = 'Cars'
id_cars = 8
query = text("""SELECT *
FROM :p1
WHERE idCars = :p2
""")
self.engine.execute(query, {'p1': db_table, 'p2': id_cars})
Any idea on how I can run the query with a dynamic table name that are also protected from sql injection?
I use PostgreSQL and psycopg2 backend. I was able to do it using:
from psycopg2 import sql
from sqlalchemy import engine
connection: sqlalchemy.engine.Connection
connection.connection.cursor().execute(
sql.SQL('SELECT * FROM {} where idCars = %s').format(sql.Identifier(db_table)),
(id_cars, )
)
For PyMySQL it's not supported.
You could just use the benefits of writing in python:
Library to use:
import sqlalchemy
from sqlalchemy import create_engine, MetaData, Table, func, event
from sqlalchemy.sql import text
from urllib.parse import quote_plus
connection (that I did not see in your example - here connection with sql azure):
params = urllib.parse.quote_plus(r'...')
conn_str = 'mssql+pyodbc:///?odbc_connect={}'.format(params)
engine_azure = create_engine(conn_str, echo=True)
Your example:
db_table = 'Cars'
id_cars = 8
query = text('SELECT * FROM ' + db_table + 'WHERE idCars = ' + id_cars)
connection = engine.connect()
connection.execute(query)
connection.close()
Related
I have the following SQL database details:
import sqlalchemy as sch
from config import Config
db_uri = os.environ["SQLALCHEMY_DATABASE_URI"] + os.environ["DB_NAME"]
in the env file I have these
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://*****:*****#instance-amazonaws.com:3306/'
DB_NAME = 'DB_NAME'
Now
db_engine = extrac_db.create_engine(Config.db_uri)
db_connection = db_engine.connect()
When I try this query:
db_connection.execute("select count(*) from table")
query_result = db_connection.fetchall()
It gives the following error:
AttributeError: 'Connection' object has no attribute 'fetchall'
What is the problem here!!!?
Wild guess:
query = db_connection.execute("select count(*) from table")
query_result = query.fetchall()
This might work with SQLAlchemy
import import sqlalchemy as sch
import pprint
with db_engine.begin() as conn:
qry = sch.text("select count(*) from table")
resultset = conn.execute(qry)
results_as_dict = resultset.mappings().all()
pprint(results_as_dict)
I have multiple SQLite3 databases for which the models are not available.
def index_db(name, tempdb):
print(f'{name.ljust(padding)} Indexing file: {tempdb}')
if tempdb.endswith('primary.sqlite'):
conn = sqlite3.connect(tempdb)
conn.execute('CREATE INDEX packageSource ON packages (rpm_sourcerpm)')
conn.commit()
conn.close()
How can I perform the same operation using SQLAlchemy?
I can come up with two ways to add that index through SQLAlchemy:
if you do not reflect, execute the SQL statement directly
if you reflect you table/model, add an index to it
Firstly, let's create the table to work on.
import sqlite3
con = sqlite3.connect("/tmp/73526761.db")
con.execute("CREATE TABLE t73526761 (id INT PRIMARY KEY, name VARCHAR)")
con.commit()
con.close()
Then, without reflecting, you can execute your raw SQL with the following.
import sqlalchemy as sa
engine = sa.create_engine("sqlite:////tmp/73526761.db", future=True)
with engine.begin() as con:
con.execute(sa.text("CREATE INDEX t73526761_name_idx ON t73526761 (name)"))
con.commit()
Or if you reflect the table only (SQLAlchemy core):
import sqlalchemy as sa
metadata_obj = sa.MetaData()
engine = sa.create_engine("sqlite:////tmp/73526761.db", future=True)
t73526761 = sa.Table("t73526761", metadata_obj, autoload_with=engine)
t73526761_name_idx = sa.Index("t73526761_name_idx", t73526761.c.name)
t73526761_name_idx.create(bind=engine) # emits CREATE INDEX t73526761_name_idx ON t73526761 (name)
Or if you reflect the model (SQLAlchemy orm):
import sqlalchemy as sa
from sqlalchemy import orm
Base = orm.declarative_base()
engine = sa.create_engine("sqlite:////tmp/73526761.db", future=True)
class K73526761(Base):
__table__ = sa.Table("t73526761", Base.metadata, autoload_with=engine)
t73526761_name_idx = sa.Index("t73526761_name_idx", K73526761.name)
t73526761_name_idx.create(bind=engine) # emits CREATE INDEX t73526761_name_idx ON t73526761 (name)
How to execute the following queries with sqlalchemy?
import pandas as pd
import urllib
from sqlalchemy import create_engine
from sqlalchemy.types import NVARCHAR
params = urllib.parse.quote_plus(r'DRIVER={SQL Server};SERVER=localhost\SQLEXPRESS;Trusted_Connection=yes;DATABASE=my_db;autocommit=true;MultipleActiveResultSets=True')
conn_str = 'mssql+pyodbc:///?odbc_connect={}'.format(params)
engine = create_engine(conn_str, encoding = 'utf-8-sig')
with engine.connect() as con:
con.execute('Declare #latest_date nvarchar(8);')
con.execute('SELECT #latest_date = max(date) FROM my_table')
df = pd.read_sql_query('SELECT * from my_db where date = #latest_date', conn_str)
However, an error occured:
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', '[42000] [Microsoft][ODBC SQL Server Driver][SQL Server]Must declare the scalar variable "#latest_date". (137) (SQLExecDirectW)')
How to solve this problem?
Thanks.
You don't need to declare a variable and use so many queries, you can do it just with one query:
SELECT *
FROM my_db
WHERE date = (SELECT max(date)
FROM my_db)
And then you can use, i use backticks because date is a reserved word:
with engine.connect() as con:
query="SELECT * FROM my_db WHERE `date` = (SELECT max(`date`) FROM my_db)"
df = pd.read_sql(query, con=con)
I know I can do it manually using sqlalchemy and pandas
dbschema ='myschema'
engine = create_engine('postgresql://XX:YY#localhost:5432/DB',
connect_args={'options': '-csearch_path={}'.format(dbschema )})
df = psql.read_sql('Select * from myschema."df"', con = engine)
But is it possible to do a loop and to get all the tables ?
I tried something like
tables = engine.table_names()
print(tables)
['A', 'B']
for table in tables :
table = psql.read_sql('Select * from myschema."%(name)s"', con = engine, params={'name' : table})
I get this message:
LINE 1: Select * from myschema.'A'
I guess the problem is caused by my quotes but I am not so sure.
EDIT :
So I tried the example here : Passing table name as a parameter in psycopg2
from psycopg2 import sql
try:
conn = psycopg2.connect("dbname='DB' user='XX' host='localhost' password='YY'")
except:
print ('I am unable to connect to the database')
print(conn)
cur = conn.cursor()
for table in tables :
table = cur.execute(sql.SQL("Select * from myschema.{}").format(sql.Identifier(table)))
But my tables are 'None' so I am doing something wrong but I can't see what.
I'm trying to figure out proper syntax for using sum and unnest, but I have yet to figure out the proper syntax as currently generated SQL is invalid for PostgreSQL:
sqlalchemy.exc.ProgrammingError: (ProgrammingError) could not determine polymorphic type because input has type "unknown"
'SELECT (SELECT sum(foo) AS sum_1 \nFROM unnest(%(unnest_1)s) AS foo) AS anon_1 \nFROM products' {'unnest_1': 'stock'}
Working SQL would be:
SELECT (SELECT sum(foo) AS sum_1 FROM unnest(stock) AS foo) AS anon_1 FROM products;
Example test script is here:
from sqlalchemy import Column, Integer, select
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy import func as F
from sqlalchemy.sql import column
import logging
Base = declarative_base()
db = create_engine('postgresql://scott:tiger#localhost/')
# create a configured "Session" class
Session = sessionmaker(bind=db)
session = Session()
logging.basicConfig()
class P(Base):
__tablename__ = 'products'
id = Column(Integer, primary_key=True)
stock = Column(ARRAY(Integer, dimensions=1))
total = Column(Integer)
def __init__(self, stock=[]):
self.stock = stock
self.total = 0
if __name__ == '__main__':
Base.metadata.create_all(db)
session.add_all([P([1, 2]), P([0, 0]), P([0, 2])])
session.commit()
_q = select([F.sum(column('foo'))]) \
.select_from(F.unnest('stock').alias('foo')).as_scalar()
q = select([_q]).select_from(P.__table__)
print session.execute(q).fetchall()
stock needs to be a column:
F.unnest(P.stock)
You passed a string bind argument.