I have an issue that I could not manage. I am developing an FastApi SqlAlchemy App connected to SQL Server. I have created my models and views on locally. But when I deployed an server The following error occurred. I have tried Ubuntu, Linux, Windows VMs but the situation is same.
from sqlalchemy import create_engine
from sqlalchemy import Integer, String, Text, Column, DATE
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy import select, MetaData, Table
from sqlalchemy.orm import scoped_session, sessionmaker, Query
from sqlalchemy import ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.dialects.mssql import pymssql
import urllib
import datetime
from sqlalchemy.sql.sqltypes import Date
params = urllib.parse.quote_plus("DRIVER={ODBC Driver 17 for SQL Server};"
"SERVER=MYSERVER"
"UID=MYUSERNAME"
"PWD=MYPASSWD"
"DATABASE=MYDB"
"Trusted_Connection=no;"
"Integrated Security=false;")
engine = create_engine("mssql+pyodbc:///?odbc_connect={}".format(params))
engine.connect()
Base = declarative_base()
metadata = Base.metadata
session = scoped_session(sessionmaker(bind=engine))
class Users(Base):
__table__ = Table('astencube_users', Base.metadata, autoload=True, autoload_with=engine)
print(session.query(*Users.__table__.columns).all())
This is a part of my app I am trying to print a table but when I try to make any query I got error below. On my local machine there is no error when I deploy any machine it appears. Also the error gives me a parameter which represents my table but when I check the other tables still problem occurs. On my local machine there is no problem. Why is it happening?
Traceback (most recent call last):
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1799, in _execute_context
self.dialect.do_execute(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Ambiguous column name 'constraint_schema'. (209) (SQLExecDirectW)")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "file.py", line 35, in <module>
class Users(Base):
File "file.py", line 36, in Users
__table__ = Table('astencube_users', Base.metadata, autoload=True, autoload_with=engine)
File "<string>", line 2, in __new__
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/util/deprecations.py", line 298, in warned
return fn(*args, **kwargs)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 607, in __new__
metadata._remove_table(name, schema)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 602, in __new__
table._init(name, metadata, *args, **kw)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 677, in _init
self._autoload(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/sql/schema.py", line 712, in _autoload
conn_insp.reflect_table(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/reflection.py", line 795, in reflect_table
self._reflect_fk(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/reflection.py", line 948, in _reflect_fk
fkeys = self.get_foreign_keys(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/reflection.py", line 564, in get_foreign_keys
return self.dialect.get_foreign_keys(
File "<string>", line 2, in get_foreign_keys
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/reflection.py", line 55, in cache
ret = fn(self, con, *args, **kw)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/dialects/mssql/base.py", line 2498, in wrap
return _switch_db(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/dialects/mssql/base.py", line 2522, in _switch_db
return fn(*arg, **kw)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/dialects/mssql/base.py", line 3400, in get_foreign_keys
for r in connection.execute(s).fetchall():
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1286, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 325, in _execute_on_connection
return connection._execute_clauseelement(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1478, in _execute_clauseelement
ret = self._execute_context(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1842, in _execute_context
self._handle_dbapi_exception(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2023, in _handle_dbapi_exception
util.raise_(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1799, in _execute_context
self.dialect.do_execute(
File "/home/ubuntu/.local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.ProgrammingError: (pyodbc.ProgrammingError) ('42000', "[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Ambiguous column name 'constraint_schema'. (209) (SQLExecDirectW)")
[SQL: WITH fk_info AS (
SELECT
ischema_ref_con.constraint_schema,
ischema_ref_con.constraint_name,
ischema_key_col.ordinal_position,
ischema_key_col.table_schema,
ischema_key_col.table_name,
ischema_ref_con.unique_constraint_schema,
ischema_ref_con.unique_constraint_name,
ischema_ref_con.match_option,
ischema_ref_con.update_rule,
ischema_ref_con.delete_rule,
ischema_key_col.column_name AS constrained_column
FROM
INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ischema_ref_con
INNER JOIN
INFORMATION_SCHEMA.KEY_COLUMN_USAGE ischema_key_col ON
ischema_key_col.table_schema = ischema_ref_con.constraint_schema
AND ischema_key_col.constraint_name =
ischema_ref_con.constraint_name
WHERE ischema_key_col.table_name = CAST(? AS NVARCHAR(max))
AND ischema_key_col.table_schema = CAST(? AS NVARCHAR(max))
),
constraint_info AS (
SELECT
ischema_key_col.constraint_schema,
ischema_key_col.constraint_name,
ischema_key_col.ordinal_position,
ischema_key_col.table_schema,
ischema_key_col.table_name,
ischema_key_col.column_name
FROM
INFORMATION_SCHEMA.KEY_COLUMN_USAGE ischema_key_col
),
index_info AS (
SELECT
sys.schemas.name AS index_schema,
sys.indexes.name AS index_name,
sys.index_columns.key_ordinal AS ordinal_position,
sys.schemas.name AS table_schema,
sys.objects.name AS table_name,
sys.columns.name AS column_name
FROM
sys.indexes
INNER JOIN
sys.objects ON
sys.objects.object_id = sys.indexes.object_id
INNER JOIN
sys.schemas ON
sys.schemas.schema_id = sys.objects.schema_id
INNER JOIN
sys.index_columns ON
sys.index_columns.object_id = sys.objects.object_id
AND sys.index_columns.index_id = sys.indexes.index_id
INNER JOIN
sys.columns ON
sys.columns.object_id = sys.indexes.object_id
AND sys.columns.column_id = sys.index_columns.column_id
)
SELECT
fk_info.constraint_schema,
fk_info.constraint_name,
fk_info.ordinal_position,
fk_info.constrained_column,
constraint_info.table_schema AS referred_table_schema,
constraint_info.table_name AS referred_table_name,
constraint_info.column_name AS referred_column,
fk_info.match_option,
fk_info.update_rule,
fk_info.delete_rule
FROM
fk_info INNER JOIN constraint_info ON
constraint_info.constraint_schema =
fk_info.unique_constraint_schema
AND constraint_info.constraint_name =
fk_info.unique_constraint_name
AND constraint_info.ordinal_position = fk_info.ordinal_position
UNION
SELECT
fk_info.constraint_schema,
fk_info.constraint_name,
fk_info.ordinal_position,
fk_info.constrained_column,
index_info.table_schema AS referred_table_schema,
index_info.table_name AS referred_table_name,
index_info.column_name AS referred_column,
fk_info.match_option,
fk_info.update_rule,
fk_info.delete_rule
FROM
fk_info INNER JOIN index_info ON
index_info.index_schema = fk_info.unique_constraint_schema
AND index_info.index_name = fk_info.unique_constraint_name
AND index_info.ordinal_position = fk_info.ordinal_position
ORDER BY constraint_schema, constraint_name, ordinal_position
]
[parameters: ('astencube_users', 'dbo')]
(Background on this error at: https://sqlalche.me/e/14/f405)
Wading through your code above we find this exact error message:
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Ambiguous column name 'constraint_schema'. (209) (SQLExecDirectW)")
So when we look to see where you've referenced constraint_schema in your code, there are a few places.
For most of them you have correctly used 2-part naming to avoid any ambiguity.
However your final order by clause does not:
ORDER BY constraint_schema, constraint_name, ordinal_position
P.S. the best way to debug this stuff yourself in the future is to take the SQL code and run it directly against the SQL Server i.e. eliminate Python from the equation as it just muddies the waters.
the problem was because of the SqlAlchemy version.The version in my local machine was 1.4.25 when I install newest version to any where I got the sql error shown above.So this problem was happened because of the version.Thanks.
Related
I've created a custom user by executing:
mysql> CREATE USER 'myuser'#'localhost' IDENTIFIED BY 'mypwd';
mysql> GRANT select, update, alter ON mydb.* TO myuser#localhost;
My user only needs to read data from the database, update entries and change a few table's structure using the ALTER command.
Now I want to execute the following script:
import mysql.connector
db = mysql.connector.connect(
host="localhost",
user="myuser",
passwd="mypwd",
database="mydb")
print(db);
I don't fully understand why, but following error occurrs:
Traceback (most recent call last):
File "conneciton.py", line 3, in <module>
weewxDb = mysql.connector.connect(
File "/usr/lib/python3/dist-packages/mysql/connector/__init__.py", line 173, in connect
return MySQLConnection(*args, **kwargs)
File "/usr/lib/python3/dist-packages/mysql/connector/connection.py", line 102, in __init__
self.connect(**kwargs)
File "/usr/lib/python3/dist-packages/mysql/connector/abstracts.py", line 735, in connect
self._open_connection()
File "/usr/lib/python3/dist-packages/mysql/connector/connection.py", line 250, in _open_connection
self._do_auth(self._user, self._password,
File "/usr/lib/python3/dist-packages/mysql/connector/connection.py", line 172, in _do_auth
self._auth_switch_request(username, password)
File "/usr/lib/python3/dist-packages/mysql/connector/connection.py", line 216, in _auth_switch_request
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1044 (42000): Access denied for user 'myuser'#'localhost' to database 'mydb'
I think my user is missing some privileges. But which? I want to give him as few permissions as possible.
Sorry for the messy code, I got a little lost in trial and error stage (I run out of coffee).
I try to build a dashboard. to track my orders, revenue etc. to do this, I created a database that include all the related data.
After that, I download a "flask dashboard template" and started editing it.
I can collect all my order data and able using it in the index.html via curly bracelets.
When I try to sum all revenue rows in IDLE;
from flask import Flask, render_template, request, jsonify
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine
from sqlalchemy.sql import text
from sqlalchemy import func
engine = create_engine('sqlite:///transactions.db', echo=True)
conn = engine.connect()
global b
a = text('SELECT * FROM mytable')
b = conn.execute(a).fetchall()
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + 'transactions.db'
db = SQLAlchemy(app)
revenue = db.engine.execute('select sum(total) from mytable').scalar()
revenue = int(revenue)
print(revenue)
it works without an issue, but when I try to implement it in my route file like this;
# -*- encoding: utf-8 -*-
from app.home import blueprint
from flask import Flask, render_template, redirect, url_for
from flask_login import login_required, current_user
from app import login_manager
from jinja2 import TemplateNotFound
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine
from sqlalchemy.sql import text
# custom block
from sqlalchemy import create_engine
engine = create_engine('sqlite:///transactions.db', echo=True)
conn = engine.connect()
global b
global revenue
a = text('SELECT * FROM mytable')
b = conn.execute(a).fetchall()
app = Flask(__name__)
dborder = SQLAlchemy(app)
db = SQLAlchemy(app)
#revenue
engin = create_engine('sqlite:///transactions.db', echo=True)
coni = engine.connect()
db1 = SQLAlchemy(app)
revenue = db1.engine.execute('select sum(total) from mytable').scalar()
revenue = int(revenue)
#revenue
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + 'transactions.db'
class mytable(dborder.Model):
name = dborder.Column(dborder.Integer, primary_key=True)
add = dborder.Column(dborder.String)
# end custom block
#blueprint.route('/index')
#login_required
def index():
#if not current_user.is_authenticated:
# return redirect(url_for('base_blueprint.login'))
global b
orderlist = b
return render_template('index.html', orderlist = orderlist, revenue = revenue)
#blueprint.route('/<template>')
def route_template(template):
if not current_user.is_authenticated:
return redirect(url_for('base_blueprint.login'))
try:
return render_template(template + '.html')
except TemplateNotFound:
return render_template('page-404.html'), 404
except:
return render_template('page-500.html'), 500
I'm getting an error;
2020-05-29 00:41:13,977 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1
2020-05-29 00:41:13,977 INFO sqlalchemy.engine.base.Engine ()
2020-05-29 00:41:13,978 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1
2020-05-29 00:41:13,978 INFO sqlalchemy.engine.base.Engine ()
2020-05-29 00:41:13,979 INFO sqlalchemy.engine.base.Engine SELECT * FROM mytable
2020-05-29 00:41:13,979 INFO sqlalchemy.engine.base.Engine ()
C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\flask_sqlalchemy\__init__.py:808: UserWarning: Neither SQLALCHEMY_DATABASE_URI nor SQLALCHEMY_BINDS is set. Defaulting SQLALCHEMY_DATABASE_URI to "sqlite:///:memory:".
warnings.warn(
C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\flask_sqlalchemy\__init__.py:829: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
warnings.warn(FSADeprecationWarning(
C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\flask_sqlalchemy\__init__.py:829: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
warnings.warn(FSADeprecationWarning(
C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\flask_sqlalchemy\__init__.py:829: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
warnings.warn(FSADeprecationWarning(
Traceback (most recent call last):
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1283, in _execute_context
self.dialect.do_execute(
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\default.py", line 590, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such table: mytable
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "run.py", line 21, in <module>
app = create_app(config_mode)
File "C:\Users\Ozgur\Documents\python-colleect\admin panel\flask-material-dashboard\app\__init__.py", line 81, in create_app
register_blueprints(app)
File "C:\Users\Ozgur\Documents\python-colleect\admin panel\flask-material-dashboard\app\__init__.py", line 24, in register_blueprints
module = import_module('app.{}.routes'.format(module_name))
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\importlib\__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "C:\Users\Ozgur\Documents\python-colleect\admin panel\flask-material-dashboard\app\home\routes.py", line 44, in <module>
revenue = db1.engine.execute('select sum(total) from mytable').scalar()
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 2244, in execute
return connection.execute(statement, *multiparams, **params)
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1012, in execute
return self._execute_text(object_, multiparams, params)
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1181, in _execute_text
ret = self._execute_context(
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1323, in _execute_context
self._handle_dbapi_exception(
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1517, in _handle_dbapi_exception
util.raise_(
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\util\compat.py", line 178, in raise_
raise exception
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\base.py", line 1283, in _execute_context
self.dialect.do_execute(
File "C:\Users\Ozgur\AppData\Local\Programs\Python\Python38-32\lib\site-packages\sqlalchemy\engine\default.py", line 590, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: mytable
[SQL: select sum(total) from mytable]
(Background on this error at: http://sqlalche.me/e/e3q8)
I think you have to put
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + 'transactions.db'
before
db1 = SQLAlchemy(app)
Explanation
when your code wants to execute this line:
revenue = db1.engine.execute('select sum(total) from mytable').scalar()
SQLAlchemy needs to know the database that contains mytable. So you have to specify the URI to your database before executing this line.
I don't use flask but I had the same OperationalError using sqlalchemy (ORM mode), Python 3.7.3 and sqlite3.
Perhaps consider this : https://docs.sqlalchemy.org/en/14/tutorial/metadata.html, section "Emitting DDL to the database".
In my case I wrote :
from sqlalchemy import MetaData
metadata = MetaData()
from sqlalchemy.orm import declarative_base
Base = declarative_base()
engine = create_engine("sqlite:///test.db")
(classes declarations)
metadata.create_all(engine)
Then I get an OperationalError.
I've just added "Base." to solve my problem :
Base.metadata.create_all(engine)
So I'm trying to save a URL to a database but I keep getting this error:
ValueError: unsupported format character 'B' (0x42) at index 568
when I try to execute the query.
The URL:
https://images-na.ssl-images-amazon.com/images/I/81Sp%2BP3kX-L._AC_SL1500_.jpg
The code:
query = f"INSERT INTO Ruby.available_links(user_email, product_name, product_price, product_currency, product_old_price, product_store, user_link, spider_link, spider, created_date, product_img)" \
f"VALUES ('{email}', '{product_name}', '{product_price}', '{str(product_currency)}', '{product_price}', '{company[0]}', '{user_short_url['link']}', '{str(url)}', '{spider}', '{str(date.today())}', '{str(product_img)}')"
conn.execute(query)
I know that is the url causing the problem because when I removed it from the query it successfully executed the query and did not display any error.
Full traceback:
File "new_link_handler.py", line 70, in runner_new_links
conn.execute(query)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 976, in execute
return self._execute_text(object_, multiparams, params)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1149, in _execute_text
parameters,
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1478, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py", line 588, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib/python3.6/dist-packages/pymysql/cursors.py", line 168, in execute
query = self.mogrify(query, args)
File "/usr/local/lib/python3.6/dist-packages/pymysql/cursors.py", line 147, in mogrify
query = query % self._escape_args(args, conn)
ValueError: unsupported format character 'B' (0x42) at index 568
When using raw SQL and SQLAlchemy, you can use the sqlalchemy.text object to to ensure that parameter values are correctly quoted in SQL sent to the database (docs). The prevents both errors due to unquoted special characters or datatypes, as seen in this question, and also SQL injection attacks.
To use the text object, declare values in the SQL like this:
stmt = """INSERT INTO my_table (some_column) VALUES (:some_value)"""
or
stmt = """SELECT some_column FROM my_table WHERE some_column = :some_value"""
then create a text object
stmt = sqlalchemy.text(sql)
then use the bindparams method to bind the parameter value into the query
stmt = stmt.bindparams(some_value='whatever')
the resulting object can now be executed by the connection.
Here's an example based on your code:
import sqlalchemy as sa
url = 'https://images-na.ssl-images-amazon.com/images/I/81Sp%2BP3kX-L._AC_SL1500_.jpg'
stmt = sa.text("""INSERT INTO links (url) VALUES (:url)""")
stmt = stmt.bindparams(url=url)
engine = sa.create_engine('mysql+pymysql://root:#localhost/test')
with engine.connect() as conn:
conn.execute(stmt)
I think your error is caused by the combination of two factors:
you're constructing the query by directly pasting arguments into it
– see comment by #rdas
your URL is url-encoded, ie. it contains a character encoded as %2B
You end up with a query containing the %2B sequence, which gets misinterpreted. Try creating a query with placeholders and passing arguments to conn.execute.
I have defined and created a SQLite database using SQLAlchemy, and added some data. This database has been working fine. However, I made a change to a column name in one the models ("User") and migrated this change using alembic. The current model looks like this, where the old name "analogy" was changed to "metaphor":
class User(Base):
__tablename__ = "user"
id = Column(Integer, primary_key=True)
username = Column(String(32), nullable=False, index=True, unique=True)
registered_on = Column(DateTime, index=True, default=datetime.now())
metaphor = Column(String(16))
The migration seemed to go fine, and I also doublechecked using the SQLite command line to confirm that the schema actually changed.
sqlite > .schema
CREATE TABLE "user" (
id INTEGER NOT NULL,
username VARCHAR(32) NOT NULL,
registered_on DATETIME,
metaphor VARCHAR(16),
PRIMARY KEY (id),
UNIQUE (wc_id)
);
CREATE UNIQUE INDEX ix_user_username ON user (username);
CREATE INDEX ix_user_registered_on ON user (registered_on);
But now when I try to query the table in Python, I get the following error, referencing the old column name and saying that there is no such column
>>> from background.database import db_session
>>> from data.models import User
>>> users = db_session.query(User).all()
Traceback (most recent call last):
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/default.py", line 509, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such column: user.analogy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/orm/query.py", line 2843, in all
return list(self)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/orm/query.py", line 2995, in __iter__
return self._execute_and_instances(context)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/orm/query.py", line 3018, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/util/compat.py", line 265, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/util/compat.py", line 248, in reraise
raise value.with_traceback(tb)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/export/scratch/pt/power/lib/python3.5/site-packages/sqlalchemy/engine/default.py", line 509, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such column: user.goal_period [SQL: 'SELECT user.id AS user_id, user.username AS user_username, user.registered_on AS user_registered_on, user.analogy AS user_analogy \nFROM user'] (Background on this error at: http://sqlalche.me/e/e3q8)
After tracing this error, it looks like the name change to the model is not reflected in the metadata imported from SQLAlchemy. So, SQLAlchemy is still using the metadata from the prior version of the table, while the actual database schema and the defined model have a different column name.
I've tried to delete all caches (.pyc, py_cache) to see if that was the source of the problem.
However, after doing this, there were still no changes to the metadata.
when I open a new interprer session and query the metadata, it shows the old schema, not the new one.
>>> from SQLAlchemy import MetaData
>>> metadata= MetaData()
>>> metadata.tables['user']
Table('user', MetaData(bind=None), Column('id', Integer(), table=<user>, primary_key=True, nullable=False), Column('username', String(length=32), table=<user>, nullable=False), Column('registered_on', DateTime(), table=<user>, default=ColumnDefault(datetime.datetime(2019, 6, 5, 12, 33, 57, 11763))), Column('goal_period', String(length=16), table=<user>, default=ColumnDefault('daily'))), schema=None)
Why is the metadata is not updated, and how do fix this problem?
It's possible to connect multiple SQLite databases together using the 'ATTACH' statement and work with them jointly. Tables in each SQLite file can be referenced using a schema/file specific keyword. This should allow you to simultaneously deal with multiple tables with the same name by scoping by file. I went through a very good tutorial on how to do this is here:
http://longweekendmobile.com/2010/05/29/how-to-attach-multiple-sqlite-databases-together/
It seemed like I should be able to use SQLAlchemy's Table 'schema' keyword to differentiate between connections to multiple files. When I went searching for a way to use SQLAlchemy with SQLite databases that had been connected via ATTACH, this was the only example I found. Unfortunately, it is out of date and does not seem to work with current versions.
https://groups.google.com/forum/#!topic/sqlalchemy/QXqs4M2MjbY
I tried to update that example using Declarative classes, etc. Here was my attempt:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import *
#from sqlalchemy.pool import SingletonThreadPool
metadata = MetaData(object)
DeclarativeBase = declarative_base(metadata=metadata)
##########################################################################
# Classes
##########################################################################
class A(DeclarativeBase):
__table__ = Table('A', DeclarativeBase.metadata,
Column('id', Integer, primary_key=True, index=True, autoincrement=True),
Column('col_a', Integer, index=True))
class B(DeclarativeBase):
__table__ = Table('B', DeclarativeBase.metadata,
Column('id', Integer, primary_key=True, index=True, autoincrement=True),
Column('col_b', Integer, index=True),
schema='database_b')
#engine = create_engine('sqlite:////tmp/database_a.sqlite',echo=True, poolclass=SingletonThreadPool)
engine = create_engine('sqlite:////tmp/database_a.sqlite',echo=True)
db = engine.connect()
db.execute("ATTACH DATABASE '/tmp/database_b.sqlite' AS database_b")
DeclarativeBase.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
session.commit()
Unfortunately, I got the following output:
monster:tmp ladmin$ python sqliteattachtest2.py
2014-04-12 18:04:58,845 INFO sqlalchemy.engine.base.Engine ATTACH DATABASE '/tmp/database_b.sqlite' AS database_b
2014-04-12 18:04:58,845 INFO sqlalchemy.engine.base.Engine ()
2014-04-12 18:04:58,846 INFO sqlalchemy.engine.base.Engine PRAGMA "database_b".table_info("B")
2014-04-12 18:04:58,846 INFO sqlalchemy.engine.base.Engine ()
2014-04-12 18:04:58,846 INFO sqlalchemy.engine.base.Engine ROLLBACK
Traceback (most recent call last):
File "sqliteattachtest2.py", line 29, in <module>
DeclarativeBase.metadata.create_all(engine)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/schema.py", line 2793, in create_all
tables=tables)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1479, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1122, in _run_visitor
**kwargs).traverse_single(element)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/sql/visitors.py", line 111, in traverse_single
return meth(obj, **kw)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/ddl.py", line 57, in visit_metadata
if self._can_create_table(t)]
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/ddl.py", line 35, in _can_create_table
table.name, schema=table.schema)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/dialects/sqlite/base.py", line 716, in has_table
cursor = _pragma_cursor(connection.execute(statement))
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 662, in execute
params)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 805, in _execute_text
statement, parameters
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 874, in _execute_context
context)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1024, in _handle_dbapi_exception
exc_info
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 195, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 867, in _execute_context
context)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 324, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (OperationalError) unknown database "database_b" 'PRAGMA "database_b".table_info("B")' ()
I read that this sort of thing can be done using Postgres schemas. Multiple attached SQLite databases seems like a natural counterpart. I suspect I'm either doing something stupid or missed some important point. Is it possible to use SQLAlchemy to work with multiple SQLite files at the same time? If so, what's the best way to do it? Are there other ORMs that make this easier than SQLAlchemy does?
Thanks!
Dan
Create Engine of SQlite with inmemory than attach different database files
from sqlalchemy import create_engine, MetaData, Table,Column,Integer,select
from sqlalchemy.orm import mapper, sessionmaker
from sqlite3 import dbapi2 as sqlite
from sqlalchemy.engine.reflection import Inspector
class Bookmarks(object):
pass
class BookmarksB(object):
pass
def loadSession():
engine = create_engine('sqlite://', echo=True)
engine.execute("attach database 'database_b' as BB;")
engine.execute("attach database 'database_a' as AA;")
metadata = MetaData(engine)
inspector = Inspector.from_engine(engine)
print inspector.get_table_names()
moz_bookmarks = Table('table_a', metadata,Column("id", Integer, primary_key=True),schema='AA', autoload=True)
mapper(Bookmarks, moz_bookmarks)
moz_bookmarksB = Table('table_b', metadata,Column("id", Integer, primary_key=True),schema='BB', autoload=True)
mapper(BookmarksB, moz_bookmarksB)
Session = sessionmaker(bind=engine)
session = Session()
return session
if __name__ == "__main__":
session = loadSession()
res = session.query(Bookmarks).all()
for m in res:
print m.msisdn,m.id
#print list(select([moz_bookmarks, moz_bookmarksB], moz_bookmarks.c.b_id == moz_bookmarksB.c.id).execute())