Develop Reference

SQLAlchemy - Data long long for column email

I use a simple MySQL Database with a SQLAlchemy Model:
from tokenize import String
from sqlalchemy import Column, Integer, String
from .database import Base
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True, index=True)
email = Column(String(256), unique=True, index=True)
hashed_password = Column(String(256))
It works fine, but when the character length exceeds a number of characters which is much smaller than 256, I get the following error:
fastapi | sqlalchemy.exc.DataError: (MySQLdb._exceptions.DataError) (1406, "Data too long for column 'email' at row 1")
fastapi | [SQL: INSERT INTO users (email, hashed_password) VALUES (%s, %s)]
fastapi | [parameters: ('sdasdasdasdasdasda', 'sdasdasdasdasdasdsadadasdasd')]
I know there is a "strict-mode" in MySQL, but I would rather like to understand the error here and make it work in the python code, since I run the Database in a Container.
SHOW CREATE TABLE users:
| users | CREATE TABLE `users` (
`id` int NOT NULL AUTO_INCREMENT,
`email` varchar(256) DEFAULT NULL,
`hashed_password` varchar(256) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `ix_users_email` (`email`),
KEY `ix_users_id` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci |

How to force SQLAlchemy to convert Boolean type from PostgreSQL to Bit type from SQL Server

I'm trying to copy a couple of tables from one database ("db1", PostgreSQL) to another ("db2", SQL Server).
Unfortunately, I face an issue due to the BOOLEAN type for some fields in the PostgreSQL database which is not recognized as a valid type for SQL Server.
Here is my code sample:
db2_engine = "postgresql+psycopg2://" + str(db2_user) + ":" + str(db2_password) + "#" + str(db2_host) + ":" + str(db2_port) + "/" + str(db2_database)
db2 = sqlalchemy.create_engine(db2_engine)
lst_tablename_totr = ["contract",
"subscription",
"contractdelivery",
"businesspartner"
]
for table_name in lst_tablename_totr:
table = Table(table_name, metadata, autoload=True, autoload_with=db2)
table.create(bind=db1)
query = """
SELECT
*
FROM """ + str(table_name) + """
"""
df_hg = pd.read_sql(query, db2_engine)
df_hg.to_sql(table_name, db1, schema='dbo', index=False, if_exists='append')
For now, the issue is located to the table = Table(table_name, metadata, autoload=True, autoload_with=db_hgzl) table.create(bind=db1) part of the code.
Here is the error message:
ProgrammingError: (pyodbc.ProgrammingError) ('42000', '[42000] [Microsoft][ODBC Driver 13 for SQL Server][SQL Server]Column, parameter or variable #8\xa0: data type BOOLEAN not found. (2715) (SQLExecDirectW)')
I couldn't find any way to force the conversion between PostgreSQL Boolean type and SQL Server Bit type.

You are seeing a difference between SQLAlchemy's dialect-specific BOOLEAN type and its generic Boolean type. For an existing PostgreSQL table
CREATE TABLE IF NOT EXISTS public.so68683260
(
id character varying(5) COLLATE pg_catalog."default" NOT NULL,
bool_col boolean NOT NULL,
CONSTRAINT so68683260_pkey PRIMARY KEY (id)
)
if we reflect the table then the boolean columns are defined as BOOLEAN
tbl = sa.Table(table_name, sa.MetaData(), autoload_with=pg_engine)
print(type(tbl.columns["bool_col"].type))
# <class 'sqlalchemy.sql.sqltypes.BOOLEAN'>
and then if we try to create the table in SQL Server we end up doing the equivalent of
tbl = sa.Table(
table_name,
sa.MetaData(),
sa.Column("id", sa.VARCHAR(5), primary_key=True),
sa.Column("bool_col", sa.BOOLEAN, nullable=False),
)
tbl.drop(ms_engine, checkfirst=True)
tbl.create(ms_engine)
and that fails with the error you cite because the DDL rendered is
CREATE TABLE so68683260 (
id VARCHAR(5) NOT NULL,
bool_col BOOLEAN NOT NULL,
PRIMARY KEY (id)
)
However, if we use the generic Boolean type
tbl = sa.Table(
table_name,
sa.MetaData(),
sa.Column("id", sa.VARCHAR(5), primary_key=True),
sa.Column("bool_col", sa.Boolean, nullable=False),
)
tbl.drop(ms_engine, checkfirst=True)
tbl.create(ms_engine)
we are successful because the DDL rendered is
CREATE TABLE so68683260 (
id VARCHAR(5) NOT NULL,
bool_col BIT NOT NULL,
PRIMARY KEY (id)
)
and BIT is the valid corresponding column type in T-SQL.
Feel free to open a SQLAlchemy issue if you believe that this behaviour should be changed.
[Note also that the text column is VARCHAR(5) because the table uses the default encoding for my PostgreSQL test database (UTF8), but creating the table in SQL Server will create a VARCHAR (non-Unicode) column instead of a NVARCHAR (Unicode) column.]

Flask SQL Alchemy try perform SET operation, when it is not needed

I am facing a very strange issue using SQL Alchemy. I have a table in SQL server, to store Sales, with this format :
CREATE TABLE Resale (
[address] VARCHAR(100) NOT NULL,
[id] BIGINT NOT NULL,
[price] INT NOT NULL,
[start_date] DATETIME2(7) NOT NULL DEFAULT GETUTCDATE(),
[sys_start_time] DATETIME2 (7) GENERATED ALWAYS AS ROW START HIDDEN NOT NULL,
[sys_end_time] DATETIME2 (7) GENERATED ALWAYS AS ROW END HIDDEN NOT NULL,
PERIOD FOR SYSTEM_TIME ([sys_start_time], [sys_end_time]),
CONSTRAINT [PK_ReSale] PRIMARY KEY CLUSTERED ([id] ASC))
WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE=[dbo].[ResaleHistory], DATA_CONSISTENCY_CHECK=ON));
in python I used flask-sqlalchemy and build this model :
class Resale(db.Model):
__tablename__ = 'Resale'
address = db.Column(db.String(100), nullable=False)
id = db.Column(db.BigInteger, primary_key=True)
price = db.Column(db.Integer, nullable=False)
start_date = db.Column(db.DateTime, nullable=False, default=datetime.utcnow())
now I am just trying to insert a record in this table using :
resale = Resale(address=address, id=id, price=price)
db.session.add(resale)
db.session.commit()
and flask return :
pyodbc.ProgrammingError: ('42000', "[42000] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Table 'Resale' does not have the identity property. Cannot perform SET operation. (8106) (SQLExecDirectW)")
[SQL: SET IDENTITY_INSERT [Resale] ON]
I really don't understand why I have that, I am not using an autoincrement in my table, so my flask wants me to do "SET IDENTITY_INSERT Resale ON ?
honnestly I am completly lost with this message, I am just trying to add a record in database

As stated in this answer for a previous similiar question you should consider put the flag "autoincrement " to False on your "id".

How can I print schema / table definitions in SQLAlchemy

How can I print out the schema of all tables using sqlalchemy?
This is how I do it using SQLite3: I run an SQL to print out the schema of all tables in the database:
import sqlite3
conn = sqlite3.connect("example.db")
cur = conn.cursor()
rs = cur.execute(
"""
SELECT name, sql FROM sqlite_master
WHERE type='table'
ORDER BY name;
""")
for name, schema, *args in rs:
print(name)
print(schema)
print()
With output that can look like this:
albums
CREATE TABLE "albums"
(
[AlbumId] INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
[Title] NVARCHAR(160) NOT NULL,
[ArtistId] INTEGER NOT NULL,
FOREIGN KEY ([ArtistId]) REFERENCES "artists" ([ArtistId])
ON DELETE NO ACTION ON UPDATE NO ACTION
)
artists
CREATE TABLE "artists"
(
[ArtistId] INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
[Name] NVARCHAR(120)
)
Is there a way to do it with pure sqlalchemy api calls, something better than this?
metadata = sqlalchemy.MetaData()
metadata.reflect(engine)
insp = sqlalchemy.inspect(engine)
for table_name in metadata.tables:
print(table_name)
for column in insp.get_columns(table_name):
for name,value in column.items():
print(' ', end='')
if value:
field = name if value in [True, 'auto'] else value
print(field, end=' ')
print()
Output:
albums
AlbumId INTEGER autoincrement primary_key
Title NVARCHAR(160) autoincrement
ArtistId INTEGER autoincrement
artists
ArtistId INTEGER autoincrement primary_key
Name NVARCHAR(120) nullable autoincrement

This bit in the SQLAlchemy docs may help: they suggest doing this:
def dump(sql, *multiparams, **params):
print(sql.compile(dialect=engine.dialect))
engine = create_engine('postgresql://', strategy='mock', executor=dump)
metadata.create_all(engine, checkfirst=False)

How to Convert SqlServer ON [PRIMARY] into Sqlalchemy Model?

I have query in sqlserver and I wanted to write a model corresponding to it using sqlalchemy orm. I need to know how to convert ON [PRIMARY] into sqlalchemy.
Here is my query:
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[UserModel](
[userid] [nvarchar](255) NULL,
[username] [nvarchar](255) NULL,
[serialnumber] [nvarchar](255) NULL
) ON [PRIMARY]
GO
Here is my model:
class UserModel(db.Model):
__tablename__ = 'UserModel'
userid = Column('userid', Unicode(255))
username = Column('username', Unicode(255))
serialnumber = Column('serialnumber', Unicode(255))
Any suggestions, how would I achieve this
Currently I m getting the following error as below:
sqlalchemy.exc.ArgumentError: Mapper Mapper|UserModel|UserModel could not assemble any primary key columns for mapped table 'UserModel'
I know there there is no primary primary key defined in my model but the above query is working fine. So I wanted to translate the above query using sqlalchemy orm.

You cannot translate the above DDL using most ORMs, at least directly, because they expect Python objects to correspond to uniquely identifiable rows in the database. Now, if you know for sure that for example UserModel.userid uniquely identifies rows, you can instruct SQLAlchemy to use it as a "primary key":
class UserModel(db.Model):
__tablename__ = 'UserModel'
userid = Column('userid', Unicode(255))
username = Column('username', Unicode(255))
serialnumber = Column('serialnumber', Unicode(255))
__mapper_args__ = {
'primary_key': [userid]
}
Keep in mind that you've lied to the ORM and any consequences are on you.
As to defining the filegroup the table should be stored in using ON [PRIMARY], you'll have to augment the mssql dialect a bit:
from sqlalchemy import Table
from sqlalchemy.schema import CreateTable
from sqlalchemy.ext.compiler import compiles
# Add our custom dialect specific argument
Table.argument_for('mssql', 'on', None)
#compiles(CreateTable, 'mssql')
def compile_create_table(create, compiler, **kw):
stmt = compiler.visit_create_table(create, **kw)
filegroup = create.element.kwargs.get('mssql_on')
if filegroup:
stmt = stmt.rstrip() # Prettier output, remove newlines
filegroup = compiler.preparer.quote(filegroup)
stmt = f"{stmt} ON {filegroup}\n\n"
return stmt
and add the necessary table args to your model definition:
class UserModel(db.Model):
...
__table_args__ = {
'mssql_on': 'PRIMARY'
}
You can verify that the resulting DDL is as desired:
In [4]: print(CreateTable(UserModel.__table__).compile(dialect=engine.dialect))
CREATE TABLE [UserModel] (
userid NVARCHAR(255) NULL,
username NVARCHAR(255) NULL,
serialnumber NVARCHAR(255) NULL
) ON [PRIMARY]
Note that at least according to this Q/A primary is the default and as such ON [PRIMARY] could just be omitted.