I have a list of dict objects
data = [
{'id': 1, 'dt':'2002-01-02' },
{'id': 2, 'dt':'2014-01-15' },
{'id': 3, 'dt':'2005-10-20' }
]
and a table in sqlite created using sqlalchemy as follows
engine = create_engine(config.SQLALCHEMY_DATABASE_URI)
metadata = MetaData()
tbl1 = Table('t1', metadata,
Column('the_id', Integer, primary_key=True),
Column('the_dt', Date))
metadata.create_all(bind=engine)
stmt_insert = tbl1.insert().values(the_id=bindparam('id'), the_dt=bindparam('dt', type_=Date)
with engine.connect() as conn:
conn.execute(stmt_insert, data)
This gives me the following error:
sqlalchemy.exc.StatementError: (builtins.TypeError) SQLite Date type
only accepts Python date objects as input.
What do I assign to the "type_" parameter to make this binding work ?
You need to convert your dt strings to date objects, for instance:
import datetime
for item in data:
item['dt'] = datetime.datetime.strptime(item['dt'], "%Y-%m-%d").date()
If you don't need the ORM part of SQLAlchemy (no class/table mapping). The easiest way is to tell SQLAlchemy that you use String instead of Date, like this:
tbl1 = Table('t1', metadata,
Column('the_id', Integer, primary_key=True),
Column('the_dt', String))
It will work because your date string use ISO 8601 format ("%Y-%m-%d").
But the best practices are:
read records from CSV
convert data to Python objects (int, date, etc.)
insert data in database.
The conversion can be done in the constructor of the class which is mapped to the table.
EDIT A kind of "solution"
from sqlalchemy import Table, Column, Integer, MetaData, Date, String
from sqlalchemy import create_engine
from sqlalchemy.sql import select
engine = create_engine('sqlite:///:memory:', echo=True)
metadata = MetaData()
I've created a "fake" table with a String type instead of a Date:
fake_tbl1 = Table('t1', metadata,
Column('id', Integer, primary_key=True),
Column('dt', String))
metadata.create_all(bind=engine)
And insert the data as-is:
data = [
{'id': 1, 'dt': '2002-01-02'},
{'id': 2, 'dt': '2014-01-15'},
{'id': 3, 'dt': '2005-10-20'}
]
stmt_insert = fake_tbl1.insert()
with engine.connect() as conn:
conn.execute(stmt_insert, data)
Then I redefine the same table with the required Date field:
tbl2 = Table('t1', metadata,
Column('id', Integer, primary_key=True),
Column('dt', Date),
extend_existing=True)
Here is a rows selection:
stmt_select = select([tbl2])
with engine.connect() as conn:
result = conn.execute(stmt_select)
for row in result:
print(row)
You'll get:
(1, datetime.date(2002, 1, 2))
(2, datetime.date(2014, 1, 15))
(3, datetime.date(2005, 10, 20))
This works for your simple case, but I won't recommend this for a generic solution.
Related
I want to insert multiple rows using connection.execute, and one of the columns must be set to the result of the database's CURRENT_TIMESTAMP function.
For example, given this table:
import sqlalchemy as sa
metadata = sa.MetaData()
foo = sa.Table('foo', metadata,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('ts', sa.TIMESTAMP))
# I'm using Sqlite for this example, but this question
# is database-agnostic.
engine = sa.create_engine('sqlite://', echo=True)
metadata.create_all(engine)
I can insert a single row like this:
conn = engine.connect()
with conn.begin():
ins = foo.insert().values(ts=sa.func.current_timestamp())
conn.execute(ins)
However when I try to insert multiple rows:
with conn.begin():
ins = foo.insert()
conn.execute(ins, [{'ts': sa.func.current_timestamp()}])
a TypeError is raised:
sqlalchemy.exc.StatementError: (builtins.TypeError) SQLite DateTime type only accepts Python datetime and date objects as input.
[SQL: INSERT INTO foo (ts) VALUES (?)]
[parameters: [{'ts': <sqlalchemy.sql.functions.current_timestamp at 0x7f3607e21070; current_timestamp>}]
Replacing the function with the string "CURRENT_TIMESTAMP" results in a similar error.
Is there a way to get the database to set the column to CURRENT_TIMESTAMP using connection.execute?
I'm aware that I can work around this by querying for the value of CURRENT_TIMESTAMP within the same transaction and using that value in the INSERT values, or executing and UPDATE after the INSERT. I'm specifically asking whether this can be done in connection.execute's *multiparams argument.
It's a hack for sure, but this appears to work for SQLite at least:
from datetime import datetime
from pprint import pprint
import sqlalchemy as sa
engine = sa.create_engine("sqlite:///:memory:")
metadata = sa.MetaData()
foo = sa.Table(
"foo",
metadata,
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column("ts", sa.TIMESTAMP),
sa.Column("txt", sa.String(50)),
)
foo.create(engine)
with engine.begin() as conn:
ins_query = str(foo.insert().compile()).replace(
" :ts, ", " CURRENT_TIMESTAMP, "
)
print(ins_query)
# INSERT INTO foo (id, ts, txt) VALUES (:id, CURRENT_TIMESTAMP, :txt)
data = [{"id": None, "txt": "Alfa"}, {"id": None, "txt": "Bravo"}]
conn.execute(sa.text(ins_query), data)
print(datetime.now())
# 2021-03-06 17:41:35.743452
# (local time here is UTC-07:00)
results = conn.execute(sa.text("SELECT * FROM foo")).fetchall()
pprint(results, width=60)
"""
[(1, '2021-03-07 00:41:35', 'Alfa'),
(2, '2021-03-07 00:41:35', 'Bravo')]
"""
I want to update two tables using one query.
I'd like something along the lines of this
UPDATE tblReceipt, tblReturn
SET tblReceipt.ReturnedDate = tblReturn.CreatedDate,
tblReturn.ReturnerName = tblReceipt.Name
WHERE tblReturn.Id = tblReceipt.ReturnId
AND tblReceipt.returned = TRUE
I currently have the following but I'm not sure how to add the second table reference. Is there a simple way to do this?
update(Receipt)
.values(ReturnedDate=Return.CreatedDate, ReturnName=Receipt.Name)
.where(Return.Id==Receipt.ReturnId)
.where(Receipt.Returned == True)
From the documentation for update():
The keys within values can be either Column objects or their string
identifiers (specifically the “key” of the Column, normally but not
necessarily equivalent to its “name”). Normally, the Column objects
used here are expected to be part of the target Table that is the
table to be updated. However when using MySQL, a multiple-table UPDATE
statement can refer to columns from any of the tables referred to in
the WHERE clause.
Emphasis mine.
I've created an example core pattern based on your example above but not exact, although should be enough to work with:
from sqlalchemy_app import engine
import sqlalchemy as sa
metadata = sa.MetaData()
receipt = sa.Table(
"receipt",
metadata,
sa.Column("id", sa.Integer, primary_key=True),
sa.Column("something", sa.Integer),
)
returns = sa.Table(
"returns",
metadata,
sa.Column("id", sa.Integer, primary_key=True),
sa.Column("receipt_id", sa.Integer, sa.ForeignKey('receipt.id')),
sa.Column("somethingelse", sa.Integer)
)
if __name__ == "__main__":
metadata.drop_all(engine)
metadata.create_all(engine)
with engine.connect() as conn:
conn.execute(
receipt.insert(
values=[{"id": 1, "something": 1}]
)
)
conn.execute(
returns.insert(
values=[{"id": 1, "receipt_id": 1, "somethingelse": 99}]
)
)
conn.execute(
sa.update(receipt)
.values({receipt.c.something: 3, returns.c.somethingelse: 4})
.where(sa.and_(receipt.c.id == 1, returns.c.id == 1))
)
As per the documentation, the parent table of the two columns that are referenced in .values() are referenced in the .where() clause.
Here is the update statement it generates:
2019-08-17 11:27:39,573 INFO sqlalchemy.engine.base.Engine UPDATE receipt, returns SET returns.somethingelse=%(returns_somethingelse)s, receipt.something=%(something)s WHERE receipt.id = %(id_1)s AND returns.id = %(id_2)s
2019-08-17 11:27:39,582 INFO sqlalchemy.engine.base.Engine {'returns_somethingelse': 4, 'something': 3, 'id_1': 1, 'id_2': 1}
Note that if you just print the query out to inspect it, you'll get something that looks like this:
UPDATE receipt SET somethingelse=:returns_somethingelse, something=:something FROM returns WHERE receipt.id = :id_1 AND returns.id = :id_2
...as this is a mysql specific behavior you would have to compile it using the mysql dialect:
from sqlalchemy.dialects import mysql
print(statement.compile(dialect=mysql.dialect())
will print:
UPDATE receipt, returns SET returns.somethingelse=%s, receipt.something=%s WHERE receipt.id = %s AND returns.id = %s
s = select([stations.c.name]).where(stations.c.name == station_name)
stations = connection.execute(s).fetchone()
I have the above code to run SELECT on a SQL table. However, while other columns of the matched entry were accessible, trying to access its primary key column by stations['id'] gives the error:
"Could not locate column in row for column 'id'"
Why is that?
Table definition:
stations = Table('stations', metadata,
Column('id', Integer, primary_key = True),
Column('name', String(16), nullable = False)
)
Note: you should avoid giving the same name to different objects because in your case after statement
stations = connection.execute(s).fetchone()
initial stations Table object is no longer accessible. You can rename fetched object to station_record, or rename stations Table object to stations_table, or both.
Answer
If you want to get id of a record – than you should query it:
s = select([stations.c.id, stations.c.name]).where(stations.c.name == station_name)
or
s = select(stations.columns).where(stations.c.name == station_name)
Finally we can have something like
from sqlalchemy import MetaData, Table, Column, Integer, String, create_engine, select
db_uri = 'sqlite://'
engine = create_engine(db_uri)
metadata = MetaData(bind=engine)
stations = Table('stations', metadata,
Column('id', Integer, primary_key=True),
Column('name', String(16), nullable=False)
)
stations.create(checkfirst=True)
connection = engine.connect()
station_name = 'sample text'
connection.execute(stations.insert().values(name=station_name))
s = select(stations.columns).where(stations.c.name == station_name)
station_record = connection.execute(s).fetchone()
station_record_id = station_record['id']
station_record_name = station_record['name']
I am executing some code in python, but need to access some data from my database. This is my first time I have used sql alchemy
I have a table in my database called reports.bigjoin it has columns with the following types
id (varchar)
id2 (varchar)
ts_min (int4)
ts_local_min (int4)
10_meter_cell (int8)
ds (date)
ds_log (date)
ds_local (date)
I need to know the number of rows for a given set of dates. For example, within python I want to execute
x= select count(*) from reports.bigjoin where (ds>='2016-01-01' and ds<='2016-01-04')
My attempt so far has been
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy import Integer, String,Date
from sqlalchemy.orm import sessionmaker
engine = sqlalchemy.create_engine(url).connect()
Session = sqlalchemy.orm.sessionmaker(bind=engine)
session = Session()
metadata = sqlalchemy.MetaData(engine)
moz_bookmarks = Table('reports.bigjoin', metadata,
Column('id', String, primary_key=True),
Column('id2', String),
Column('ts_min', Integer),
Column('ts_local', Integer),
Column('10m_cell', Integer),
Column('ds', Date),
Column('ds_log', Date),
Column('ds_local', Date)
)
x = session.query(moz_bookmarks).filter(
(moz_bookmarks.ds >= '2016-01-01', moz_bookmarks.ds <= '2016-01-04')).count()
this has failed. Any help would be greatly appreciated.
cnt = (
session
.query(func.count('*').label('cnt'))
.filter(moz_bookmarks.c.ds >= '2016-01-01')
.filter(moz_bookmarks.c.ds <= '2016-01-04')
)
print(cnt)
After searching a bit and applying
How to use variables in SQL statement in Python?
I found that
connection = create_engine('url').connect()
result = connection.execute("select count(*) from reports.bigjoin where (ds>= %s and ds<= %s)", (x,y))
solved the problem
I'm using SQLAlchemy core functionality without ORM and I need to load records from arbitrary table by primary key passing a list as arguments.
Currently I'm doing it like:
records = select([arbitrary_table], list(arbitrary_table.primary_key.columns._all_cols)[0].in_([1, 2, 3]))
arbitrary_table is passed as parameter and it could be any table with restriction that primary key is integer and not composite, but primary key could have different names.
I have few questions about it:
Q1:
Is there any way to optimize it?
I'm much sure that list(self.arbitrary_table.primary_key.columns._all_cols)[0] is NOT the best way to get a primary key column.
Q2:
How to do the same with ORM query?
The example from SA site has my_user = session.query(User).get(5) but it takes only one ID as argument and has no override to take a list of IDs.
I will be very thankful for your suggestions.
the "primary_key" attribute is an instance of PrimaryKeyConstraint, which is iterable in place, so list(table.primary_key)[0] is all you need to get at that column.
The example below illustrates two composite-compatible techniques (edit: oh, you said "not composite". Well it's cool code anyway...use select_by_single_pk()), one requires tuple support (postgresql, maybe mysql), the other one strings out AND/OR, as well as a single PK approach. Then it shows query.get() which accepts a tuple:
from sqlalchemy import tuple_, or_, and_
def select_by_composite_pk(table, values):
"works only in a high-capability database like postgresql"
return table.select().where(tuple_(*table.primary_key).in_(values))
def select_by_composite_pk_no_tuples(table, values):
"works in any database"
return table.select().where(
or_(
*[
and_(*[col == val for col, val in zip(table.primary_key, val)])
for val in values
]
))
def select_by_single_pk(table, values):
"works in any database"
return table.select().where(list(table.primary_key)[0].in_(values))
if __name__ == '__main__':
from sqlalchemy import create_engine, Table, Column, Integer, MetaData
eng = create_engine("postgresql://scott:tiger#localhost/test", echo=True)
conn = eng.connect()
trans = conn.begin()
m = MetaData()
# single PK column
a = Table('a', m, Column('x', Integer, primary_key=True),
Column('y', Integer))
# composite PK column
b = Table('b', m, Column('x', Integer, primary_key=True),
Column('y', Integer, primary_key=True))
m.create_all(conn)
conn.execute(a.insert(), [
{'x': i, 'y': i * 2} for i in xrange(10)
])
conn.execute(b.insert(), [
{'x': i, 'y': i * 2} for i in xrange(10)
])
print conn.execute(
select_by_composite_pk(a,
[tuple_(3, ), tuple_(5, ), tuple_(9, )])).fetchall()
print conn.execute(
select_by_composite_pk(b,
[tuple_(3, 6), tuple_(5, 10), tuple_(9, 18)])).fetchall()
print conn.execute(
select_by_composite_pk_no_tuples(b,
[(3, 6), (5, 10), (9, 18)])).fetchall()
print conn.execute(
select_by_single_pk(b, [3, 5, 9])).fetchall()
# ORM query version
from sqlalchemy.orm import Session
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class MyCompositeClass(Base):
__table__ = b
# get accepts a tuple
print Session(conn).query(MyCompositeClass).get((5, 10))