Develop Reference

How can I Update Boolean Value Using SQLAlchemy and Python?

I am trying to update a boolean value within a database using Python and SQLAlchemy. Here is my code:
def update_record_to_hide_or_show(e2e_id, hide_error, event_time, study_id):
connection_string = _get_connection_string()
db = create_engine(connection_string)
roi_e2e_events = define_roi_e2e_events_table()
with db.connect() as conn:
if hide_error == "True":
update = roi_e2e_events.update().values(hide_error=True).where(roi_e2e_events.c.e2e_id == e2e_id)\
.where(roi_e2e_events.c.event_time == event_time)\
.where(roi_e2e_events.c.study_id == study_id)
print(update)
result = conn.execute(update)
else:
update = roi_e2e_events.update().values(hide_error=False).where(roi_e2e_events.c.e2e_id == e2e_id) \
.where(roi_e2e_events.c.event_time == event_time). \
where(roi_e2e_events.c.study_id == study_id)
result = conn.execute(update)
return result
I am able to enter the first portion of the conditional without problem, there is not an execution error displayed as I attempt to submit the query to the database, I've created the Metadata in a separate function and the update query looks like this:
UPDATE roi_e2e_events SET hide_error=:hide_error WHERE roi_e2e_events.e2e_id = :e2e_id_1 AND roi_e2e_events.event_time = :event_time_1 AND roi_e2e_events.study_id = :study_id_1
I don't see that the boolean value is changed to "True" after running this, what am I doing wrong here?

Without an example of the table + schema or what the arguments for this function look like (particularly hide_error), it's a little hard to be sure, but it looks like therecould be an issue on the hide_error == "True" line, since it's checking if that hide_error is the string "True", rather than the boolean True.
If it is in fact a boolean, we can actually get around the whole issue of checking what value it is by just using the not operator. Something like this:
def update_record_to_hide_or_show(e2e_id, hide_error, event_time, study_id):
connection_string = _get_connection_string()
db = create_engine(connection_string)
roi_e2e_events = define_roi_e2e_events_table()
with db.connect() as conn:
# no if statement here
update_query = roi_e2e_events.update().values(
hide_error=not hide_error # but notice the `not` here
).where(
roi_e2e_events.c.e2e_id == e2e_id
).where(
roi_e2e_events.c.event_time == event_time
).where(
roi_e2e_events.c.study_id == study_id
)
result = conn.execute(update)
return result
Also, if hide_error is being retrieved from the database, you can bundle it all up into a single UPDATE query like this
from sqlalchemy import not_ # this is used to invert the value of a boolean column in a query - on the database, rather than a value we have stored locally in a variable
def update_record_to_hide_or_show(e2e_id, event_time, study_id): # notice that `hide_error` is gone
connection_string = _get_connection_string()
db = create_engine(connection_string)
roi_e2e_events = define_roi_e2e_events_table()
with db.connect() as conn:
# still no if statement here
update_query = roi_e2e_events.update().values(
hide_error=not_(roi_e2e_events.c.hide_error) # here we use column, rather than its value, much like the `where` clauses
).where(
roi_e2e_events.c.e2e_id == e2e_id
).where(
roi_e2e_events.c.event_time == event_time
).where(
roi_e2e_events.c.study_id == study_id
)
result = conn.execute(update)
return result
where the update_query should look something like this:
UPDATE roi_e2e_events
SET hide_error=NOT roi_e2e_events.hide_error
WHERE roi_e2e_events.e2e_id = :e2e_id_1
AND roi_e2e_events.event_time = :event_time_1
AND roi_e2e_events.study_id = :study_id_1

Mock not overriding the return of a function in Python

I am implementing unit test on one of the classes of my project. The method that I want to test is queryCfsNoteVariations:
class PdfRaportDaoImpl:
def queryCfsNoteVariations(self, reportId):
sql = """
select v.* from item_value_table v
where v.table_id in
(select table_id from table_table t
where t.report_id=%s and table_name='CFS')
"""
cfsItemList = dbFind(sql, (reportId))
sql = "select * from variations_cfs_note"
cfsNoteVariations = dbFind(sql)
if cfsNoteVariations == None or len(cfsNoteVariations) == 0:
raise Exception("cfs note variations is null!")
cfsNoteVariationList = []
for itemInfo in cfsItemList:
for cfsNoteVariation in cfsNoteVariations:
if (
cfsNoteVariation["item_name_cfs"].lower()
== itemInfo["item_name"].lower()
):
cfsNoteVariationList.append(cfsNoteVariation["item_name_cfs_note"])
if len(cfsNoteVariationList) > 0:
return cfsNoteVariationList, itemInfo["item_note"]
return None, None
Which has a path: /com/pdfgather/PDFReportDao.py
In my test I am doing patch on dbFind() method which is located in /com/pdfgather/GlobalHelper.py. My current test looks like this:
from com.pdfgather.PDFReportDao import PdfReportDaoImpl
#patch("com.pdfgather.GlobalHelper.dbFind")
def test_query_cfs_note_variations(self, mock_find):
mock_find.side_effect = iter([
[{"item_name" : "foo"}, {"item_name" : "hey"}],
[{"item_name_cfs": "foo"},
{"item_name_cfs": "foo"},
{"item_name_cfs": "hey"}]]
])
report_id = 3578
result = TestingDao.dao.queryCfsNoteVariations(report_id)
# Printing result
print(result)
However I am not getting my desired result which is getting inside a loop and returning from inside a loop. Instead the dbFind is returning nothing (but it shouldn't as I already preassigned returning values for dbFind).
Thanks in advance!

Python refers com.pdfgather.PDFReportDao.dbFind and com.pdfgather.GlobalHelper.dbFind as two different classes. The second one is the import you want to patch. Try changing your patch to:
#patch("com.pdfgather.PDFReportDao.dbFind")

Bulk INSERT IGNORE using Flask-SQLAlchemy

I'm trying to update a database using API-gathered data, and I need to make sure all tables are being updated.
Sometime I will receive data that's already in the database, so I want to do an INSERT IGNORE.
My current code is something like this:
def update_orders(new_orders):
entries = []
for each_order in new_orders:
shipping_id = each_order['id']
title = each_order['title']
price = each_order['price']
code = each_order['code']
source = each_order['source']
phone = each_order['phone']
category = each_order['delivery_category']
carrier = each_order['carrier_identifier']
new_entry = Orders(
id=shipping_id,
title=title,
code=code,
source=source,
phone=phone,
category=category,
carrier=carrier,
price=price
)
entries.append(new_entry)
if len(entries) == 0:
print('No new orders.')
break
else:
print('New orders:', len(entries))
db.session.add_all(entries)
db.session.commit()
This works well when I'm creating the database from scratch, but it will give me an error if there's duplicate data, and I'm not able to commit the inserts.
I've been reading for a while, and found a workaround that uses prefix_with:
print('New orders:', len(entries))
if len(entries) == 0:
print('No new orders.')
else:
insert_command = Orders.__table__.insert().prefix_with('OR IGNORE').values(entries)
db.session.execute(insert_command)
db.session.commit()
The problem is that values(entries) is a bunch of objects:
<shop.database.models.Orders object at 0x11986def0> instead of being the instance of the class, is the class instance object in memory.
Anybody has any suggestion on approaching this problem?
Feel free to suggest a different approach, or just an adjustment.
Thanks a lot.

What database are you using ? Under MySQL, "INSERT OR IGNORE" is not valid syntax, instead one should use "INSERT IGNORE". I had the same situation and got my query to work with the following:
insert_command = Orders.__table__.insert().prefix_with(' IGNORE').values(entries)

Python Flask and SQLAlchemy, selecting all data from a column

I am attempting to query all rows for a column called show_id. I would then like to compare each potential item to be added to the DB with the results. Now the simplest way I can think of doing that is by checking if each show is in the results. If so pass etc. However the results from the below snippet are returned as objects. So this check fails.
Is there a better way to create the query to achieve this?
shows_inDB = Show.query.filter(Show.show_id).all()
print(shows_inDB)
Results:
<app.models.user.Show object at 0x10c2c5fd0>,
<app.models.user.Show object at 0x10c2da080>,
<app.models.user.Show object at 0x10c2da0f0>
Code for the entire function:
def save_changes_show(show_details):
"""
Save the changes to the database
"""
try:
shows_inDB = Show.query.filter(Show.show_id).all()
print(shows_inDB)
for show in show_details:
#Check the show isnt already in the DB
if show['id'] in shows_inDB:
print(str(show['id']) + ' Already Present')
else:
#Add show to DB
tv_show = Show(
show_id = show['id'],
seriesName = str(show['seriesName']).encode(),
aliases = str(show['aliases']).encode(),
banner = str(show['banner']).encode(),
seriesId = str(show['seriesId']).encode(),
status = str(show['status']).encode(),
firstAired = str(show['firstAired']).encode(),
network = str(show['network']).encode(),
networkId = str(show['networkId']).encode(),
runtime = str(show['runtime']).encode(),
genre = str(show['genre']).encode(),
overview = str(show['overview']).encode(),
lastUpdated = str(show['lastUpdated']).encode(),
airsDayOfWeek = str(show['airsDayOfWeek']).encode(),
airsTime = str(show['airsTime']).encode(),
rating = str(show['rating']).encode(),
imdbId = str(show['imdbId']).encode(),
zap2itId = str(show['zap2itId']).encode(),
added = str(show['added']).encode(),
addedBy = str(show['addedBy']).encode(),
siteRating = str(show['siteRating']).encode(),
siteRatingCount = str(show['siteRatingCount']).encode(),
slug = str(show['slug']).encode()
)
db.session.add(tv_show)
db.session.commit()
except Exception:
print(traceback.print_exc())

I have decided to use the method above and extract the data I wanted into a list, comparing each show to the list.
show_compare = []
shows_inDB = Show.query.filter().all()
for item in shows_inDB:
show_compare.append(item.show_id)
for show in show_details:
#Check the show isnt already in the DB
if show['id'] in show_compare:
print(str(show['id']) + ' Already Present')
else:
#Add show to DB

For querying a specific column value, have a look at this question: Flask SQLAlchemy query, specify column names. This is the example code given in the top answer there:
result = SomeModel.query.with_entities(SomeModel.col1, SomeModel.col2)
The crux of your problem is that you want to create a new Show instance if that show doesn't already exist in the database.
Querying the database for all shows and looping through the result for each potential new show might become very inefficient if you end up with a lot of shows in the database, and finding an object by identity is what an RDBMS does best!
This function will check to see if an object exists, and create it if not. Inspired by this answer:
def add_if_not_exists(model, **kwargs):
if not model.query.filter_by(**kwargs).first():
instance = model(**kwargs)
db.session.add(instance)
So your example would look like:
def add_if_not_exists(model, **kwargs):
if not model.query.filter_by(**kwargs).first():
instance = model(**kwargs)
db.session.add(instance)
for show in show_details:
add_if_not_exists(Show, id=show['id'])
If you really want to query all shows upfront, instead of putting all of the id's into a list, you could use a set instead of a list which will speed up your inclusion test.
E.g:
show_compare = {item.show_id for item in Show.query.all()}
for show in show_details:
# ... same as your code

SQLAlchemy 'on_conflict_do_update' does not update

I have the following code which I would like to do an upsert:
def add_electricity_reading(
*, period_usage, period_started_at, is_estimated, customer_pk
):
from sqlalchemy.dialects.postgresql import insert
values = dict(
customer_pk=customer_pk,
period_usage=period_usage,
period_started_at=period_started_at,
is_estimated=is_estimated,
)
insert_stmt = insert(ElectricityMeterReading).values(**values)
do_update_stmt = insert_stmt.on_conflict_do_update(
constraint=ElectricityMeterReading.__table_args__[0].name,
set_=dict(
period_usage=period_usage,
period_started_at=period_started_at,
is_estimated=is_estimated,
)
)
conn = DBSession.connection()
conn.execute(do_update_stmt)
return DBSession.query(ElectricityMeterReading).filter_by(**dict(
period_usage=period_usage,
period_started_at=period_started_at,
customer_pk=customer_pk,
is_estimated=is_estimated,
)).one()
def test_updates_existing_record_for_started_at_if_already_exists():
started_at = datetime.now(timezone.utc)
existing = add_electricity_reading(
period_usage=0.102,
customer_pk=customer.pk,
period_started_at=started_at,
is_estimated=True,
)
started_at = existing.period_started_at
reading = add_electricity_reading(
period_usage=0.200,
customer_pk=customer.pk,
period_started_at=started_at,
is_estimated=True,
)
# existing record was updated
assert reading.period_usage == 0.200
assert reading.id == existing.id
In my test when I add an existing record with period_usage=0.102 and then execute the query again but change to period_usage=0.2. When the final query at the bottom returns the record the period_usage is still 0.102.
Any idea why this could be happening?

This behaviour is explained in "Session Basics" under "What does the Session do?" The session holds references to objects it has loaded in a structure called the identity map, and so ensures that only 1 unique object per primary key value exists at a time during a session's lifetime. You can verify this with the following assertion in your own code:
assert existing is reading
The Core insert (or update) statements you are executing do not keep the session in sync with the changes taking place in the database the way for example Query.update() does. In order to fetch the new values you can expire the ORM loaded state of the unique object:
DBSession.expire(existing) # or reading, does not matter
# existing record was updated
assert reading.period_usage == 0.200
assert reading.id == existing.id