I have a CRUD with insert and update functions with commit at the end of the each one as follows:
#staticmethod
def insert(db: Session, item: Item) -> None:
db.add(item)
db.commit()
#staticmethod
def update(db: Session, item: Item) -> None:
...
db.commit()
I have an endpoint which receives a sqlalchemy session from a FastAPI dependency and needs to insert and update atomically (DB transaction).
What's the best practice when working with transactions? I can't work with the CRUD since it does more than one commit.
How should I handle the transactions? Where do you commit your session? in the CRUD? or only once in the FastAPI dependency function for each request?
I had the same problem while using FastAPI. I couldn't find a way to use commit in separate methods and have them behave transactionally.
What I ended up doing was a flush instead of the commit, which sends the changes to the db, but doesn't commit the transaction.
One thing to note, is that in FastAPI every request opens a new session and closes it once its done. This would be a rough example of what is happening using the example in the SQLAlchemy docs.
def run_my_program():
# This happens in the `database = SessionLocal()` of the `get_db` method below
session = Session()
try:
ThingOne().go(session)
ThingTwo().go(session)
session.commit()
except:
session.rollback()
raise
finally:
# This is the same as the `get_db` method below
session.close()
The session that is generated for the request is already a transaction. When you commit that session what is actually doing is this
When using the Session in its default mode of autocommit=False, a new transaction will be begun immediately after the commit, but note that the newly begun transaction does not use any connection resources until the first SQL is actually emitted.
In my opinion after reading that it makes sense handling the commit and rollback at the endpoint scope.
I created a dummy example of how this would work. I use everything form the FastAPI guide.
def create_user(db: Session, user: UserCreate):
"""
Create user record
"""
fake_hashed_password = user.password + "notreallyhashed"
db_user = models.User(email=user.email, hashed_password=fake_hashed_password)
db.add(db_user)
db.flush() # Changed this to a flush
return db_user
And then use the crud operations in the endpoint as follows
from typing import List
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
...
def get_db():
"""
Get SQLAlchemy database session
"""
database = SessionLocal()
try:
yield database
finally:
database.close()
#router.post("/users", response_model=List[schemas.User])
def create_users(user_1: schemas.UserCreate, user_2: schemas.UserCreate, db: Session = Depends(get_db)):
"""
Create two users
"""
try:
user_1 = crud.create_user(db=db, user=user_1)
user_2 = crud.create_user(db=db, user=user_2)
db.commit()
return [user_1, user_2]
except:
db.rollback()
raise HTTPException(status_code=400, detail="Duplicated user")
In the future I might investigate moving this to a middleware, but I don't think that using commit you can get the behavior you want.
A more pythonic approach is to let a context manager perform a commit or rollback depending on whether or not there was an exception.
A Transaction is a nice abstraction of what we are trying to accomplish.
class Transaction:
def __init__(self, session: Session = Depends(get_session)):
self.session = session
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
# rollback and let the exception propagate
self.session.rollback()
return False
self.session.commit()
return True
And, use it in your APIs, like so:
def some_api(tx: Transaction = Depends(Transaction)):
with tx:
ThingOne().go()
ThingTwo().go()
No need to pass session to ThingOne and ThingTwo. Inject it into them, like so:
class ThingOne:
def __init__(self, session: Session = Depends(get_session)):
...
class ThingTwo:
def __init__(self, session: Session = Depends(get_session)):
...
I would also inject ThingOne and ThingTwo in the APIs as well:
def some_api(tx: Transaction = Depends(Transaction),
one: ThingOne = Depends(ThingOne),
two: ThingTwo = Depends(ThingTwo)):
with tx:
one.go()
two.go()
Related
I want to store my session data on redis dataset. I have set SESSION_ENGINE = 'redis' in settings.py.
Code for redis.py
#redis.py
from django.contrib.sessions.backends.base import SessionBase
from django.utils.functional import cached_property
from redis import Redis
class SessionStore(SessionBase):
#cached_property
def _connection(self):
return Redis(
host='127.0.0.1',
port='6379',
db=0,
decode_responses=True
)
def load(self):
return self._connection.hgetall(self.session_key)
def exists(self, session_key):
return self._connection.exists(session_key)
def create(self):
# Creates a new session in the database.
self._session_key = self._get_new_session_key()
self.save(must_create=True)
self.modified = True
def save(self, must_create=False):
# Saves the session data. If `must_create` is True,
# creates a new session object. Otherwise, only updates
# an existing object and doesn't create one.
if self.session_key is None:
return self.create()
data = self._get_session(no_load=must_create)
session_key = self._get_or_create_session_key()
self._connection.hmset(session_key, data)
self._connection.expire(session_key, self.get_expiry_age())
def delete(self, session_key=None):
# Deletes the session data under the session key.
if session_key is None:
if self.session_key is None:
return
session_key = self.session_key
self._connection.delete(session_key)
#classmethod
def clear_expired(cls):
# There is no need to remove expired sessions by hand
# because Redis can do it automatically when
# the session has expired.
# We set expiration time in `save` method.
pass
I am receiving 'hmset' with mapping of length 0 error on accessing http://localhost:8000/admin in django.
After removing SESSION_ENGINE='redis' I am not receiving this error.
From redis documentation:
As per Redis 4.0.0, HMSET is considered deprecated. Please use HSET in new code.
I have replaced this line in save() method:
self._connection.hmset(session_key, data)
with:
self._connection.hset(session_key, 'session_key', session_key, data)
On making the change, it works as expected.
I cover the project with tests using pytest.
In each application (module), I created the tests folder, inside which placed the files with the application tests.
In each tests folder there are conftest fixtures for each application.
When I run the tests separately for each application (like pytest apps/users) everything works fine.
But when I run the tests entirely for the whole project (just pytest) for the first application the tests pass, but then it throws the sqlalchemy.exc.ResourceClosedError: This Connection is closed error for other application
Example of conftest.py
import os
import pytest
TESTDB = "test.db"
TESTDB_PATH = os.path.join(basedir, TESTDB)
#pytest.fixture(scope="session")
def app(request):
"""Session-wide test `Flask` application."""
app = create_app("config.TestConfig")
# Establish an application context before running the tests.
ctx = app.app_context()
ctx.push()
def teardown():
ctx.pop()
request.addfinalizer(teardown)
return app
#pytest.fixture(scope="session")
def db(app, request):
"""Session-wide test database."""
if os.path.exists(TESTDB_PATH):
os.unlink(TESTDB_PATH)
def teardown():
_db.drop_all()
try:
os.unlink(TESTDB_PATH)
except FileNotFoundError:
pass
_db.app = app
_db.create_all()
permission = PermissionModel(title="can_search_articles")
role = RoleModel(title="API User", permissions=[permission])
tag = TagModel(name="Test tag")
article = ArticleModel(
title="Test article",
legal_language="en",
abstract="",
state="Alaska",
tags=[tag],
)
_db.session.add_all([role, permission, tag, article])
_db.session.commit()
user1 = UserModel(email="test#gmail.com", role_id=role.id)
user2 = UserModel(email="test2#gmail.com")
_db.session.add_all([user1, user2])
# Commit the changes for the users
_db.session.commit()
request.addfinalizer(teardown)
return _db
#pytest.fixture(scope="function")
def session(db, request):
"""Creates a new database session for a test."""
connection = db.engine.connect()
transaction = connection.begin()
options = dict(bind=connection, binds={})
session = db.create_scoped_session(options=options)
db.session = session
def teardown():
transaction.rollback()
connection.close()
session.remove()
request.addfinalizer(teardown)
return session
#pytest.fixture(scope="module")
def client(app):
client = app.test_client()
ctx = app.app_context()
ctx.push()
yield client
ctx.pop()
structure of project
proj/
__apps/
____articles/
______models.py, views.py, __init__.py etc
______tests/
________|__init__.py
________test_models.py
________conftest.py
____users/
______models.py, views.py, __init__.py etc
______tests/
________|__init__.py
________test_models.py
________conftest.py
______init__.py # Here I load my models, register blueprints
__main.py # Here I run my application
You can not have two simultaneous connections to sqlite database. Also you have two connections here, one explicit in session fixture, you open and close it by your self, and second implicit in db fixture (_db.session), probably closing not happen here. So, try use connection implicit and only once, instead db and session fixtures make only session fixture:
#pytest.fixture
def session(app):
"""Creates a new database session for a test."""
db.app = app
db.create_all()
with db.engine.connect() as connection:
with connection.begin() as transaction:
options = dict(bind=connection, binds={})
session = db.create_scoped_session(options=options)
db.session = session
prepare_data(session)
yield session
transaction.rollback()
db.drop_all()
here prepare_data is your data filling of new db:
def prepare_data(session):
permission = PermissionModel(title="can_search_articles")
role = RoleModel(title="API User", permissions=[permission])
tag = TagModel(name="Test tag")
article = ArticleModel(
title="Test article",
legal_language="en",
abstract="",
state="Alaska",
tags=[tag],
)
session.add_all([role, permission, tag, article])
session.commit()
user1 = UserModel(email="test#gmail.com", role_id=role.id)
user2 = UserModel(email="test2#gmail.com")
session.add_all([user1, user2])
# Commit the changes for the users
session.commit()
because session fixture here is function-scope, in each test you will have your one database. It will be more practical dont fill database fully each time, but split this prepare_data to few separate fixtures, each for one object, and use them in tests where they exactly needed.
I have a Flask app, which performs operations on a mysql database. It uses sqlalchemy, and creates a session per request. In my testing framework, I set up a session, then insert appropriate data and ensure removal afterward. However, in my test, when I attempt to make sure that something was deleted (via a DELETE request to my Flask app), the row in my session is unaffected by the external thread. Do I need to close and reopen my session to ensure proper deletion, or can I refresh it somehow, or am I doing this all wrong somehow that I'm missing?
class Endpoint(flask_restful.Resource):
# in the webapp thread
def delete(self,id):
db.session.query(db.Table).filter(id=id).delete()
db.session.commit()
class Test(unittest.TestCase):
# in the testing thread
def setUp(self):
self.data = db.session.add(db.Table())
db.session.commit()
def tearDown(self):
self.data.delete()
db.session.commit()
def test_delete(self):
requests.delete(flask.url_for(Endpoint, id=self.data.id))
assert db.session.query(db.Table).filter(id = self.data.id).count() == 0
I'm using SqlAlchemy for my session management database, and have created a custom SessionInterface as per the documentation.
My dev database went down today, and now I cannot access my site, as would be expected. Is there a way for me to fall back to Flask's default session manager in this event?
Here is my current implementation of SessionInterface
class SqlAlchemySessionInterface(SessionInterface):
#...
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if sid:
# error is raised here when database is down
stored_session = DBSession.query.filter_by(sid=sid).first()
# ...
I have a naive solution to the problem of a crashed database, that leverages a in memory dict as a backup:
# A backup memory storage for sessions
backup = {}
class SqlAlchemySessionInterface(SessionInterface):
#...
def open_session(self, app, request):
sid = request.cookies.get(app.session_cookie_name)
if sid:
try:
stored_session = DBSession.query.filter_by(sid=sid).first()
except DatabaseError:
stored_session = backup.get('sid')
# ...
You can extend a default secure cookie session interface implementation.
class ReliableSessionInterface(SecureCookieSessionInterface):
def open_session(self, app, request):
try:
return self._open_db_session(app, request)
except DatabaseError:
return super().open_session(app, request)
def save_session(app, session, response):
try:
self._save_session_to_db(app, session, response)
except DatabaseError:
super().save_session(app, session, response)
However, such requirement sounds a little bit strange.
So I have a controller that renders a page. In the controller, I call multiple functions from the model that create its own sessions. For example:
def page(request):
userid = authenticated_userid(request)
user = User.get_by_id(userid)
things = User.get_things()
return {'user': user, 'things': things}
Where in the model I have:
class User:
...
def get_by_id(self, userid):
return DBSession.query(User)...
def get_things(self):
return DBSession.query(Thing)...
My question is, is creating a new session for each function optimal, or should I start a session in the controller and use the same session throughout the controller (assuming I'm both querying as well as inserting into the database in the controller)? Ex.
def page(request):
session = DBSession()
userid = authenticated_userid(request)
user = User.get_by_id(userid, session)
things = User.get_things(session)
...
return {'user': user, 'things': things}
class User:
...
def get_by_id(self, userid, session=None):
if not session:
session = DBSession()
return session.query(User)...
def get_things(self, session=None):
if not session:
session = DBSession()
return session.query(Thing)...
Your first code is OK, if your DBSession is a ScopedSession. DBSession() is not a constructor then, but just an accessor function to thread-local storage. You might speed up things a bit by passing the session explicitly, but premature optimization is the root of all evil.