I'm using flask with sqlalchemy and sqlite db. I have 2 ajax that send some data from html to my .py file.
The problem is every time when i do any of these 2 operations, the second one become unavailable because of lock of db. Also, if first chosen action will be deleting, then exception firing no matter what operation will be chosen after. with first choice of adding, we can add without limitations that's strange too, because functions seem similar.
I've tried timeouts, closing sessions in a different ways, the result is always the same.
Here are two functions-handlers:
app = Flask(__name__)
csrf = CSRFProtect(app)
app.config.from_object('config')
db = SQLAlchemy(app)
import forms
import models
#app.route('/delete', methods = ['GET', 'POST'])
def delete():
if request.method == "POST":
if request.form['type'] == "delete":
print("delete")
engine = create_engine(SQLALCHEMY_DATABASE_URI)
Session = sessionmaker(bind=engine)
session = Session()
try:
print("try")
requested = request.form['id']
print(requested)
models.Income.query.filter(models.Income.id == requested).delete()
session.commit()
except:
print("rollback")
session.rollback()
finally:
print("fin")
session.close()
ellist = models.Income.query.all()
return render_template("incomeSection.html", list=ellist)
#app.route('/add', methods=['GET', 'POST'])
def add():
if request.method == "POST":
if request.form['type'] == "add":
print('add')
engine = create_engine(SQLALCHEMY_DATABASE_URI)
Session = sessionmaker(bind=engine)
session = Session()
try:
print("try")
newItem = models.Income(name=request.form['name'], tag=request.form['tag'],
account=request.form['account'],
date=date(*(int(i) for i in request.form['date'].split("-"))))
session.add(newItem)
session.commit()
except:
print('rollback')
session.rollback()
finally:
print("fin")
session.close()
ellist = models.Income.query.all()
print(ellist)
return render_template("incomeSection.html", list=ellist)
I've read that this exception caused by non-closed connections, but I have .close() in every finally block. I think the problem might be because of the db = SQLAlchemy(app) but I don't know how to fix if that is the case. Because I use this variable to connect with db in forms.py where I have the form template and in models.py where I defined my tables within db.
So, aperrently, thete was an issue with number of connections.
The thing that solved my problem was the context manager for sqlalchemy, i used this one:
class SQLAlchemyDBConnection(object):
def __init__(self, connection_string):
self.connection_string = connection_string
self.session = None
def __enter__(self):
engine = create_engine(self.connection_string)
Session = sessionmaker()
self.session = Session(bind=engine)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.session.commit()
self.session.close()
and in the handler just
with SQLAlchemyDBConnection(SQLALCHEMY_DATABASE_URI) as db:
newItem = models.Income(*your params*)
db.session.add(newItem)
And now it works fine, but i still don't know what was the issue in version that was earlier. They are seem to be same just with or without context manager
Related
I have a CRUD with insert and update functions with commit at the end of the each one as follows:
#staticmethod
def insert(db: Session, item: Item) -> None:
db.add(item)
db.commit()
#staticmethod
def update(db: Session, item: Item) -> None:
...
db.commit()
I have an endpoint which receives a sqlalchemy session from a FastAPI dependency and needs to insert and update atomically (DB transaction).
What's the best practice when working with transactions? I can't work with the CRUD since it does more than one commit.
How should I handle the transactions? Where do you commit your session? in the CRUD? or only once in the FastAPI dependency function for each request?
I had the same problem while using FastAPI. I couldn't find a way to use commit in separate methods and have them behave transactionally.
What I ended up doing was a flush instead of the commit, which sends the changes to the db, but doesn't commit the transaction.
One thing to note, is that in FastAPI every request opens a new session and closes it once its done. This would be a rough example of what is happening using the example in the SQLAlchemy docs.
def run_my_program():
# This happens in the `database = SessionLocal()` of the `get_db` method below
session = Session()
try:
ThingOne().go(session)
ThingTwo().go(session)
session.commit()
except:
session.rollback()
raise
finally:
# This is the same as the `get_db` method below
session.close()
The session that is generated for the request is already a transaction. When you commit that session what is actually doing is this
When using the Session in its default mode of autocommit=False, a new transaction will be begun immediately after the commit, but note that the newly begun transaction does not use any connection resources until the first SQL is actually emitted.
In my opinion after reading that it makes sense handling the commit and rollback at the endpoint scope.
I created a dummy example of how this would work. I use everything form the FastAPI guide.
def create_user(db: Session, user: UserCreate):
"""
Create user record
"""
fake_hashed_password = user.password + "notreallyhashed"
db_user = models.User(email=user.email, hashed_password=fake_hashed_password)
db.add(db_user)
db.flush() # Changed this to a flush
return db_user
And then use the crud operations in the endpoint as follows
from typing import List
from fastapi import Depends, HTTPException
from sqlalchemy.orm import Session
...
def get_db():
"""
Get SQLAlchemy database session
"""
database = SessionLocal()
try:
yield database
finally:
database.close()
#router.post("/users", response_model=List[schemas.User])
def create_users(user_1: schemas.UserCreate, user_2: schemas.UserCreate, db: Session = Depends(get_db)):
"""
Create two users
"""
try:
user_1 = crud.create_user(db=db, user=user_1)
user_2 = crud.create_user(db=db, user=user_2)
db.commit()
return [user_1, user_2]
except:
db.rollback()
raise HTTPException(status_code=400, detail="Duplicated user")
In the future I might investigate moving this to a middleware, but I don't think that using commit you can get the behavior you want.
A more pythonic approach is to let a context manager perform a commit or rollback depending on whether or not there was an exception.
A Transaction is a nice abstraction of what we are trying to accomplish.
class Transaction:
def __init__(self, session: Session = Depends(get_session)):
self.session = session
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
# rollback and let the exception propagate
self.session.rollback()
return False
self.session.commit()
return True
And, use it in your APIs, like so:
def some_api(tx: Transaction = Depends(Transaction)):
with tx:
ThingOne().go()
ThingTwo().go()
No need to pass session to ThingOne and ThingTwo. Inject it into them, like so:
class ThingOne:
def __init__(self, session: Session = Depends(get_session)):
...
class ThingTwo:
def __init__(self, session: Session = Depends(get_session)):
...
I would also inject ThingOne and ThingTwo in the APIs as well:
def some_api(tx: Transaction = Depends(Transaction),
one: ThingOne = Depends(ThingOne),
two: ThingTwo = Depends(ThingTwo)):
with tx:
one.go()
two.go()
when I run tests It succeeds to connect to the database, but it does not create tables. I think maybe there is a different way to create tables when I use flask-sqlalchemy, but I can't find the solution.
This is app.py
db = SQLAlchemy()
def create_app(config_name):
app = Flask(__name__, template_folder='templates')
app.wsgi_app = ProxyFix(app.wsgi_app)
app.config.from_object(config_name)
app.register_blueprint(api)
db.init_app(app)
#app.route('/ping')
def health_check():
return jsonify(dict(ok='ok'))
#app.errorhandler(404)
def ignore_error(err):
return jsonify()
app.add_url_rule('/urls', view_func=Shorty.as_view('urls'))
return app
This is run.py
environment = environ['TINY_ENV']
config = config_by_name[environment]
app = create_app(config)
if __name__ == '__main__':
app.run()
This is config.py
import os
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
"""
set Flask configuration vars
"""
# General config
DEBUG = True
TESTING = False
# Database
SECRET_KEY = os.environ.get('SECRET_KEY', 'my_precious_secret_key')
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root#localhost:3306/tiny'
SQLALCHEMY_TRACK_MODIFICATIONS = False
SERVER_HOST = 'localhost'
SERVER_PORT = '5000'
class TestConfig(Config):
"""
config for test
"""
TESTING = True
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root#localhost:3306/test_tiny'
config_by_name = dict(
test=TestConfig,
local=Config
)
key = Config.SECRET_KEY
This is models.py
from datetime import datetime
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class URLS(db.Model):
__tablename__ = 'urls'
id = db.Column(db.Integer, primary_key=True)
original_url = db.Column(db.String(400), nullable=False)
short_url = db.Column(db.String(200), nullable=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow()
This is test config setting.
db = SQLAlchemy()
#pytest.fixture(scope='session')
def app():
test_config = config_by_name['test']
app = create_app(test_config)
app.app_context().push()
return app
#pytest.fixture(scope='session')
def client(app):
return app.test_client()
#pytest.fixture(scope='session')
def init_db(app):
db.init_app(app)
db.create_all()
yield db
db.drop_all()
The following might be the problem that is preventing your code from running multiple times and/or preventing you from dropping/creating your tables. Regardless if it solves your problem, it is something one might not be aware of and quite important to keep in mind. :)
When you are running your tests multiple times, db.drop_all() might not be called (because one of your tests failed) and therefore, it might not be able to create the tables on the next run (since they are already existing). The problem lies in using a context manager without a try: finally:. (NOTE: Every fixture using yield is a context manager).
from contextlib import contextmanager
def test_foo(db):
print('begin foo')
raise RuntimeError()
print('end foo')
#contextmanager
def get_db():
print('before')
yield 'DB object'
print('after')
This code represents your code, but without using the functionality of pytest. Pytest is running it more or less like
try:
with get_db(app) as db:
test_foo(db)
except Exception as e:
print('Test failed')
One would expect an output similar to:
before
begin_foo
after
Test failed
but we only get
before
begin_foo
Test failed
While the contextmanager is active (yield has been executed), our test method is running. If an exception is raised during the execution of our test function, the execution is stopped WITHOUT running any code after the yield statement. To prevent this, we have to wrap our fixture/contextmanager in a try: ... finally: block. As finally is ALWAYS executed regardless of what has happened.
#contextmanager
def get_db():
print('before')
try:
yield 'DB object'
finally:
print('after')
The code after the yield statement is now executed as expected.
before
begin foo
after
Test failed
If you want to learn more, see the relevant section in the contextmanager docs:
At the point where the generator yields, the block nested in the with statement is
executed. The generator is then resumed after the block is exited. If an unhandled
exception occurs in the block, it is reraised inside the generator at the point
where the yield occurred. Thus, you can use a try…except…finally statement to trap
the error (if any), or ensure that some cleanup takes place.
I am building an app where users will occasionally initiate a longer-running process. While running, the process will commit updates to a database entry.
Since the process takes some time, I am using the threading module to execute it. But values updated while in the thread are never actually committed.
An example:
from flask import Flask, url_for, redirect
from flask_sqlalchemy import SQLAlchemy
import time, threading, os
if os.path.exists('test.db'): os.remove('test.db')
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
db = SQLAlchemy(app)
class Item(db.Model):
id = db.Column(db.Integer, primary_key=True)
value = db.Column(db.Integer)
def __init__(self, value): self.value = value
db.create_all()
item = Item(1)
db.session.add(item)
db.session.commit()
#app.route('/go', methods=['GET'])
def go():
def fun(item):
time.sleep(2)
item.value += 1
db.session.commit()
thr = threading.Thread(target=fun, args=(item,))
# thr.daemon = True
thr.start()
return redirect(url_for('view'))
#app.route('/view', methods=['GET'])
def view(): return str(Item.query.get(1).value)
app.run(host='0.0.0.0', port=8080, debug=True)
My expectation was that the item's value would be asynchronously updated after two seconds (when the fun completes), and that additional requests to /view would reveal the updated value. But this never occurs. I am not an expert on what is going on in the threading module; am I missing something?
I have tried setting thr.daemon=True as pointed out in some posts; but that is not it. The closest SO post I have found is this one; that question does not have a minimal and verifiable example and has not been answered.
I guess this is due to the fact that sessions are local threaded, as mentioned in the documentation. In your case, item was created in one thread and then passed to a new thread to be modified directly.
You can either use scoped sessions as suggested in the documentation, or simply change your URI config to bypass this behavior:
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db?check_same_thread=False'
After some debugging I figured out a solution; though I still do not understand the problem. It has to do with referencing a variable for the database object. If fun updates an object returned by a query, it works as expected:
def fun(item_id):
time.sleep(2)
Item.query.get(item_id).value += 1
db.session.commit()
In context:
from flask import Flask, url_for, redirect
from flask_sqlalchemy import SQLAlchemy
import time, threading, os
if os.path.exists('test.db'): os.remove('test.db')
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///test.db'
db = SQLAlchemy(app)
class Item(db.Model):
id = db.Column(db.Integer, primary_key=True)
value = db.Column(db.Integer)
def __init__(self, value): self.value = value
db.create_all()
item = Item(1)
db.session.add(item)
db.session.commit()
#app.route('/go', methods=['GET'])
def go():
def fun(item_id):
time.sleep(2)
Item.query.get(item_id).value += 1
db.session.commit()
thr = threading.Thread(target=fun, args=(item.id,))
# thr.daemon = True
thr.start()
return redirect(url_for('view'))
#app.route('/view', methods=['GET'])
def view(): return str(Item.query.get(1).value)
app.run(host='0.0.0.0', port=8080, debug=True)
I would be very pleased to hear from anyone knows what exactly is going on here!
I am trying to run my create_db.py file to create the posts.db database, but it will not get created in my project directory. And when I run the main file for the blog and try to login in I get the error below.
I have looked up this error and seen that other people have gotten it as well and asked about it here on Stackoverflow, but none of them seems to help me. I have read that this could be because something in my blog.py file is running main before the database gets created. But, I am thinking that it has something to do with the configuration. Mainly the PATH of the database could be getting mixed up with the app.config['SQLAlCHEMY_DATABASE_URI'] = 'sqlite:///' line.
Any ideas?
Here is the error
sqlalchemy.exc.OperationalError
OperationalError: (OperationalError) no such table: posts u'SELECT posts.id AS posts_id, posts.title AS posts_title, posts.post AS posts_post \nFROM posts' ()
OperationalError: (OperationalError) no such table: posts u'SELECT posts.id AS posts_id, posts.title AS posts_title, posts.post AS posts_post \nFROM posts' ()
Here is my code. There are three files here: blog.py, models.py, create_db.py
blog.py
# controller of blog app
from flask import Flask, render_template, request, session,\
flash, redirect, url_for, g
from flask.ext.sqlalchemy import SQLAlchemy
import sqlite3
from functools import wraps
# create the application object
app = Flask(__name__)
# configuration
app.secret_key = 'x13xa8xf5}[xfexd4Zxb8+x07=7xc9xe1Bxcfxbdt.ox87oxc9'
app.config['SQLAlCHEMY_DATABASE_URI'] = 'sqlite:///'
# create sqlalchemy object
db = SQLAlchemy(app)
from models import *
# login required decorator
def login_required(test):
#wraps(test)
def wrap(*args, **kwargs):
if 'logged_in' in session:
return test(*args, **kwargs)
else:
flash('You need to login first.')
return redirect(url_for('login'))
return wrap
#app.route('/', methods = ['GET','POST'])
def login():
error = None
if request.method == 'POST':
if request.form['username'] != 'admin' or request.form['password'] != 'admin':
error = 'Invalid Credentils. Please try again.'
else:
session['logged_in'] = True
return redirect(url_for('main'))
return render_template('login.html', error=error)
#app.route('/main')
#login_required
def main():
posts = db.session.query(BlogPost).all()
return render_template('main.html', posts=posts)
#app.route('/add', methods=['POST'])
#login_required
def add():
title = request.form['title']
post = request.form['post']
if not title or not post:
flash("All fields are required. Please try again.")
return redirect(url_for('main'))
else:
db.session.add(title)
db.session.add(post)
db.session.commit()
db.session.close()
flash('New entry was successfully posted!')
return redirect(url_for('main'))
#app.route('/logout')
def logout():
session.pop('logged_in', None)
flash('You were logged out')
return redirect(url_for('login'))
def connect_db():
return sqlite.connect('posts.db')
if __name__ == '__main__':
app.run(debug = True)
models.py:
from blog import db
class BlogPost(db.Model):
__tablename__ = "posts"
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String, nullable=False)
post = db.Column(db.String, nullable=False)
def __init__(self, title, post):
self.title = title
self.post = post
def __repr__(self):
return '<title {}'.format(self.title)
create_db.py
from blog import db
from models import BlogPost
# create the database and the db tables
db.create_all()
# insert
db.session.add(BlogPost("Good","I\'m good."))
db.session.add(BlogPost("Well","I\'m well."))
db.session.add(BlogPost("Post","I\'m a post."))
# commit the changes
db.session.commit()
There's a typo in SQLAlCHEMY_DATABASE_URI, should be SQLALCHEMY_DATABASE_URI, the 2nd l.
When running from blog import db some statements in blog.py get executed, including the one with sqlite:/// which is where the path is set. Modifying this line
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///posts.db'
and then after running create_db.py should create a database in app's root directory. There doesn't seem to be another place in the code where the db path is set.
It looks like you may also run into some circular import problems (will throw ImportError: cannot import name [name]). Some solutions are putting the app init stuff like db = SQLAlchemy.. into a separate file or importing at the end of the file.
I have a flask application that has a view function that looks like this:
#login_required
#app.route('/suspect_tracker/new_list', methods=['GET', 'POST'])
def new_list():
form = ListForm()
if form.validate_on_submit():
if form.private:
privacy_setting = 1
else:
privacy_setting = 0
new_list = List(name=form.name.data, last_updated=datetime.utcnow(), user_id=g.user.id, private=privacy_setting)
db.session.add(new_list)
db.session.commit()
flash('New list %s added' % new_list.name)
return redirect('/suspect_tracker/' + form.name.data)
else:
flash(form.name.errors)
return render_template('newlist.html', title="Make a new list!", form=form)
And I am attempting to write a test for it, with the test looking like this:
from config import basedir
from app import app, db
from app.models import User, List, Suspect, SuspectList
from flask import g
from flask_testing import TestCase
class TestViews(TestCase):
def create_app(self):
app.config['TESTING'] = True
app.config['WTF_CSRF_ENABLED'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'test.db')
self.client = app.test_client()
return app
def setUp(self):
self.app = self.create_app()
db.create_all()
test_user = User(nickname="test")
db.session.add(test_user)
db.session.commit()
g.user = User.query.filter_by(id=1).first()
def tearDown(self):
db.session.remove()
db.drop_all()
def test_new_list_view(self):
self.client.get('/suspect_tracker/new_list/')
form = {'name':'Test', 'private':False}
g.user = User.query.filter_by(id=1).first()
self.client.post('/suspect_tracker/new_list', data=form)
assert List.query.filter_by(name="Test").first() != None
After running the test, the assertion fails, and I have tested to see that after running self.client.post, List does not contain a new list, and is still empty. This leads me to believe that form.validate_on_submit() returned false, which leads me to believe that I am not passing the correct data in the self.client.post() function in test_new_list_view(). My question is, how do I correctly create a ListForm() in the testing function and then POST it with self.client.post() to test the new_list() function?
I've fixed the issue with getting the form data to properly be sent, so now the form validates but I have no idea how to set g.user to be the mock user so when I try running the test I get an "AttributeError: 'AnonymousUserMixin' object has no attribute 'id'" error.