I have a FastAPI app and I need to populate a testing DB with some data needed for testing using pyTest.
This is my code for testing DB in conftest.py:
SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db"
engine = create_engine(
SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
)
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base.metadata.drop_all(bind=engine)
Base.metadata.create_all(bind=engine)
def override_get_db():
"""Redirect request to use testing DB."""
try:
db = TestingSessionLocal()
yield db
finally:
db.close()
app.dependency_overrides[get_db] = override_get_db
#pytest.fixture(scope="module")
def test_client():
"""Test client initiation for all tests."""
client = TestClient(app)
yield client
I need to implement something like this:
#pytest.fixture(scope="function")
def test_data(get_db):
waiter = Waiter(
id=1,
username="User name",
password="$12$BQhTQ6/OLAmkG/LU6G2J2.ngFk6EI9hBjFNjeTnpj2eVfQ3DCAtT.",
)
dish = Dish(
id=1,
name="Some dish",
description="Some description",
image_url="https://some.io/fhjhd.jpg",
cost=1.55,
)
get_db.add(waiter)
get_db.add(dish)
get_db.commit()
And here is a test:
def test_get_waiter(test_client, waiter_data):
"""Test Get a waiter by id."""
response = test_client.get("/waiters/1")
assert response.status_code == 200
But in this case I get fixture 'get_db' not found. How do I?
you must set a fixture with get_db name:
#pytest.fixture(name="get_db")
def get_test_db():
return override_get_db()
I suggest you have a conftest.py and put this config into it.
I've handled it in this way in conftest.py:
#pytest.fixture(scope="module")
def db_session():
"""Fixture to connect with DB."""
connection = engine.connect()
session = TestingSessionLocal(bind=connection)
yield session
session.close()
#pytest.fixture(scope="function")
def test_db_data(db_session):
waiter = Waiter(
username="User name",
password="$12$BQhTQ6/OLAmkG/LU6G2J2.ngFk6EI9hBjFNjeTnpj2eVfQ3DCAtT.",
)
db_session.add(waiter)
db_session.commit()
yield db_session
Related
I am trying to get the python testcontainer working with marshmallow to run a test but I don't seem to be able to get it working. I always get the connection refused when I run the test. I have created a sqlalchemy engine and tested it, and this works fine but when passing the testcontainer connection string as a marshmallow config, it just doesn't work. Below is my base test class.
class BaseTestCase(TestCase):
base_url = '/api/v1'
def create_app(self):
config = MySqlContainer('mysql:8.0.19')
with config as mysql:
print(mysql.get_connection_url())
e = sqlalchemy.create_engine(mysql.get_connection_url())
result = e.execute("select version()")
for row in result:
print("Printing::::::::::::::::::::::" + str(row))
result.close()
logging.getLogger('connexion.operation').setLevel('ERROR')
connex_app = connexion.App(__name__, specification_dir='../../api/')
connex_app.app.json_encoder = JSONEncoder
connex_app.add_api('static/openapi.yaml')
app = connex_app.app
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = mysql.get_connection_url()
app.config['LOG_LEVEL'] = 'DEBUG'
bcrypt.init_app(app)
db.init_app(app)
ma.init_app(app)
print("Finished setting up")
return app
I can get this to work when I use sqlite as the connection string.
class BaseTestCase(TestCase):
base_url = '/api/v1'
def create_app(self):
class Config:
PORT = 5000
SQLALCHEMY_TRACK_MODIFICATIONS = False
FLASK_ENV = 'local'
SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
LOG_LEVEL = 'DEBUG'
logging.getLogger('connexion.operation').setLevel('ERROR')
connex_app = connexion.App(__name__, specification_dir='../../api/')
connex_app.app.json_encoder = JSONEncoder
connex_app.add_api('static/openapi.yaml')
app = connex_app.app
app.config.from_object(Config)
bcrypt.init_app(app)
db.init_app(app)
ma.init_app(app)
print("Finished setting up test")
return app
Any help will be much appreciated.
Paul
I decided to use testcontainers.compose and the connection string seems to work.
class BaseTestCase(TestCase):
base_url = '/api/v1'
def create_app(self):
self.compose = testcontainers.compose.DockerCompose(".")
self.compose.start()
time.sleep(10)
class Config:
PORT = 3306
SQLALCHEMY_TRACK_MODIFICATIONS = False
FLASK_ENV = 'local'
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root:password#0.0.0.0:3306/test-db'
LOG_LEVEL = 'DEBUG'
logging.getLogger('connexion.operation').setLevel('ERROR')
connex_app = connexion.App(__name__, specification_dir='../../api/')
connex_app.app.json_encoder = JSONEncoder
connex_app.add_api('static/openapi.yaml')
app = connex_app.app
app.config.from_object(Config)
bcrypt.init_app(app)
db.init_app(app)
ma.init_app(app)
print("Finished setting up test")
return app
I cover the project with tests using pytest.
In each application (module), I created the tests folder, inside which placed the files with the application tests.
In each tests folder there are conftest fixtures for each application.
When I run the tests separately for each application (like pytest apps/users) everything works fine.
But when I run the tests entirely for the whole project (just pytest) for the first application the tests pass, but then it throws the sqlalchemy.exc.ResourceClosedError: This Connection is closed error for other application
Example of conftest.py
import os
import pytest
TESTDB = "test.db"
TESTDB_PATH = os.path.join(basedir, TESTDB)
#pytest.fixture(scope="session")
def app(request):
"""Session-wide test `Flask` application."""
app = create_app("config.TestConfig")
# Establish an application context before running the tests.
ctx = app.app_context()
ctx.push()
def teardown():
ctx.pop()
request.addfinalizer(teardown)
return app
#pytest.fixture(scope="session")
def db(app, request):
"""Session-wide test database."""
if os.path.exists(TESTDB_PATH):
os.unlink(TESTDB_PATH)
def teardown():
_db.drop_all()
try:
os.unlink(TESTDB_PATH)
except FileNotFoundError:
pass
_db.app = app
_db.create_all()
permission = PermissionModel(title="can_search_articles")
role = RoleModel(title="API User", permissions=[permission])
tag = TagModel(name="Test tag")
article = ArticleModel(
title="Test article",
legal_language="en",
abstract="",
state="Alaska",
tags=[tag],
)
_db.session.add_all([role, permission, tag, article])
_db.session.commit()
user1 = UserModel(email="test#gmail.com", role_id=role.id)
user2 = UserModel(email="test2#gmail.com")
_db.session.add_all([user1, user2])
# Commit the changes for the users
_db.session.commit()
request.addfinalizer(teardown)
return _db
#pytest.fixture(scope="function")
def session(db, request):
"""Creates a new database session for a test."""
connection = db.engine.connect()
transaction = connection.begin()
options = dict(bind=connection, binds={})
session = db.create_scoped_session(options=options)
db.session = session
def teardown():
transaction.rollback()
connection.close()
session.remove()
request.addfinalizer(teardown)
return session
#pytest.fixture(scope="module")
def client(app):
client = app.test_client()
ctx = app.app_context()
ctx.push()
yield client
ctx.pop()
structure of project
proj/
__apps/
____articles/
______models.py, views.py, __init__.py etc
______tests/
________|__init__.py
________test_models.py
________conftest.py
____users/
______models.py, views.py, __init__.py etc
______tests/
________|__init__.py
________test_models.py
________conftest.py
______init__.py # Here I load my models, register blueprints
__main.py # Here I run my application
You can not have two simultaneous connections to sqlite database. Also you have two connections here, one explicit in session fixture, you open and close it by your self, and second implicit in db fixture (_db.session), probably closing not happen here. So, try use connection implicit and only once, instead db and session fixtures make only session fixture:
#pytest.fixture
def session(app):
"""Creates a new database session for a test."""
db.app = app
db.create_all()
with db.engine.connect() as connection:
with connection.begin() as transaction:
options = dict(bind=connection, binds={})
session = db.create_scoped_session(options=options)
db.session = session
prepare_data(session)
yield session
transaction.rollback()
db.drop_all()
here prepare_data is your data filling of new db:
def prepare_data(session):
permission = PermissionModel(title="can_search_articles")
role = RoleModel(title="API User", permissions=[permission])
tag = TagModel(name="Test tag")
article = ArticleModel(
title="Test article",
legal_language="en",
abstract="",
state="Alaska",
tags=[tag],
)
session.add_all([role, permission, tag, article])
session.commit()
user1 = UserModel(email="test#gmail.com", role_id=role.id)
user2 = UserModel(email="test2#gmail.com")
session.add_all([user1, user2])
# Commit the changes for the users
session.commit()
because session fixture here is function-scope, in each test you will have your one database. It will be more practical dont fill database fully each time, but split this prepare_data to few separate fixtures, each for one object, and use them in tests where they exactly needed.
when I run tests It succeeds to connect to the database, but it does not create tables. I think maybe there is a different way to create tables when I use flask-sqlalchemy, but I can't find the solution.
This is app.py
db = SQLAlchemy()
def create_app(config_name):
app = Flask(__name__, template_folder='templates')
app.wsgi_app = ProxyFix(app.wsgi_app)
app.config.from_object(config_name)
app.register_blueprint(api)
db.init_app(app)
#app.route('/ping')
def health_check():
return jsonify(dict(ok='ok'))
#app.errorhandler(404)
def ignore_error(err):
return jsonify()
app.add_url_rule('/urls', view_func=Shorty.as_view('urls'))
return app
This is run.py
environment = environ['TINY_ENV']
config = config_by_name[environment]
app = create_app(config)
if __name__ == '__main__':
app.run()
This is config.py
import os
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
"""
set Flask configuration vars
"""
# General config
DEBUG = True
TESTING = False
# Database
SECRET_KEY = os.environ.get('SECRET_KEY', 'my_precious_secret_key')
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root#localhost:3306/tiny'
SQLALCHEMY_TRACK_MODIFICATIONS = False
SERVER_HOST = 'localhost'
SERVER_PORT = '5000'
class TestConfig(Config):
"""
config for test
"""
TESTING = True
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://root#localhost:3306/test_tiny'
config_by_name = dict(
test=TestConfig,
local=Config
)
key = Config.SECRET_KEY
This is models.py
from datetime import datetime
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class URLS(db.Model):
__tablename__ = 'urls'
id = db.Column(db.Integer, primary_key=True)
original_url = db.Column(db.String(400), nullable=False)
short_url = db.Column(db.String(200), nullable=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow()
This is test config setting.
db = SQLAlchemy()
#pytest.fixture(scope='session')
def app():
test_config = config_by_name['test']
app = create_app(test_config)
app.app_context().push()
return app
#pytest.fixture(scope='session')
def client(app):
return app.test_client()
#pytest.fixture(scope='session')
def init_db(app):
db.init_app(app)
db.create_all()
yield db
db.drop_all()
The following might be the problem that is preventing your code from running multiple times and/or preventing you from dropping/creating your tables. Regardless if it solves your problem, it is something one might not be aware of and quite important to keep in mind. :)
When you are running your tests multiple times, db.drop_all() might not be called (because one of your tests failed) and therefore, it might not be able to create the tables on the next run (since they are already existing). The problem lies in using a context manager without a try: finally:. (NOTE: Every fixture using yield is a context manager).
from contextlib import contextmanager
def test_foo(db):
print('begin foo')
raise RuntimeError()
print('end foo')
#contextmanager
def get_db():
print('before')
yield 'DB object'
print('after')
This code represents your code, but without using the functionality of pytest. Pytest is running it more or less like
try:
with get_db(app) as db:
test_foo(db)
except Exception as e:
print('Test failed')
One would expect an output similar to:
before
begin_foo
after
Test failed
but we only get
before
begin_foo
Test failed
While the contextmanager is active (yield has been executed), our test method is running. If an exception is raised during the execution of our test function, the execution is stopped WITHOUT running any code after the yield statement. To prevent this, we have to wrap our fixture/contextmanager in a try: ... finally: block. As finally is ALWAYS executed regardless of what has happened.
#contextmanager
def get_db():
print('before')
try:
yield 'DB object'
finally:
print('after')
The code after the yield statement is now executed as expected.
before
begin foo
after
Test failed
If you want to learn more, see the relevant section in the contextmanager docs:
At the point where the generator yields, the block nested in the with statement is
executed. The generator is then resumed after the block is exited. If an unhandled
exception occurs in the block, it is reraised inside the generator at the point
where the yield occurred. Thus, you can use a try…except…finally statement to trap
the error (if any), or ensure that some cleanup takes place.
I am trying to write some test for my application. I can manage to test URLs which does not contain anything related to db.
#pytest.fixture
def client():
"""
Load pytest fixture.
:return: test_client()
"""
# yield application.test_client()
db_fd, application.config['DATABASE'] = tempfile.mkstemp()
db.create_all()
application.config['TESTING'] = True
client = application.test_client()
yield client
os.unlink(application.config['DATABASE'])
my test file:
def test_membership_register(client):
"""Test entry point of website."""
rv = client.get('/register')
new_user = User(
username='test_user_1',
email='test_email#test1.com',
password='test1_Password',
)
db.session.add(new_user)
db.session.commit()
assert 200 == rv.status_code
ERROR:
E sqlalchemy.exc.InvalidRequestError: Table 'membership_user' is already defined for this MetaData instance. Specify 'extend_existing=True' to redefine options and columns on an existing Table object.
I'm trying to use pytest to write functional tests for a Flask application which interfaces with a Neo4j graph database via the Neo4j driver.
The toy example using the Movie Database is outlined below using a route which deletes a record from the database. For testing purposes I would like to wrap the session in a transaction which would be rolled-back rather than committed.
The application has routes which runs Cypher statements within an auto-commit transaction via, session.run(...), however I can circumvent this logic during testing by enforcing the use of a transaction prior to request,
session.begin_transaction()
...
session.rollback_transaction()
My question is I'm unsure how to leverage this pattern using pytest. Do I have to somehow bind the database to the client? Also is there a fixture I can use which will ensure that every test leveraging the client will be wrapped in a transaction which can be rolled back?
myapp/app.py:
from flask import _app_ctx_stack, Flask, Response
from flask_restplus import Api, Resource
from neo4j.v1 import GraphDatabase
class FlaskGraphDatabase(object):
def __init__(self, app=None):
self.app = app
if app is not None:
self.init_app(app)
def init_app(self, app):
#app.teardown_appcontext
def teardown(exception):
ctx = _app_ctx_stack.top
if hasattr(ctx, 'neo4j_session'):
ctx.neo4j_session.close()
if hasattr(ctx, 'neo4j_driver'):
ctx.neo4j_driver.close()
#property
def driver(self):
ctx = _app_ctx_stack.top
if ctx is not None:
if not hasattr(ctx, 'neo4j_driver'):
ctx.neo4j_driver = GraphDatabase.driver('bolt://localhost:7687')
return ctx.neo4j_driver
#property
def session(self):
ctx = _app_ctx_stack.top
if ctx is not None:
if not hasattr(ctx, 'neo4j_session'):
ctx.neo4j_session = self.driver.session()
return ctx.neo4j_session
api = Api()
gdb = FlaskGraphDatabase()
#api.route('/<source>/acted_in/<target>')
class Friend(Resource):
def delete(self, source, target):
statement = """
MATCH (s:Person)-[r:ACTED_IN]->(t:Movie)
WHERE s.name = {source} AND t.title = {target}
DELETE r
"""
cursor = gdb.session.run(statement, source=source, target=target)
status = 204 if cursor.summary().counters.contains_updates else 404
return Response(status=status)
def create_app():
app = Flask(__name__)
gdb.init_app(app)
api.init_app(app)
return app
if __name__ == '__main__':
app = create_app()
app.run()
tests/conftest.py:
import pytest
from myapp.app import create_app
#pytest.yield_fixture(scope='session')
def app():
yield create_app()
#pytest.yield_fixture(scope='session')
def client(app):
with app.test_client() as client:
yield client
tests/test_routes.py:
def test_delete(client):
res = client.delete('/Keanu Reeves/acted_in/The Matrix')
assert res.status_code == 204
Yes you can use a fixture to achieve this : add an autouse fixture with session scope in your conftest.py which will start a transaction at the begining of the test session and roll it back at the end.
tests/conftest.py:
from neomodel import db
#pytest.fixture(autouse=True, scope="session")
def setup_neo_test_db():
print("Initializing db transaction")
db.begin()
yield
print("Rolling back")
db.rollback()