Develop Reference

Flask testing with pytest, ENV is set to production?

I have built a flask app and i would like to create a test suite. Reading around it looks like pytest is the way to go; however, I am finding it very difficult to understand how to get going, I have looked at https://flask.palletsprojects.com/en/2.0.x/testing/ but am struggling to relate it to my app.
my project has a run.py at its base:
from wahoo_connect import init_app, db
from flask_migrate import Migrate
#run the app
app = init_app()
migrate = Migrate(app, db)
this is run using flask run and the .flaskenv sets the mode
FLASK_APP=run.py
#FLASK_ENV=production
FLASK_ENV=development
I have an application factory set up:
"""Main entry point into App"""
#import libraries
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager
from flask_mail import Mail
#from flask_caching import Cache
from flask_session import Session
from os import path, mkdir
import logging
from logging.handlers import SMTPHandler, RotatingFileHandler
"""Flask Global Variables"""
#database connection
db = SQLAlchemy()
migrate = Migrate()
#login manager
login = LoginManager()
login.login_view = 'auth_bp.login'
login.login_message = 'Please log in to access this page.'
login.login_message_category = "info"
#email
mail = Mail()
#cache
#cache = Cache()
#session
sess = Session()
#initialise app
def init_app():
"""Initialize the core application."""
app = Flask(__name__)
print(app.config['ENV'])
if app.config['ENV'] == 'production':
app.config.from_object('wahoo_connect.config.ProductionConfig')
elif app.config['ENV'] == 'development':
app.config.from_object('wahoo_connect.config.DevelopmentConfig')
elif app.config['ENV'] == 'testing':
app.config.from_object('wahoo_connect.config.TestingConfig')
# Initialize Plugins
db.init_app(app)
migrate.init_app(app, db)
login.init_app(app)
mail.init_app(app)
# cache.init_app(app)
sess.init_app(app)
with app.app_context():
# Import and Register Blueprints
from wahoo_connect.errors.views import errors_bp
from wahoo_connect.auth.views import auth_bp
from wahoo_connect.profile.views import profile_bp
from wahoo_connect.wahoo.views import wahoo_bp
from wahoo_connect.home.views import home_bp
app.register_blueprint(errors_bp)
app.register_blueprint(auth_bp, url_prefix='/auth')
app.register_blueprint(profile_bp)
app.register_blueprint(wahoo_bp, url_prefix='/wahoo')
app.register_blueprint(home_bp)
if not app.debug:
#log to email
if app.config['MAIL_SERVER']:
auth = None
if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']:
auth = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD'])
secure = None
if app.config['MAIL_USE_TLS']:
secure = ()
mail_handler = SMTPHandler(
mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']),
fromaddr=app.config['MAIL_USERNAME'],
toaddrs=app.config['ADMINS'],
subject='Error in Wahoo-Connect',
credentials=auth, secure=secure)
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
#log to file - heroku
if app.config['LOG_TO_STDOUT']:
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
app.logger.addHandler(stream_handler)
else:
logdir = path.dirname(app.instance_path) + '/logs'
if not path.exists(logdir):
mkdir(logdir)
file_handler = RotatingFileHandler(logdir + '/wahoo-connect.log',
maxBytes=10240, backupCount=10)
file_handler.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s '
'[in %(pathname)s:%(lineno)d]'))
file_handler.setLevel(logging.INFO)
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
app.logger.info('Wahoo-Connect Startup')
return app
from wahoo_connect import models
Everything works and I can run the app on a production server. I am trying to get started with pytest and have set up conftest.py:
import pytest
from wahoo_connect import init_app
from wahoo_connect.models import User
#pytest.fixture(scope="session")
def app():
app = init_app()
return app
this always runs the app in production mode, how do I get it to run in testing mode so that it uses the correct config from config.py
"""Flask configuration."""
from os import environ, path
from dotenv import load_dotenv
from datetime import timedelta
#basedir = path.abspath(path.dirname(__file__))
basedir = path.dirname(path.dirname(path.abspath(__file__)))
load_dotenv(path.join(basedir, '.env'))
class Config(object):
"""Base config."""
DEBUG = False
TESTING = False
SECRET_KEY = environ.get('SECRET_KEY') or 'topsecretkey'
STATIC_FOLDER = 'static'
TEMPLATES_FOLDER = 'templates'
SESSION_COOKIE_SECURE = True
#Database
SQLALCHEMY_DATABASE_URI = environ.get('DATABASE_URL') or \
'sqlite:///' + path.join(basedir, 'app.db')
#fix for heroku postgres db
if SQLALCHEMY_DATABASE_URI.startswith("postgres://"):
SQLALCHEMY_DATABASE_URI = SQLALCHEMY_DATABASE_URI.replace("postgres://", "postgresql://", 1)
SQLALCHEMY_TRACK_MODIFICATIONS = False
#Wahoo
WAHOO_CLIENT_ID=environ.get('WAHOO_CLIENT_ID')
WAHOO_CLIENT_SECRET=environ.get('WAHOO_CLIENT_SECRET')
WAHOO_CLIENT_REDIRECT=environ.get('WAHOO_CLIENT_REDIRECT')
#Email
MAIL_SERVER = environ.get('MAIL_SMTP_SERVER')
MAIL_PORT = int(environ.get('MAIL_SMTP_PORT') or 25)
MAIL_USERNAME = environ.get('MAIL_SMTP_LOGIN')
MAIL_PASSWORD = environ.get('MAIL_SMTP_PASSWORD')
MAIL_USE_TLS = environ.get('MAIL_USE_TLS') is not None
ADMINS = ['martyndwheeler#gmail.com']
#Logging
LOG_TO_STDOUT = environ.get('LOG_TO_STDOUT') or None
# Flask-Caching related configs
CACHE_TYPE = 'FileSystemCache'
CACHE_DIR = environ.get('CACHE_DIR') or None
CACHE_DEFAULT_TIMEOUT = 300
CACHE_THRESHOLD = 100
# Flask-Session related configs
SESSION_PERMANENT = False
PERMANENT_SESSION_LIFETIME = timedelta(minutes=30)
SESSION_USE_SIGNER = True
SESSION_TYPE = "filesystem"
SESSION_FILE_DIR = environ.get('SESSION_FILE_DIR')
SESSION_FILE_THRESHOLD = 100
class ProductionConfig(Config):
pass
class DevelopmentConfig(Config):
DEBUG = True
SESSION_COOKIE_SECURE = False
class TestingConfig(Config):
TESTING = True
SESSION_COOKIE_SECURE = False
If there is a good tutorial that I have missed I'd be pleased to hear.

What is Dotenv
Use Dotenv package
#test.py
from os import environ
print(environ.get("ENV_VAR")) # Output : test
#.env
ENV_VAR=test
How to use it in youre case
Option 1
You can use this to store an Boolean in .env file and read it to define wich mode you are running. be careful as env variables get read as string. if you wanna use boolean you need to parse them from the string.
Option 2
An other option is to store in env file which config you wanna use and make an if else tree in python script:
if os.environ.get("CONFIG") == "1":
app.config.from_object("config.conf1")
elif os.environ.get("CONFIG") == "2":
app.config.from_object("config.conf2")
else:
app.config.from_object("config.default")
Why use Dotenv
The advantage of using environment variables is that you can ignore them in git and every time the server is seetup in linux or docker all settings can be managed from one file. also does the standart server admin know bash script and env files but not necesseraly python.

Flask API facing InterfaceError with PostgreSQL

I have a Flask API based on Flask RestPlus extension and is hosted on Google App Engine. The API does a basic job of fetching data from a Google Cloud SQL PostgreSQL. The API is working fine otherwise but sometimes it starts returning InterfaceError: cursor already closed.
Strangely, when I do a gcloud app deploy, the API starts working fine again.
Here's a basic format of the API:
import simplejson as json
import psycopg2
from flask import Flask, jsonify
from flask_restplus import Api, Resource, fields
from psycopg2.extras import RealDictCursor
app = Flask(__name__)
app.config['SWAGGER_UI_JSONEDITOR'] = True
api = Api(app=app,
doc='/docs',
version="1.0",
title="Title",
description="description")
app.config['SWAGGER_UI_JSONEDITOR'] = True
ns_pricing = api.namespace('cropPricing')
db_user = "xxxx"
db_pass = "xxxx"
db_name = "xxxxx"
cloud_sql_connection_name = "xxxxxx"
conn = psycopg2.connect(user=db_user,
password=db_pass,
host='xxxxx',
dbname=db_name)
#ns_pricing.route('/list')
class States(Resource):
def get(self):
"""
list all the states for which data is available
"""
cur = conn.cursor(cursor_factory=RealDictCursor)
query = """
SELECT
DISTINCT state
FROM
db.table
"""
conn.commit()
cur.execute(query)
states = json.loads(json.dumps(cur.fetchall()))
if len(states) == 0:
return jsonify(data=[],
status="Error",
message="Requested data not found")
else:
return jsonify(status="Success",
message="Successfully retreived states",
data=states)
What should I fix to not see the error anymore?

It would be good to use the ORMs such as SQLAlchemy / Flask-SQLAlchemy which would handle the establishing / re-establishing the connection part.
Though, if using psycopg2. you can use try except to catch the exception and re-establish the connection again.
try:
cur.execute(query)
except psycopg2.InterfaceError as err:
print err.message
conn = psycopg2.connect(....)
cur = conn.cursor()
cur.execute(query)

Stranger error in SqlAlchemy. Bug?

I tried to use Flask-Restful + SqlAlchemy (with Automap) + MySql SGDB but I don't understand why this error occured in my code:
I sent the request in my controller and it worked normally, but after 10s an error is generated about a loss of connection with a SGDB.
itens = session.query(estados).filter(estados.ativo == True)
But stranger is that if I use SQL string syntax, the problem does not occur.
itens = engine.execute("SELECT `TBEstados`.`id`, `TBEstados`.`nome`, `TBEstados`.`ativo` FROM `intbr_webapp`.`TBEstados`;")
I'm using SqlAlchemy 1.2 but I did try also the 1.1 version. I did try also use the pre-ping=true and I didn't obtain success.
Does someone know anything about this? I don't understand why using ORM structure doesn't work, but with SQL syntax it does work. The connection is same, but the result is not.
My code is below:
estado.py
from flask import jsonify
from flask_restful import Resource
from json import dumps
from resources.database import Base, session, engine
#from resources.dataEncoder import JsonModel
from models.TBEstados import TBEstadosSchema
class Estados(Resource):
def get(self):
estados = Base.classes.TBEstados
itens = session.query(estados).filter(estados.ativo == True)
result = TBEstadosSchema(many=True).dump(itens)
return jsonify(result.data)
database.py (imported in EstadoModel)
from flask import Flask, g
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import sessionmaker
from flask_marshmallow import Marshmallow
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = my conn string
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_POOL_RECYCLE'] = 3600
app.config['SQLALCHEMY_POOL_TIMEOUT'] = 30
Base = automap_base()
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True)
# reflect the tables
Base.prepare(engine, reflect=True)
Session = sessionmaker(bind=engine)
session = Session()
ma = Marshmallow()
run.py
from flask import Flask, g
from flask_restful import Resource, Api
import resources.database
from controllers.Estados import Estados
app = Flask(__name__)
api = Api(app)
api.add_resource(Estados, '/estados')
if __name__ == '__main__':
app.run(debug=True, port=9002)
The exact error:
sqlalchemy.exc.OperationalError
sqlalchemy.exc.OperationalError: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query') [SQL: 'SELECT TBEstados.id AS TBEstados_id, TBEstados.nome AS TBEstados_nome, TBEstados.ativo AS TBEstados_ativo \nFROM TBEstados \nWHERE TBEstados.ativo = true']

I solved my problem!
I changed value of the MySql variable wait_timeout of the "10s" to 100s (to test) and error in SqlAlchemy using PyMySql provider doesn't occurred again.
https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_wait_timeout
Thanks for help!

Flask RESTful API: Issue with db connection pool

I'm trying to create REST API endpoints using flask framework. This is my fully working script:
from flask import Flask, jsonify
from flask_restful import Resource, Api
from flask_restful import reqparse
from sqlalchemy import create_engine
from flask.ext.httpauth import HTTPBasicAuth
from flask.ext.cors import CORS
conn_string = "mssql+pyodbc://x:x#x:1433/x?driver=SQL Server"
auth = HTTPBasicAuth()
#auth.get_password
def get_password(username):
if username == 'x':
return 'x'
return None
app = Flask(__name__)
cors = CORS(app)
api = Api(app)
class Report(Resource):
decorators = [auth.login_required]
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('start', type = str)
parser.add_argument('end', type = str)
args = parser.parse_args()
e = create_engine(conn_string)
conn = e.connect()
stat = """
select x from report
"""
query = conn.execute(stat)
json_dict = []
for i in query.cursor.fetchall():
res = {'x': i[0], 'xx': i[1]}
json_dict.append(res)
conn.close()
e.dispose()
return jsonify(results=json_dict)
api.add_resource(Report, '/report')
if __name__ == '__main__':
app.run(host='0.0.0.0')
The issue is that I get results when I call this API only for a day or so after which I stop getting results unless I restart my script (or sometimes even my VM) after which I get results again. I reckon there is some issue with the database connection pool or something but I'm closing the connection and disposing it as well. I have no idea why the API gives me results only for some time being because of which I have to restart my VM every single day. Any ideas?

Per my experience, the issue was caused by coding create_engine(conn_string) to create db pool inside the Class Report so that always do the create & destory operations of db pool for per restful request. It's not correct way for using SQLAlchemy ORM, and be cause IO resouce clash related to DB connection, see the engine.dispose() function description below at http://docs.sqlalchemy.org/en/rel_1_0/core/connections.html#sqlalchemy.engine.Engine:
To resolve the issue, you just need to move e = create_engine(conn_string) to the below of the code conn_string = "mssql+pyodbc://x:x#x:1433/x?driver=SQL Server" and remove the code e.dispose() both in the Class Report, see below.
conn_string = "mssql+pyodbc://x:x#x:1433/x?driver=SQL Server"
e = create_engine(conn_string) # To here
In the def get(delf) function：
args = parser.parse_args()
# Move: e = create_engine(conn_string)
conn = e.connect()
and
conn.close()
# Remove: e.dispose()
return jsonify(results=json_dict)

RuntimeError: working outside of application context

app.py
from flask import Flask, render_template, request,jsonify,json,g
import mysql.connector
app = Flask(__name__)
**class TestMySQL():**
#app.before_request
def before_request():
try:
g.db = mysql.connector.connect(user='root', password='root', database='mysql')
except mysql.connector.errors.Error as err:
resp = jsonify({'status': 500, 'error': "Error:{}".format(err)})
resp.status_code = 500
return resp
#app.route('/')
def input_info(self):
try:
cursor = g.db.cursor()
cursor.execute ('CREATE TABLE IF NOT EXISTS testmysql (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, name VARCHAR(40) NOT NULL, \
email VARCHAR(40) NOT NULL UNIQUE)')
cursor.close()
test.py
from app import *
class Test(unittest.TestCase):
def test_connection1(self):
with patch('__main__.mysql.connector.connect') as mock_mysql_connector_connect:
object=TestMySQL()
object.before_request() """Runtime error on calling this"
I am importing app into test.py for unit testing.On calling 'before_request' function into test.py ,it is throwing RuntimeError: working outside of application context
same is happening on calling 'input_info()'

Flask has an Application Context, and it seems like you'll need to do something like:
def test_connection(self):
with app.app_context():
#test code
You can probably also shove the app.app_context() call into a test setup method as well.

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///todo.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
app.app_context().push()
Run in terminal
>python
>>>from app import app
>>>from app import db
>>>db.create_all()
Now it should work

I followed the answer from #brenns10 when I ran into a similar problem when using pytest.
I followed the suggestion of putting it into test setup, this works:
import pytest
from src.app import app
#pytest.fixture
def app_context():
with app.app_context():
yield
def some_test(app_context):
# <test code that needs the app context>

I am using python3.8 and had to use a small variation to the answers already posted. I included the the below in pytests and didn't have to change anything else in the rest of the test file.
from flask import Flask
#pytest.fixture(autouse=True)
def app_context():
app = Flask(__name__)
with app.app_context():
yield
This can also be used with a context manager as well.
The main different to note here is the creation of the Flask app within the test file rather than it being imported from the main application file.