I have an application with python, flask, and flask_mysqldb. When I execute the first query, everything works fine, but the second query always throws an error (2006, server has gone away). Everything I found on the web says this error is a timeout issue, which doesn't seem to be my case because:
1 - I run the second query just a few seconds after running the first
2 - My timeout configuration is set to 8 hours
I don't know what else this could be, here is the code that I am running:
import os
from flask import Flask
from flask import render_template
from flaskext.mysql import MySQL
import endpoints.usuario as usuario
app = Flask(__name__, static_folder='/root/sftp/atom-projects/flask-example/public/')
app.config['MYSQL_HOST'] = '123'
app.config['MYSQL_USER'] = '123'
app.config['MYSQL_PASSWORD'] = '123'
app.config['MYSQL_DB'] = '123'
app.add_url_rule('/usuarios', 'usuarios', usuario.list_all, methods=['GET'])
#app.errorhandler(404)
def not_found(e):
return app.send_static_file('index.html')
here is the code for the usuarios file:
from flask_mysqldb import MySQL
from flask import Flask, make_response
from flask import current_app
from flask import request
import bcrypt
def list_all():
mysql = MySQL(current_app)
cursor = mysql.connection.cursor()
cursor.execute("select * from usuario")
records = cursor.fetchall()
usuarios = []
for row in records:
usuarios.append({"id":row[0], "nome":row[1], "email":row[2], "senha":row[3], "tipo":row[4]})
for usuario in usuarios:
tipo = None
cursor.execute("select * from tipo_usuario where id = %s", [usuario['tipo']])
records = cursor.fetchall()
for row in records:
usuario['tipo'] = {"id":row[0], "permissao":row[1]}
return make_response({"msg":'', "error":False, "data":usuarios})
I have this running on nginx + gunicorn, here is the log :
gunicorn -w 1 --reload main:app
[2019-12-19 12:53:21 +0000] [5356] [INFO] Starting gunicorn 20.0.4
[2019-12-19 12:53:21 +0000] [5356] [INFO] Listening at: http://127.0.0.1:8000 (5356)
[2019-12-19 12:53:21 +0000] [5356] [INFO] Using worker: sync
[2019-12-19 12:53:21 +0000] [5359] [INFO] Booting worker with pid: 5359
[2019-12-19 12:53:28 +0000] [5359] [ERROR] Error handling request /usuarios
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/gunicorn/workers/sync.py", line 134, in handle
self.handle_request(listener, req, client, addr)
File "/usr/local/lib/python3.5/dist-packages/gunicorn/workers/sync.py", line 175, in handle_request
respiter = self.wsgi(environ, resp.start_response)
File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 2463, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 2457, in wsgi_app
ctx.auto_pop(error)
File "/usr/local/lib/python3.5/dist-packages/flask/ctx.py", line 452, in auto_pop
self.pop(exc)
File "/usr/local/lib/python3.5/dist-packages/flask/ctx.py", line 438, in pop
app_ctx.pop(exc)
File "/usr/local/lib/python3.5/dist-packages/flask/ctx.py", line 238, in pop
self.app.do_teardown_appcontext(exc)
File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 2320, in do_teardown_appcontext
func(exc)
File "/usr/local/lib/python3.5/dist-packages/flask_mysqldb/__init__.py", line 100, in teardown
ctx.mysql_db.close()
MySQLdb._exceptions.OperationalError: (2006, '')
If I run it with more workers, I can run a few more (depending on how many workers) queries, what could be causing this?
I had a similar error on my python flask application. And when I tried using from flask import current_app as app I still get an error. Eventually, I manged to remove the error by creating a separate file for flask app object app = Flask(__name__) and then importing it inside all the files that needed the object.
For Instance.
App object module: main.py
from flask import Flask, request
app = Flask(__name__)
Other files like usuarios.py or config.py
from main import app
from flask_mysqldb import MySQL
app.config['MYSQL_HOST'] = '123'
app.config['MYSQL_USER'] = '123'
app.config['MYSQL_PASSWORD'] = '123'
app.config['MYSQL_DB'] = '123'
mysql = MySQL(app)
Note: I used flask_mysqldb instead of flaskext.mysql but I don't think that makes any difference as long as your using the right parameters for either.
Finally found a solution.
Turns out Flask Mysql needs to be performed within the try...catch..finally block
And also make sure to close the cursor if you are using flask_mysqldb, which doesn't handle closing for you.
This is how my Query looks like now.
try:
cur.execute('SELECT * FROM users WHERE `email` = %s', (email))
results = cur.fetchall()
cur.close()
except Exception:
return 'Error: unable to fetch items'
cur.close()
Go to the below link and use mysqlpool instead of mysqldb or mysqlext.
https://pypi.org/project/flask-mysqlpool/
connection = mysql.connection.get_connection()
cur=connection.cursor()
query = "Enter your Query"
cur.execute(query)
connection.commit()
cur.close()
and one more thing that
app.config['MYSQL_POOL_SIZE'] = 30
this pool size must be less than or equal to 32
Related
How do I save a psycopg2 connection on a flask server once I execute a query? I am trying to save connection from a different python file on flask. I am using this code:-
from flask import current_app
cursor = connection.cursor() # Connection with Database
cursor.execute(self.query)
connection.commit()
current_app.config['pool'].putconn(connection)
It gives me this error.
'Flask' object is not subscriptable
I use this in the flask init file to create a connection Pool:-
app.config['pool'] = psycopg2.pool.SimpleConnectionPool(
1, 10,
host = config["HOST"],
database = config["DATABASE"],
user = config["USER"],
password = config["PASSWORD"]
)
This is the get_db function. It works well.
from flask import current_app
def get_db(self):
return current_app.config['pool'].getconn()
For testing, I amend the MYSQL (RDS) parameters as follows;
wait_timeout = 40 (default was 28800)
max_allowed_packet = 1GB (max - just to be sure issue not caused by small packets)
net_read_timeout = 10
interactive_timeout unchanged
Then tested my app without pool_pre_ping options set (defaults to False), kept the app inactive for 40 seconds, tried to login, and i get
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: Traceback (most recent call last):
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: File "/var/www/api_server/venv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: context)
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: File "/var/www/api_server/venv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 507, in do_execute
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: cursor.execute(statement, parameters)
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: File "/var/www/api_server/venv/lib/python3.6/site-packages/MySQLdb/cursors.py", line 206, in execute
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: res = self._query(query)
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: File "/var/www/api_server/venv/lib/python3.6/site-packages/MySQLdb/cursors.py", line 312, in _query
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: db.query(q)
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: File "/var/www/api_server/venv/lib/python3.6/site-packages/MySQLdb/connections.py", line 224, in query
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: _mysql.connection.query(self, query)
Nov 14 20:05:20 ip-172-31-33-52 gunicorn[16962]: MySQLdb._exceptions.OperationalError: (2013, 'Lost connection to MySQL server during query')
Added the pool_pre_ping like this (Using flask_sqlalchamy version 2.4.1);
import os
from flask import Flask
from flask_sqlalchemy import SQLAlchemy as _BaseSQLAlchemy
class SQLAlchemy(_BaseSQLAlchemy):
def apply_pool_defaults(self, app, options):
super(SQLAlchemy, self).apply_pool_defaults(app, options)
options["pool_pre_ping"] = True
# options["pool_recycle"] = 30
# options["pool_timeout"] = 35
db = SQLAlchemy()
class DevConfig():
SQLALCHEMY_ENGINE_OPTIONS = {'pool_recycle': 280, 'pool_timeout': 100, 'pool_pre_ping': True} # These configs doesn't get applied in engine configs :/
DEBUG = True
# SERVER_NAME = '127.0.0.1:5000'
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI_DEV')
SQLALCHEMY_TRACK_MODIFICATIONS = False
config = dict(
dev=DevConfig,
)
app = Flask(__name__, instance_relative_config=True)
app.config.from_object(config['dev'])
# INIT DATABASE
db.init_app(app)
with app.app_context():
db.create_all()
-----------run.py
app.run(host='127.0.0.1', port=5000)
With this, now the webapp manages to get new connection even after MySQL server has closed the previous connection. It always works fine when I access the database right after its closed by server (tried max 50 seconds after)... but when I keep connection inactive for long time (haven't noted, but ~ >10-15 min), again I see same error.
According to the docs, (especially the section Dealing with disconnects), the pool_pre_ping option should handle this kind of scenario at background rite? Or is there any other timeout variable that I need to change in MySQL server?
I set the following settings:
SQLALCHEMY_ENGINE_OPTIONS = {
'pool_size': 10,
'pool_recycle': 60,
'pool_pre_ping': True
}
The last few months have stopped falling...
From the Flask-SQLAlchemy Configuration docs:
Certain database backends may impose different inactive connection
timeouts, which interferes with Flask-SQLAlchemy’s connection pooling.
By default, MariaDB is configured to have a 600 second timeout. This
often surfaces hard to debug, production environment only exceptions
like
2013: Lost connection to MySQL server during query.
If you are using a backend (or a pre-configured database-as-a-service)
with a lower connection timeout, it is recommended that you set
SQLALCHEMY_POOL_RECYCLE to a value less than your backend’s timeout.
The script cited in the question shows discrepancies between its MySQL timeout-configs (wait_timeout, net_read_timeout) and its SQLAlchemy (pool_recycle, pool_timeout) and Flask-SQLAlchemy timeouts (SQLALCHEMY_POOL_RECYCLE, SQLALCHEMY_POOL_TIMEOUT).
We can resolve this by using the DevConfig helper-class to coordinate the db connection config constants across the app. To do that, we assign our config to static attributes and refer back to them so that there are no conflicting timeout expectations. Here is an implementation:
import os
from flask import Flask
from flask_sqlalchemy import SQLAlchemy as _BaseSQLAlchemy
# Coordinate DevConfig with SQLAlchemy and Flask-SQLAlchemy (don't repeat yourself!)
class DevConfig():
SQLALCHEMY_POOL_RECYCLE = 35 # value less than backend’s timeout
SQLALCHEMY_POOL_TIMEOUT = 7 # value less than backend’s timeout
SQLALCHEMY_PRE_PING = True
SQLALCHEMY_ENGINE_OPTIONS = {'pool_recycle': SQLALCHEMY_POOL_RECYCLE, 'pool_timeout': SQLALCHEMY_POOL_TIMEOUT, 'pool_pre_ping': SQLALCHEMY_PRE_PING}
DEBUG = True
# SERVER_NAME = '127.0.0.1:5000'
SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI_DEV')
SQLALCHEMY_TRACK_MODIFICATIONS = False
class SQLAlchemy(_BaseSQLAlchemy):
def apply_pool_defaults(self, app, options):
super(SQLAlchemy, self).apply_pool_defaults(app, options)
options["pool_pre_ping"] = DevConfig.SQLALCHEMY_PRE_PING
# options["pool_recycle"] = 30
# options["pool_timeout"] = 35
db = SQLAlchemy()
config = dict(
dev=DevConfig,
)
app = Flask(__name__, instance_relative_config=True)
app.config.from_object(config['dev'])
# INIT DATABASE
db.init_app(app)
with app.app_context():
db.create_all()
If you like, you can check the diff for the changes I made: diffchecker.com/Q1e85Hhc
I am working on a project to work on a website that connects to a mysql server using flask and sqlAlchemy (Hosted on AWS RDS) and I am following this tutorial but when I try to do (/api/bar) I get this error. When I just do my localhost:8080 it shows "Hello World" perfectly fine.
sqlalchemy.exc.OperationalError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on 'localhost' ([WinError 10061] No connection could be made because the target machine actively refused it)") (Background on this error at: http://sqlalche.me/e/e3q8)
config.py
database_url = "mysql+pymysql://username:password#localhost:3306/barbeerdrinker"
Here is my __init__.py
from flask import Flask
from flask import jsonify
from flask import make_response
from flask import request
import json
from barbeerdrinker import database
#Flask application
app = Flask(__name__)
#app.route("/")
def hello_world():
return "Hello World"
#app.route("/api/bar", methods=["GET"])
def get_bars():
return jsonify(database.get_bars())
#app.route("/api/bar/<name>", methods=["GETS"])
def find_bar(name):
try:
if name is None:
raise ValueError("Bar is not specified")
bar = database.find_bar(name)
if bar is None:
return make_response("No bar found within given name", 404)
return jsonify(bar)
except ValueError as e:
return make_response(str(e), 400)
except Exception as e:
return make_response(str(e), 500)
#app.route("/api/beers_cheaper_than", methods=["POST"])
def find_beers_cheaper_than():
body = json.loads(request.data)
max_price = body['maxPrice']
return jsonify(database.filter_beers(max_price))
database.py
from sqlalchemy import create_engine
from sqlalchemy import sql
from barbeerdrinker import config
engine = create_engine(config.database_url)
def get_bars():
with engine.connect() as con:
rs = con.execute("SELECT name, address, city, opening, closing, phoneNum FROM bars")
return [dict(row) for row in rs]
def find_bar(name):
with engine.connect() as con:
query = sql.text("SELECT * FROM bars WHERE name = :name;")
rs = con.execute(query, name=name)
result = rs.first()
if result is None:
return None
return dict(result)
def filter_beers(max_price):
with engine.connect() as con:
query = sql.text("SELECT * FROM sells WHERE price < : max_price;")
rs = con.execute(query, max_price=max_price)
results = [dict(row) for row in rs]
for r in results:
r['price'] = float(r['price'])
return results
**Edit: So it seems like the problem is not an issue with my code but a Windows error. One solution I tried to do was to open up the required ports through my firewall to no avail.
I just figured it out, turns out the issue is not a windows issue. The problem is within my config.py:
Instead of:
database_url = "mysql+pymysql://username:password#localhost:3306/barbeerdrinker"
It should be:
database_url = "mysql+pymysql://username:password#**AWSENDPOINT/HOSTNAME**:3306/barbeerdrinker"
I am new to using python to connect to a mysql DB and I am getting the following error:
OperationalError: (pymysql.err.OperationalError) (1045, u"Access denied for user 'xxxxxxadmin'#'xx.xx.xx.xx' (using password: YES)") (Background on this error at: http://sqlalche.me/e/e3q8)
xx.xxx.216.44 - - [02/Apr/2018 17:27:49] "GET /testconnect HTTP/1.1" 500 -
This is most of the connect script in my python file:
#!/usr/bin/python3
from flask import Flask, request
from flask_restful import Resource, Api
from sqlalchemy import create_engine
from json import dumps
from flask.ext.jsonpify import jsonify
db_connect = create_engine("mysql+pymysql://xxxxxxxadmin:password#,mymaindb.xxxxxxxx.us-east-2.rds.amazonaws.com:3306/myDBname")
app = Flask(__name__)
api = Api(app)
class TestConnect(Resource):
def get(self):
conn = db_connect.connect() # connect to database
query = conn.execute("select * from Players") # This line performs query and returns json result
return {'employees': [i[0] for i in query.cursor.fetchall()]} # Fetches first column that is Employee ID
api.add_resource(TestConnect, '/testconnect') # Route_1
if __name__ == '__main__':
app.run(host='0.0.0.0', debug = False)
Other background:
But when I try to connect to the same mysql database using the exact same credentials via the command line on the server running the python script I am able to get in.
Not sure how to test more to get a better error result that will help me figure this issue out.
UPDATE
So I was able to connect to my DB via mysql workbench with the connection strings and information I have in the python script. Does this mean my python script is doing something wrong?
Why not use:
mysql+pymysql://xxxxxxxadmin:password#mymaindb.xxxxxxxx.us-east-2.rds.amazonaws.com:3306/myDBname
instead of
mysql+pymysql://xxxxxxxadmin:password#**,**mymaindb.xxxxxxxx.us-east-2.rds.amazonaws.com:3306/myDBname
Not sure why you're connection string has a comma. Might just be a typo?
On that note, I usually build the connection URL before passing it to create_engine just to make it easier to manage in the future incase I have to pull the actual values from the environmental variables:
HOST = "mymaindb.xxxxxxxx.us-east-2.rds.amazonaws.com"
PORT = 3306
USERNAME = "xxxxxxxadmin"
PASSWORD = "password"
DBNAME = "myDBname"
CONNECTION_URL = 'mysql+pymysql://%s:%s#%s:%s/%s' % (
USERNAME,
PASSWORD,
HOST,
PORT,
DBNAME
)
I'm having issues with reading data out of mysql with a cursor and displaying it over HTTP via flask. Everytime I run this code:
import os
from flask import Flask, request, session, url_for, redirect, \
render_template, abort, g, flash, _app_ctx_stack
from flaskext.mysql import MySQL
from contextlib import closing
from hashlib import md5
from datetime import datetime
import time
import os
from werkzeug import check_password_hash, generate_password_hash
import config
app = Flask(__name__)
mysql = MySQL()
app.config['MYSQL_DATABASE_HOST'] = config.MYSQL_DATABASE_HOST
app.config['MYSQL_DATABASE_PORT'] = config.MYSQL_DATABASE_PORT
app.config['MYSQL_DATABASE_USER'] = config.MYSQL_DATABASE_USER
app.config['MYSQL_DATABASE_PASSWORD'] = config.MYSQL_DATABASE_PASSWORD
app.config['MYSQL_DATABASE_DB'] = config.MYSQL_DATABASE_DB
mysql.init_app(app)
if 'SECRET_KEY' in os.environ:
app.config['SECRET_KEY'] = os.environ['SECRET_KEY']
else:
app.config['SECRET_KEY'] = os.urandom(24)
###
# Routing for your application.
###
def init_db():
with closing(connect_db()) as db:
with app.open_resource('init.sql') as f:
db.cursor().execute(f.read())
db.commit()
def connect_db():
return mysql.connect()
if __name__ == '__main__':
init_db()
app.run(debug=True)
I get this result:
(venv)λ fh → λ git master* → python app.py
Exception _mysql_exceptions.ProgrammingError: (2014, "Commands out of sync; you can't ru
n this command now") in <bound method Cursor.__del__ of <MySQLdb.cursors.Cursor object a
t 0x10bc3ddd0>> ignored
Traceback (most recent call last):
File "app.py", line 124, in <module>
init_db()
File "app.py", line 38, in init_db
db.commit()
_mysql_exceptions.ProgrammingError: (2014, "Commands out of sync; you can't run this com
mand now")
Any ideas? I'm new to flask.
Seems you need to start the app before trying to do your DB stuff, as you are accessing the DB through Flask. I'm not experienced with Flask, so I'm not sure how you can access the DB after starting the server, but I can look if you don't know. Basically,
init_db()
app.run(debug=True)
needs to be reversed to
app.run(debug=True)
init_db()
In some way.