I'm currently running an flask python application running on an NGINX server with the help of uWGI. The static pages are always reachable but the pages that use a connection (MySQL) time out after 2 minutes. What's going on? They become simply become unavailable.
Things I have tried:
Not using a global
working with pools
turning off firewall
.
# using python version 2.7.10
from flask import Flask, jsonify, request, session
import mysql.connector.pooling
#Make a connection with the DB
dbconfig = {
"host" : "12.34.5.78",
"database": "db",
"user": "user",
"password": "pass"
}
conn = mysql.connector.connect(pool_name = "mypool",
pool_size = 6,
**dbconfig)
#Define the root
app = Flask(__name__)
#Landings page
#app.route('/')
def index():
return "Hello World."
# return all resources by name
#app.route('/resources', methods=['GET'])
def allResourceNames():
conn1 = mysql.connector.connect(pool_name="mypool")
reader = conn1.cursor()
query = ("SELECT name FROM resources")
reader.execute(query)
resources = []
for name in reader:
resources.append({'name' : name[0]})
reader.close()
conn1.close()
return jsonify({"resources" : resources})
if __name__ == "__main__":
app.run(debug=True)
According to MySQL Python Connector Doc
you may want to set connection_timeout as connect option. E.g.,
conn = mysql.connector.connect(pool_name = "mypool",
pool_size = 6, connection_timeout=3600,
**dbconfig)
Related
I'm working on app, where user have to enter database connection parametrs by hands
So I've made an api here it is:
from flask_mysqldb import MySQL
from flask import Flask, request
import MySQLdb.cursors
app = Flask(__name__)
# /hello
#app.route("/hello", methods=['GET'])
def hello():
return "Hello word"
# /get_tables
#app.route(f"/get_tables", methods=['GET'])
def get_tables():
host = request.args.get('host', type=str)
user = request.args.get('user', type=str)
password = request.args.get('password', type=str)
database = request.args.get('database', type=str)
app.config['MYSQL_HOST'] = "localhost"
app.config['MYSQL_USER'] = "root"
app.config['MYSQL_PASSWORD'] = "password"
app.config['MYSQL_DB'] = "restaurant"
mysql = MySQL(app)
cursor = mysql.connection.cursor(MySQLdb.cursors.DictCursor)
query = "SHOW TABLES"
cursor.execute(query)
records = cursor.fetchone()
print(records)
return records
"hello" endpoint is working fine, but when run "get_tables" I recieve an error:
AssertionError: The setup method 'teardown_appcontext' can no longer be called on the application. It has already handled its first request, any changes will not be applied consistently. Make sure all imports, decorators, functions, etc. needed to set up the application are done before running it.
I also was trying to debug my api and found out that errorr ocures on a mysql = MySQL(app) line
What can I do to solve this problem? or maybe it's better not to use api at all and just connect to DB from ui?
I am a beginner in programming and I am creating an application to manage machines with OpenStack. However, when trying to create a postman machine, I get an error.
Could you help me?
My requirements.txt:
Click==7.0
Flask==1.0.3
itsdangerous==1.1.0
Jinja2==2.10.1
MarkupSafe==1.1.1
Werkzeug==0.15.4
openstacksdk
python code from Github Tutorial:
from flask import Flask, request
from openstack import connection
from pprint import pprint
import json
import os
app = Flask(__name__)
# Source the openstack project rc file before running this app, to create the
# environment variables needed
conn = connection.Connection(auth_url=os.environ['OS_AUTH_URL'],
project_name=os.environ['OS_PROJECT_NAME'],
username=os.environ['OS_USERNAME'],
password=os.environ['OS_PASSWORD'],
user_domain_id="default",
project_domain_id=os.environ['OS_PROJECT_DOMAIN_ID'])
#app.route("/")
def hello():
return "Hello from Flask!"
#app.route("/list")
def list_openstack_resources():
# Check if connection is established
print("conn: ", conn)
# Print list of servers, images, flavors, endpoints, projects, users
server_list = list(conn.compute.servers())
image_list = list(conn.compute.images())
flavor_list = list(conn.compute.flavors())
project_list = list(conn.identity.projects())
user_list = list(conn.identity.users())
pprint(server_list)
pprint(image_list)
pprint(flavor_list)
pprint(project_list)
pprint(user_list)
return "List printed to stdout"
#app.route("/create_server")
def create_server():
# Check if connection is established
print("conn: ", conn)
# Create the volume first
volume_size = request.args.get('volume_size')
print("Starting to create volume with size (in GiB): ", volume_size)
volume = conn.create_volume(size=volume_size,
image="cirros-0.4.0-x86_64-disk",
wait=True,
bootable=False, #Or True
)
print("Created volume: ", json.dumps(volume))
# Create the server using the server_name parameter in the GET request
server_name = request.args.get('server_name')
print("Starting to create the server with name: ", server_name)
server = conn.create_server(name=server_name,
flavor="m1.micro",
terminate_volume=True,
timeout=180,
boot_volume=volume.id,
key_name="mayank-public-key",
)
print("Created server: ", json.dumps(server))
return "Server create request sent!"
if __name__ == "__main__":
app.run(debug=True, host='0.0.0.0', port=8080)
When I try this:
GET http://localhost:8080/create_server?server_name=server1&volume_size=2
I receive this error:
I am working on a project to work on a website that connects to a mysql server using flask and sqlAlchemy (Hosted on AWS RDS) and I am following this tutorial but when I try to do (/api/bar) I get this error. When I just do my localhost:8080 it shows "Hello World" perfectly fine.
sqlalchemy.exc.OperationalError: (pymysql.err.OperationalError) (2003, "Can't connect to MySQL server on 'localhost' ([WinError 10061] No connection could be made because the target machine actively refused it)") (Background on this error at: http://sqlalche.me/e/e3q8)
config.py
database_url = "mysql+pymysql://username:password#localhost:3306/barbeerdrinker"
Here is my __init__.py
from flask import Flask
from flask import jsonify
from flask import make_response
from flask import request
import json
from barbeerdrinker import database
#Flask application
app = Flask(__name__)
#app.route("/")
def hello_world():
return "Hello World"
#app.route("/api/bar", methods=["GET"])
def get_bars():
return jsonify(database.get_bars())
#app.route("/api/bar/<name>", methods=["GETS"])
def find_bar(name):
try:
if name is None:
raise ValueError("Bar is not specified")
bar = database.find_bar(name)
if bar is None:
return make_response("No bar found within given name", 404)
return jsonify(bar)
except ValueError as e:
return make_response(str(e), 400)
except Exception as e:
return make_response(str(e), 500)
#app.route("/api/beers_cheaper_than", methods=["POST"])
def find_beers_cheaper_than():
body = json.loads(request.data)
max_price = body['maxPrice']
return jsonify(database.filter_beers(max_price))
database.py
from sqlalchemy import create_engine
from sqlalchemy import sql
from barbeerdrinker import config
engine = create_engine(config.database_url)
def get_bars():
with engine.connect() as con:
rs = con.execute("SELECT name, address, city, opening, closing, phoneNum FROM bars")
return [dict(row) for row in rs]
def find_bar(name):
with engine.connect() as con:
query = sql.text("SELECT * FROM bars WHERE name = :name;")
rs = con.execute(query, name=name)
result = rs.first()
if result is None:
return None
return dict(result)
def filter_beers(max_price):
with engine.connect() as con:
query = sql.text("SELECT * FROM sells WHERE price < : max_price;")
rs = con.execute(query, max_price=max_price)
results = [dict(row) for row in rs]
for r in results:
r['price'] = float(r['price'])
return results
**Edit: So it seems like the problem is not an issue with my code but a Windows error. One solution I tried to do was to open up the required ports through my firewall to no avail.
I just figured it out, turns out the issue is not a windows issue. The problem is within my config.py:
Instead of:
database_url = "mysql+pymysql://username:password#localhost:3306/barbeerdrinker"
It should be:
database_url = "mysql+pymysql://username:password#**AWSENDPOINT/HOSTNAME**:3306/barbeerdrinker"
I am trying to re-factor and re-structure my routes into views/routes.py, but all these routes are using one common open cassandra connection. open means that the cassandra connection will be connected when starting server and will stay open during the server running. The connected connection will be used by all routes. I am thinking about blueprints. But how to re-factor and re-structure my codes?
app = Flask(__name__)
dbConnection = None
#app.route('/api/v1/<para1>', methods=['GET', 'OPTIONS'])
#crossdomain(origin="*")
def funnel(para1):
# codes using cassandra Connection
#app.route('/api/v1/xxx/<para2>', methods=['GET', 'OPTIONS'])
#crossdomain(origin="*")
def funnel(para2):
# codes using cassandra Connection
#app.route('/api/v1/yyy/<para3>', methods=['GET', 'OPTIONS'])
#crossdomain(origin="*")
def funnel(para3):
# codes using cassandra Connection
if __name__ == '__main__':
# Connect db connection
app.run(host='0.0.0.0', port=8080, debug=True)
The following codes seem work. But any comments welcomed.
myblueprint.py
from flask import Blueprint
class MyBlueprint(Blueprint):
dbconnection = None
def setDb( self, db ):
self.dbconnection = db
myview.py
my_api = MyBlueprint('my_api', __name__ )
#my_api.route('/api/v1/yyy/<para3>', methods=['GET', 'OPTIONS'])
#crossdomain(origin="*")
def myapi(para3):
# codes using funnel_api.dbconnection
app.py
app = Flask(__name__)
cassandraConnection = None
if __name__ == '__main__':
my_api.setDb( cassandraConnection )
app.register_blueprint( my_api )
#SETUP RESOURCE AND START SERVER
app.run(host='0.0.0.0', port=5000, debug=True)
I'm using Python script to check if user requested exists.
using:
import MySQLdb
from flask import Flask, request, abort
app = Flask(__name__)
try:
db = MySQLdb.connect('xxx1','my_username','my_password','my_db_name')
db1 = MySQLdb.connect('xxx2','my_username','my_password','my_db_name')
db2=
db3=
except MySQLdb.OperationalError as e:
print "Caught an exception : " + str(.message)
#app.route('/')
#app.route('/<path:path>')
def page(path = ''):
user = request.args.get('user', None)
if not mac:
abort (403)
cursor = db.cursor()
query = 'Select ID from f_member where Name=%s'
db.commit()
cursor execute(query, (user, ))
row = cursor.fetchone()
cursor.close()
#cursor.db1 here
if row == None and row1 == None:
abort (403)
return 'OK', 200
if __name__ == '__main__':
app.run(host=host, port=port)
Then i have 5 nginx servers with this:
location = /auth {
proxy_pass http://xxx.xxx$request_uri;
proxy_pass_request_body off;
proxy_set_header Content_Lenght "";
proxy_set_header X-Real-IP $remote_addr;
So the thing is, this script checks if user is found in one
of the databases, if true then access the page.
Problem is my user list is now getting up to 5k users. and when i run the .py script it runs so fast (even with errors 403 people who are trying to connect), then broken pipe starts to show up.
Seems like it is getting overloaded, is there a better way to handle my script so it runs better and more efficient?
You may use a dictionary for username/id map in your Python program. Basically, when the program starts it will make a query for all the users and populate the map. Afterwards, every 20 seconds or so it will make a query to get the "changes" in f_member to update the dictionary. Lookup for username happens always in this map. If a user name is not found in the map, then only it makes a DB query (and if the user detail is found on DB, update the local map as well).
If you don't have millions of users in the table, this approach will work. Otherwise use a LRU cache.
So, after a lengthy comment thread, it appears like your flask instances might be competing for DB resources. There's also another hypothesis that saving your connections off in global scope could have some bad side effects (I could be wrong about this, but I'd be concerned about timeouts, not closing the connections, etc). Here's how I might rewrite it:
import MySQLdb
from flask import Flask, request, abort
app = Flask(__name__)
def get_db_connection_args():
try:
db_args = { 'host':'xxx1', 'user':'my_username', 'passwd':'my_password', 'db':'my_db_name' }
db1_args = { 'host':'xxx2', 'user':'my_username', 'passwd':'my_password', 'db':'my_db_name' }
db2_args = { 'host':'xxx3', 'user':'my_username', 'passwd':'my_password', 'db':'my_db_name' }
db3_args = { 'host':'xxx4', 'user':'my_username', 'passwd':'my_password', 'db':'my_db_name' }
except MySQLdb.OperationalError as e:
print "Caught an exception : " + str(.message)
return (db_args, db1_args, db2_args, db3_args)
#app.route('/')
#app.route('/<path:path>')
def page(path = ''):
user = request.args.get('user', None)
#I don't know what mac is...but it was in your original code.
if not mac:
abort (403)
found = False
db_connection_args = get_db_connection_args()
for db_connection_arg_dict in db_connection_args:
if not found:
db_conn = MySQLdb.connect(**db_connection_arg_dict)
try:
cursor = db_conn.cursor()
cursor.execute('Select ID from f_member where Name=%s', (user, ))
row = cursor.fetchone()
if row:
found = True
finally:
db_conn.close()
if found:
return 'OK', 200
abort (403)
if __name__ == '__main__':
app.run(host=host, port=port)