Develop Reference

Flask-SQLAlchemy Queue Pool limit reached

I have an application where a service makes a POST request to a flask app which then inserts the body of request into a postgresql database into a table as well as inserting data into several other tables.
The problem occurs once a curtain amount of requests are made that the queue pool limit is reach and the app throws an error. i have isolated it down to the extra functions called to do the inserting into other tables. i don't understand what is happening.
Each service is a docker container
172.30.0.17 - - [16/Jun/2022 06:25:07] "GET /api/orgs HTTP/1.1" 500 -
Traceback (most recent call last):
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/service/routes/get/get_org.py", line 17, in base_route
query = [convert_class(item) for item in Organization.query.all()]
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1711, in execute
conn = self._connection_for_bind(bind)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1552, in _connection_for_bind
return self._transaction._connection_for_bind(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
conn = bind.connect()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3234, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3313, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3280, in _wrap_pool_connect
return fn()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 476, in checkout
rec = pool._do_get()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 134, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)
Config
class Config:
# Database
SQLALCHEMY_DATABASE_URI = environ.get("DB_URI")
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_size": 1,
"max_overflow": 0,
}
App init
db = SQLAlchemy()
# create flask App
def create_app():
app = Flask(__name__, instance_relative_config=False)
# DB config - postgreSQL
app.config['SQLALCHEMY_DATABASE_URI'] = getenv("DB_URI")
app.config.from_object('config.Config')
# config server with CORS
# cors = CORS(app, resources={r"/*": {"origins": "*"}})
with app.app_context():
db.init_app(app)
from service.routes.post import post_event
return app
app = create_app()
POST request
#app.route('/api/orgs/<org_id>/sites/<site_id>/sessions/<session_id>/events', methods=["POST"])
def edger_post_request(org_id, site_id, session_id):
print("URL params: {}, {}, {}".format(
org_id, site_id, session_id
))
# parse body of post request
try:
request_body = request.json
event_data = json.loads(request_body["data"])
except:
return "error in body of request", 400
print("Body of request: ", request_body)
# check if body of request is valid
errors_body, has_error = validate_input(request_body)
print(errors_body, has_error)
if has_error:
return Response(errors_body, 400)
# insert event
event_dict = dict()
try:
event_dict = {
"id": request_body["id"],
"timestamp": request_body["timestamp"],
"type": request_body["type"],
"event_type": event_data["type"],
"data": request_body["data"],
"org_id": org_id,
"site_id": site_id,
"device_id": request_body["device_id"],
"session_id": request_body["session_id"]
}
if "frame_url" in request_body.keys():
event_dict["frame_url"] = request_body["frame_url"]
else:
event_dict["frame_url"] = None
insert_event = Event(**event_dict)
db.session.add(insert_event)
# commit event to DB
db.session.commit()
# Insert extra tables
insert_threads(org_id, site_id, event_dict, event_data)
db.session.commit()
db.session.close()
return {"result": "success"}, 200
except Exception as e:
print("- - - - - - - - - - - - - - - - -")
db.session.rollback()
print("Error inserting Event", e)
print("- - - - - - - - - - - - - - - - -")
db.session.close()
return {"result": "unsuccessful"}, 409
insert_threads function
def insert_threads(org_id, site_id, body, data):
event_data_insert(body, data),
event_type_insert(data["type"]),
insert_org(org_id),
insert_site(org_id, site_id),
insert_device(org_id, site_id, body['device_id']),
insert_session(org_id, site_id, body['device_id'], body['session_id'], body['timestamp'])
def insert_org(org_id):
org_list = getattr(g, "org_list", None)
# query organization insert if doesn't exist
if org_list is None:
check = [org.id for org in Organization.query.all()]
g.org_list = check
org_list = check
if org_id not in org_list:
insert_org = Organization(id=org_id, name="temp_name")
db.session.add(insert_org)
org_list.append(insert_org.id)
g.org_list = org_list
def insert_site(org_id, site_id):
site_list = getattr(g, "site_list", None)
# query Site insert if doesn't exist
if site_list is None:
check = [site.id for site in Site.query.all()]
g.site_list = check
site_list = check
if site_id not in site_list:
insert_site = Site(id=site_id, org_id=org_id, name="temp_name")
db.session.add(insert_site)
site_list.append(insert_site.id)
g.site_list = site_list
def insert_device(org_id, site_id, device_id):
device_list = getattr(g, "device_list", None)
# query device insert if doesn't exist
if device_list is None:
check = [device.id for device in Device.query.all()]
g.device_list = check
device_list = check
if device_id not in device_list:
insert_device = Device(id=device_id, site_id=site_id, org_id=org_id)
db.session.add(insert_device)
device_list.append(insert_device.id)
g.device_list = device_list
def insert_session(org_id, site_id, device_id, session_id, timestamp):
session_list = getattr(g, "session_list", None)
# query session and insert if it doesn't already exist
if session_list is None:
check = [session.id for session in Session.query.all()]
g.session_list = check
session_list = check
if session_id not in session_list:
insert_session = Session(
id=session_id,
timestamp=timestamp,
org_id=org_id,
site_id=site_id,
device_id=device_id
)
db.session.add(insert_session)
session_list.append(insert_session.id)
g.session_list = session_list
def event_data_insert(event, event_data):
# get white list from global var else None
query_list = getattr(g, "query_list", None)
# if query_list is None get query_list from DB
if query_list is None:
check = [convert_class(item) for item in Event_data_query_list.query.all()]
g.query_list = check
query_list = check
# loop though query_list and insert key/value pair into event_data table
for key in query_list:
print("the key: ", key)
key_path = key["key_name"].split(".")
# check if keys exist in data Object and continue down levels till the value is reached
if key_path[0] in event_data["data"] and key["type"] == event_data["type"]:
print("inside first if")
value = event_data['data']
for sub_key in key_path:
print("inside loop")
if sub_key in value.keys():
print("in subkey" , sub_key)
value = value[sub_key]
# handle if keys and values dont exist
else:
value = None
break
if value is None:
continue
# determine where value is an Integer or a String and insert into correct column
val_int = None
val_str = None
if type(value) == str:
val_str = value
elif type(value) == int:
val_int = value
# add event_data row to session
insert = Event_data(event_id=event["id"], type=event_data["type"], key=key["key_name"], value_int=val_int, value_str=val_str )
print("insert object", insert)
db.session.add(insert)
def event_type_insert(event_type):
event_type_list = getattr(g, "event_type_list", None)
if event_type_list is None:
check = [item.event_type for item in Event_type.query.all()]
g.event_type_list = check
event_type_list = check
if event_type not in event_type_list:
insert = Event_type(event_type=event_type)
db.session.add(insert)
g.event_type_list.append(event_type)

__init__.py : TypeError: %d format: a number is required, not str

I'm very new to Python. I'm trying to make a simple web server that pulls data from my mongodb DB.
This is my files architecture:
This is my code:
# Database
CONNECTION_URL = os.environ['mongoConnectionURL']
DATABASE_NAME = os.environ['mongoDatabaseName']
NEWS_COLLECTION = os.environ['mongodbNewsCollection']
app = FastAPI()
# TODO JS has next function i'm currently unaware of
app.add_middleware(
CORSMiddleware,
allow_origins=['*'],
allow_headers=["Origin, X-Requested-With, Content-Type, Accept"],
)
#app.get('/')
def index():
return 'See /entries'
#app.get('/entries')
def getEntries():
client = MongoClient(CONNECTION_URL)
databaseMongo = client.get_database(DATABASE_NAME)
collectionMongo = databaseMongo.get_collection(NEWS_COLLECTION)
result = list(collectionMongo.find())
for entry in result:
return {
'data': result
}
if __name__ == "__main__":
uvicorn.run(app, port=os.environ['PORT'])
This is my error:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/logging/__init__.py", line 1083, in emit
msg = self.format(record)
File "/usr/local/lib/python3.9/logging/__init__.py", line 927, in format
return fmt.format(record)
File "/usr/local/lib/python3.9/logging/__init__.py", line 663, in format
record.message = record.getMessage()
File "/usr/local/lib/python3.9/logging/__init__.py", line 367, in getMessage
msg = msg % self.args
TypeError: %d format: a number is required, not str
The longer error log is:

uvicorn.run(app, port=os.environ['PORT'])
That line is causing the problem. port needs to be an integer, but you're passing it as a string.
Convert it to an integer by wrapping the value in int(), like this:
uvicorn.run(app, port=int(os.environ['PORT']))

I think that's because port is an str instead of an int - try changing the last line into:
uvicorn.run(app, port=int(os.environ['PORT']))

Why database datas are being deleted after restarting flask server?

Recently, I have started flask. I found flask-sqlalchemy to work with flask. So I was using this. But I faced some problems. I am working on repl.it. When the repl goes assleep, I tried to use with the flask enabled website. But saw that no datas were showing. But before that I added a lot of datas. I can't figure out what is wrong but could you help me?
It may help you:
from flask_sqlalchemy import SQLAlchemy
import re
from flask import Flask, abort, jsonify, redirect, request
import os
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///db.sqlite3'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
class apikey(db.Model):
apikey = db.Column(db.String(25), primary_key=True, unique = True)
def __init__(self, apikey):
self.apikey = apikey
class shorturl(db.Model):
short_query = db.Column(db.String(15), primary_key=True, unique = True)
original = db.Column(db.String(1000))
visits = db.Column(db.Integer)
def __init__(self, short_query,original, visits):
self.short_query = short_query
self.original = original
self.visits = visits
def url_valid(url):
return re.match(regex, url) is not None
def bad_request(message):
response = jsonify({'message': message})
response.status_code = 400
return response
def errreq(message):
response = jsonify({'message': message})
response.status_code = 404
return response
#app.route('/')
def show_all():
return redirect("https://www.cburst.ml", code=301)
#app.route('/addapi', methods=['GET'])
def addapi():
if request.args.get('apikey') is not None:
api_key = request.args.get('apikey')
apiadd = apikey(apikey=api_key)
db.session.add(apiadd)
db.session.commit()
return jsonify({'message':"Done"}), 200
else:
return bad_request("Nothing Provided")
#app.route('/add', methods=['POST'])
def add():
if not request.json:
return bad_request('Url must be provided in json format.')
if "original" not in request.json :
return bad_request('Url parameter not found.')
if "short_query" not in request.json:
return bad_request('Url parameter not found.')
original = request.json['original']
short = request.json['short_query']
if shorturl.query.filter_by(short_query = short).first() is not None:
return bad_request("Already Exists")
visits = 0
if original[:4] != 'http':
original = 'http://' + original
if not url_valid(original):
return bad_request('Provided url is not valid.')
url_db = shorturl(
short_query=short, original=original, visits=visits)
shortened_url = short
db.session.add(url_db)
db.session.commit()
return jsonify({'link': shortened_url}), 201
#app.route('/add', methods=['GET'])
def add_get():
if request.args.get('apikey') is not None and request.args.get('original') is not None and request.args.get('short_query') is not None:
api_key = request.args.get('apikey')
original = request.args.get('original')
short_query = request.args.get('short_query')
if apikey.query.filter_by(apikey = api_key).first() is None:
return errreq("Invalid API Key")
else:
if shorturl.query.filter_by(short_query=short_query).first() is not None:
return errreq("Already Exists")
else:
if original[:4] != 'http':
original = 'http://' + original
if not url_valid(original):
return bad_request('Provided url is not valid.')
url_db = shorturl(
short_query=short_query, original=original, visits=0)
db.session.add(url_db)
db.session.commit()
return jsonify({'link': short_query}), 200
else:
return bad_request("Nothing Provided")
#app.route('/view/<alias>', methods=['GET'])
def get_viewcount(alias):
if shorturl.query.filter_by(short_query=alias).first() is None:
return bad_request('Unknown alias.')
else:
return jsonify({'visits':shorturl.query.filter_by(short_query=alias).first().visits}),200
#app.route('/<alias>', methods=['GET'])
def get_shortened(alias):
if shorturl.query.filter_by(short_query=alias).first() is None:
return bad_request('Unknown alias.')
visits = shorturl.query.filter_by(short_query=alias).first().visits
url_db = shorturl.query.filter_by(short_query=alias).first()
url_db.visits = visits + 1
db.session.commit()
url = shorturl.query.filter_by(short_query=alias).first().original
return redirect(url, code=302)
# From https://stackoverflow.com/questions/7160737/python-how-to-validate-a-url-in-python-malformed-or-not#7160778
# Slightly modified to not use ftp.
regex = re.compile(
r'^(?:http)s?://'
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
r'localhost|'
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
r'(?::\d+)?'
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
if __name__ == '__main__':
'''
if os.path.exists('./db.sqlite3'):
app.run(host="0.0.0.0", port=8000, debug=False)
else: '''
db.create_all()
app.run(host="0.0.0.0", port=8000, debug=False)
Here if I add api key in /appapi with key in query string, I can add the key in the database. But the key added in the database does not work after a couple of hours.
Thanks in advance.

I figured this out. It is because I have added db.create_all() inside the main function. Removing this just works.

Python Graphene Subscription Server

I try to subscribe from my react frontend using ApolloClient to a Python GraphQL server I implemented according to https://github.com/graphql-python/graphql-ws/blob/master/examples/flask_gevent/app.py.
My frontend subscription looks like this
client.subscribe({
query: gql`
subscription{
count_seconds
}
`
}).subscribe({
next(data) {
console.log("New data received from subscription");
}
});
but my server complains that
Traceback (most recent call last):
File "C:\Users\Adrian\Miniconda3\lib\site-packages\gevent\pywsgi.py", line 976, in handle_one_response
self.run_application()
File "C:\Users\Adrian\Miniconda3\lib\site-packages\geventwebsocket\handler.py", line 75, in run_application
self.run_websocket()
File "C:\Users\Adrian\Miniconda3\lib\site-packages\geventwebsocket\handler.py", line 52, in run_websocket
list(self.application(self.environ, lambda s, h, e=None: []))
File "C:\Users\Adrian\Miniconda3\lib\site-packages\flask\app.py", line 2463, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\Adrian\Miniconda3\lib\site-packages\flask_sockets.py", line 45, in __call__
handler(environment, **values)
File "C:\Users\Adrian\Miniconda3\lib\site-packages\flask_cors\decorator.py", line 128, in wrapped_function
resp = make_response(f(*args, **kwargs))
File "C:\Users\Adrian\Miniconda3\lib\site-packages\flask\helpers.py", line 223, in make_response
return current_app.make_response(args)
File "C:\Users\Adrian\Miniconda3\lib\site-packages\flask\app.py", line 2130, in make_response
" {rv.__class__.__name__}.".format(rv=rv)
TypeError: The view function did not return a valid response. The return type must be a string, dict, tuple, Response instance, or WSGI callable, but it was a list.
2020-03-27T21:16:07Z {'REMOTE_ADDR': '::1', 'REMOTE_PORT': '55650', 'HTTP_HOST': 'localhost:5000', (hidden keys: 32)} failed with TypeError
My server implementation
from flask import Flask, make_response, request
from flask_sockets import Sockets
from flask_cors import CORS, cross_origin
from flask_graphql import GraphQLView
from graphql_ws.gevent import GeventSubscriptionServer
from schema import schema
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
from graphql.backend import GraphQLCoreBackend
app = Flask(__name__)
app.debug = False
cors = CORS(app, resources={r"/graphql/*": {"origins": "*"}})
app.config['CORS_HEADERS'] = 'Content-Type'
sub_server = GeventSubscriptionServer(schema)
sockets = Sockets(app)
class Server:
def __init__(self):
self.server = pywsgi.WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
app.add_url_rule('/graphql', view_func=GraphQLView.as_view('graphql', schema=schema, graphiql=False))
app.app_protocol = lambda environ_path_info: 'graphql-ws'
self.server.serve_forever()
#sockets.route('/subscriptions')
#cross_origin()
def echo_socket(ws):
sub_server.handle(ws)
return []
I couldn't figure out what the root cause for this is,... where exactly is a list returned where I should have something else?
Maybe the schema is also relevant
import graphene
from rx import Observable
import random
class Query(graphene.ObjectType):
hello = graphene.String(name=graphene.String(default_value="World"))
def resolve_hello(self, info, name):
print("answer query")
return 'Hello ' + name
class RandomType(graphene.ObjectType):
seconds = graphene.Int()
random_int = graphene.Int()
class Subscription(graphene.ObjectType):
count_seconds = graphene.Int(up_to=graphene.Int())
random_int = graphene.Field(RandomType)
def resolve_count_seconds(self, info, up_to=5):
print("new subscription")
return Observable.interval(1000) \
.map(lambda i: "{0}".format(i)) \
.take_while(lambda i: int(i) <= up_to)
def resolve_random_int(self, info):
return Observable.interval(1000).map(lambda i: RandomType(seconds=i, random_int=random.randint(0, 500)))
schema = graphene.Schema(query=Query, subscription=Subscription)

cgi.FieldStorage not reads json data from requests.post

I've setup simple server as described in Python Cookbook (ch.11)
# server.py
import cgi
def notfound_404(environ, start_response):
start_response('404 Not found', [('Content-type', 'text-plain')])
return [b'Not found']
class PathDispatcher:
def __init__(self):
self.pathmap = {}
def __call__(self, environ, start_response):
path = environ['PATH_INFO']
post_env = environ.copy()
post_env['QUERY_STRING'] = ''
params = cgi.FieldStorage(fp=environ['wsgi.input'], environ=post_env, keep_blank_values=True)
environ['params'] = {key: params.getvalue(key) for key in params}
method = environ['REQUEST_METHOD'].lower()
handler = self.pathmap.get((method, path), notfound_404)
return handler(environ, start_response)
def register(self, method, path, function):
self.pathmap[method.lower(), path] = function
return function
and
# app.py
def send_json(environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
params = environ['params']
result = ""
for key, param in params.iteritems():
result += str(key) + ' :: ' + str(param) + '\n'
yield result.encode('utf-8')
if __name__ == '__main__':
from server import PathDispatcher
from wsgiref.simple_server import make_server
dispatcher = PathDispatcher()
dispatcher.register('POST', '/send-json', send_json)
httpd = make_server('', 8080, dispatcher)
print('Listening on 8080...')
httpd.handle_request()
Simple agent sends some json data with python.requests
# agent.py
import requests
import json
json_data = {'some': 'data', 'moredata':[{1: 'one'}, {2: 'two'}]}
url = "http://localhost:8080/send-json"
headers = {'Content-Type': 'application/json'}
r = requests.post(url=url, data=json.dumps(json_data), headers=headers)
print r.text
Unfortunately, it produces errors like this
Traceback (most recent call last):
File "/usr/lib/python2.7/wsgiref/handlers.py", line 85, in run
self.result = application(self.environ, self.start_response)
File "/home/phux/PycharmProjects/printoscope_sql_injection/server.py", line 24, in __call__
environ['params'] = {key: params.getvalue(key) for key in params}
File "/usr/lib/python2.7/cgi.py", line 517, in __iter__
return iter(self.keys())
File "/usr/lib/python2.7/cgi.py", line 582, in keys
raise TypeError, "not indexable"
TypeError: not indexable
127.0.0.1 - - [23/Sep/2015 12:25:17] "POST /initial-scan HTTP/1.1" 500 59
Application cannot iterate over the received data and wsgi.FieldStorage doesn't contain MiniFieldStorage fields just raw json data
FieldStorage(None, None, '{"moredata": [{"1": "one"}, {"2": "two"}], "some": "data"}')
If I try to send data like this
r = requests.post(url=url, data=json_data)
everything works fine and FieldStorage looks fine
FieldStorage(None, None, [MiniFieldStorage('moredata', '1'), MiniFieldStorage('moredata', '2'), MiniFieldStorage('some', 'data')])
BUT I need to receive json data in the final application, so ...
Thanks in advance
Phux

--------------SOLUTION-------------
Just replace these lines in server.py
post_env = environ.copy()
post_env['QUERY_STRING'] = ''
params = cgi.FieldStorage(fp=environ['wsgi.input'], environ=post_env, keep_blank_values=True)
environ['params'] = {key: params.getvalue(key) for key in params}
With this
try:
request_body_size = int(environ.get('CONTENT_LENGTH', 0))
except ValueError:
request_body_size = 0
request_body = environ['wsgi.input'].read(request_body_size)
params = json.loads(request_body)
environ['params'] = {key: params[key] for key in params}
cgi.FieldStorage expects form and I don't send one ... and this is the root of the problem. Some slight modification in app.py is also needed, but this is not the case and can be easily adjusted.