I have an application where a service makes a POST request to a flask app which then inserts the body of request into a postgresql database into a table as well as inserting data into several other tables.
The problem occurs once a curtain amount of requests are made that the queue pool limit is reach and the app throws an error. i have isolated it down to the extra functions called to do the inserting into other tables. i don't understand what is happening.
Each service is a docker container
172.30.0.17 - - [16/Jun/2022 06:25:07] "GET /api/orgs HTTP/1.1" 500 -
Traceback (most recent call last):
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/service/routes/get/get_org.py", line 17, in base_route
query = [convert_class(item) for item in Organization.query.all()]
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1711, in execute
conn = self._connection_for_bind(bind)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1552, in _connection_for_bind
return self._transaction._connection_for_bind(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
conn = bind.connect()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3234, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3313, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3280, in _wrap_pool_connect
return fn()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 476, in checkout
rec = pool._do_get()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 134, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)
Config
class Config:
# Database
SQLALCHEMY_DATABASE_URI = environ.get("DB_URI")
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_size": 1,
"max_overflow": 0,
}
App init
db = SQLAlchemy()
# create flask App
def create_app():
app = Flask(__name__, instance_relative_config=False)
# DB config - postgreSQL
app.config['SQLALCHEMY_DATABASE_URI'] = getenv("DB_URI")
app.config.from_object('config.Config')
# config server with CORS
# cors = CORS(app, resources={r"/*": {"origins": "*"}})
with app.app_context():
db.init_app(app)
from service.routes.post import post_event
return app
app = create_app()
POST request
#app.route('/api/orgs/<org_id>/sites/<site_id>/sessions/<session_id>/events', methods=["POST"])
def edger_post_request(org_id, site_id, session_id):
print("URL params: {}, {}, {}".format(
org_id, site_id, session_id
))
# parse body of post request
try:
request_body = request.json
event_data = json.loads(request_body["data"])
except:
return "error in body of request", 400
print("Body of request: ", request_body)
# check if body of request is valid
errors_body, has_error = validate_input(request_body)
print(errors_body, has_error)
if has_error:
return Response(errors_body, 400)
# insert event
event_dict = dict()
try:
event_dict = {
"id": request_body["id"],
"timestamp": request_body["timestamp"],
"type": request_body["type"],
"event_type": event_data["type"],
"data": request_body["data"],
"org_id": org_id,
"site_id": site_id,
"device_id": request_body["device_id"],
"session_id": request_body["session_id"]
}
if "frame_url" in request_body.keys():
event_dict["frame_url"] = request_body["frame_url"]
else:
event_dict["frame_url"] = None
insert_event = Event(**event_dict)
db.session.add(insert_event)
# commit event to DB
db.session.commit()
# Insert extra tables
insert_threads(org_id, site_id, event_dict, event_data)
db.session.commit()
db.session.close()
return {"result": "success"}, 200
except Exception as e:
print("- - - - - - - - - - - - - - - - -")
db.session.rollback()
print("Error inserting Event", e)
print("- - - - - - - - - - - - - - - - -")
db.session.close()
return {"result": "unsuccessful"}, 409
insert_threads function
def insert_threads(org_id, site_id, body, data):
event_data_insert(body, data),
event_type_insert(data["type"]),
insert_org(org_id),
insert_site(org_id, site_id),
insert_device(org_id, site_id, body['device_id']),
insert_session(org_id, site_id, body['device_id'], body['session_id'], body['timestamp'])
def insert_org(org_id):
org_list = getattr(g, "org_list", None)
# query organization insert if doesn't exist
if org_list is None:
check = [org.id for org in Organization.query.all()]
g.org_list = check
org_list = check
if org_id not in org_list:
insert_org = Organization(id=org_id, name="temp_name")
db.session.add(insert_org)
org_list.append(insert_org.id)
g.org_list = org_list
def insert_site(org_id, site_id):
site_list = getattr(g, "site_list", None)
# query Site insert if doesn't exist
if site_list is None:
check = [site.id for site in Site.query.all()]
g.site_list = check
site_list = check
if site_id not in site_list:
insert_site = Site(id=site_id, org_id=org_id, name="temp_name")
db.session.add(insert_site)
site_list.append(insert_site.id)
g.site_list = site_list
def insert_device(org_id, site_id, device_id):
device_list = getattr(g, "device_list", None)
# query device insert if doesn't exist
if device_list is None:
check = [device.id for device in Device.query.all()]
g.device_list = check
device_list = check
if device_id not in device_list:
insert_device = Device(id=device_id, site_id=site_id, org_id=org_id)
db.session.add(insert_device)
device_list.append(insert_device.id)
g.device_list = device_list
def insert_session(org_id, site_id, device_id, session_id, timestamp):
session_list = getattr(g, "session_list", None)
# query session and insert if it doesn't already exist
if session_list is None:
check = [session.id for session in Session.query.all()]
g.session_list = check
session_list = check
if session_id not in session_list:
insert_session = Session(
id=session_id,
timestamp=timestamp,
org_id=org_id,
site_id=site_id,
device_id=device_id
)
db.session.add(insert_session)
session_list.append(insert_session.id)
g.session_list = session_list
def event_data_insert(event, event_data):
# get white list from global var else None
query_list = getattr(g, "query_list", None)
# if query_list is None get query_list from DB
if query_list is None:
check = [convert_class(item) for item in Event_data_query_list.query.all()]
g.query_list = check
query_list = check
# loop though query_list and insert key/value pair into event_data table
for key in query_list:
print("the key: ", key)
key_path = key["key_name"].split(".")
# check if keys exist in data Object and continue down levels till the value is reached
if key_path[0] in event_data["data"] and key["type"] == event_data["type"]:
print("inside first if")
value = event_data['data']
for sub_key in key_path:
print("inside loop")
if sub_key in value.keys():
print("in subkey" , sub_key)
value = value[sub_key]
# handle if keys and values dont exist
else:
value = None
break
if value is None:
continue
# determine where value is an Integer or a String and insert into correct column
val_int = None
val_str = None
if type(value) == str:
val_str = value
elif type(value) == int:
val_int = value
# add event_data row to session
insert = Event_data(event_id=event["id"], type=event_data["type"], key=key["key_name"], value_int=val_int, value_str=val_str )
print("insert object", insert)
db.session.add(insert)
def event_type_insert(event_type):
event_type_list = getattr(g, "event_type_list", None)
if event_type_list is None:
check = [item.event_type for item in Event_type.query.all()]
g.event_type_list = check
event_type_list = check
if event_type not in event_type_list:
insert = Event_type(event_type=event_type)
db.session.add(insert)
g.event_type_list.append(event_type)
Related
So I have been working on Flask and Stripe for a week now, and I have made some progress(I wanna thank stack overflow). I really need help with sessions though. I keep receiving a KeyError when using Flask not sure why. I have added a comment to each piece of code im referring to.
The comment is: #THIS IS THE CODE IM REFERRING TOO
Any help would be appreciated, I assume this might be due to the conflicting variable names that stripe uses and flask uses(since they both use session) but wouldn't I get an error for that specifically? Instead I just get a KeyError.
Thank you.
from flask import Flask, render_template, url_for, request, abort, session
import random
import subprocess
import stripe
app = Flask(__name__)
app.config['STRIPE_PUBLIC_KEY'] = 'PUBLIC KEY'
app.config['STRIPE_SECRET_KEY'] = 'SECRET KEY'
stripe.api_key = app.config['STRIPE_SECRET_KEY']
app.secret_key="SECRETFORSESSIONS"
#app.route('/', methods =["GET", "POST"])
def home():
if request.method == "POST":
location = request.form["location"] #THIS IS THE CODE IM REFERRING TOO
industry = request.form["industry"] #THIS IS THE CODE IM REFERRING TOO
session['location'] = location #THIS IS THE CODE IM REFERRING TOO
session['industry'] = industry #THIS IS THE CODE IM REFERRING TOO
return render_template("home.html")
else:
return render_template("home.html")
return render_template("home.html")
#app.route('/initialpay')
def index():
return render_template("index.html")
#app.route('/stripe_pay')
def stripe_pay():
session = stripe.checkout.Session.create(
payment_method_types=['card'],
line_items=[{
'price': 'priceofanitem',
'quantity': 1,
}],
mode='payment',
success_url=url_for('thanks', _external=True) + '?session_id={CHECKOUT_SESSION_ID}',
cancel_url=url_for('index', _external=True),
)
return {
'checkout_session_id': session['id'],
'checkout_public_key': app.config['STRIPE_PUBLIC_KEY']
}
#app.route('/thanks')
def thanks():
return render_template("thanks.html")
#app.route('/stripe_webhook', methods=['POST'])
def stripe_webhook():
print('WEBHOOK CALLED')
if request.content_length > 1024 * 1024:
print('REQUEST TOO BIG')
abort(400)
payload = request.get_data()
sig_header = request.environ.get('HTTP_STRIPE_SIGNATURE')
endpoint_secret = 'ENDPOINT_SECRET'
event = None
try:
event = stripe.Webhook.construct_event(
payload, sig_header, endpoint_secret
)
except ValueError as e:
# Invalid payload
print('INVALID PAYLOAD')
return {}, 400
except stripe.error.SignatureVerificationError as e:
# Invalid signatures
print('INVALID SIGNATURE')
return {}, 400
# Handle the checkout.session.completed event
if event['type'] == 'checkout.session.completed':
session = event['data']['object']
print(session)
line_items = stripe.checkout.Session.list_line_items(session['id'], limit=1)
print(line_items['data'][0]['description'])
location = session["location"] #THIS IS THE CODE IM REFERRING TOO
industry = session["industry"] #THIS IS THE CODE IM REFERRING TOO
print(location)
print(industry)
emaildata = session.customer_details.email
sessionParam = str(random.randint(10000, 90000))
if checkout_session.payment_status == 'paid':
#DO TASKS
return {}
if __name__ == "__main__":
app.run(threaded=True)
Here is the error:
[2022-11-05 07:11:51,884] ERROR in app: Exception on /stripe_webhook [POST]
Traceback (most recent call last):
File "/usr/local/lib/python3.10/dist-packages/flask/app.py", line 2525, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.10/dist-packages/flask/app.py", line 1822, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/usr/local/lib/python3.10/dist-packages/flask/app.py", line 1820, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.10/dist-packages/flask/app.py", line 1796, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)
File "/mainapp/__init__.py", line 85, in stripe_webhook
location = s["location"]
File "/usr/local/lib/python3.10/dist-packages/flask/sessions.py", line 80, in __getitem__
return super().__getitem__(key)
KeyError: 'location'
How to import data into mysql from the given JSON file in Python? I'm getting error as
Traceback (most recent call last): File "C:/Users/risha/PycharmProjects/Test/JSON_TEST.py", line 24, in
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306') File
"C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql__init__.py",
line 90, in Connect
return Connection(*args, **kwargs) File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 704, in init
self.connect() File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 964, in connect
self.host_info = "socket %s:%d" % (self.host, self.port) TypeError: %d format: a number is required, not str
import pymysql
import os
import json
# read JSON file which is in the next parent folder
file = 'C://Users//risha//PycharmProjects//Test//video.json'
json_data= open(file).read()
json_obj = json.loads(json_data)
# do validation and checks before insert
def validate_string(val):
if val != None:
if type(val) is int:
#for x in val:
# print(x)
return str(val).encode('utf-8')
else:
return val
# connect to MySQL
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306')
cursor = con.cursor()
# parse json data to SQL insert
for i, item in enumerate(json_obj):
person = validate_string(item.get("person", None))
year = validate_string(item.get("year", None))
company = validate_string(item.get("company", None))
cursor.execute("INSERT INTO testp (person, year, company) VALUES (%s, %s, %s)", (person, year, company))
con.commit()
con.close()
I need to Copy every mail from my 'inbox'+sub folders to 2 new folders, one for short subjects and another for the long ones.
The crawl always abort after the 10th mail, and I need to repeat the process for 10 000 of them.. if someone had a solution I would be very grateful, here is my code :
import imaplib, getpass, re, email
pattern_uid = re.compile(b'\d+ \(UID (?P<uid>\d+)\)')
def parse_mailbox(data):
flags, b, c = data.partition(' ')
separator, b, name = c.partition(' ')
return (flags, separator.replace('"', ''), name.replace('"', ''))
def connect(email):
imap = imaplib.IMAP4_SSL("outlook.office365.com",993)
password = "*******"
imap.login(email, password)
return imap
def disconnect(imap):
imap.logout()
def parse_uid(data):
match = pattern_uid.match(data)
return match.group('uid')
if __name__ == '__main__':
imap = connect('************')
rv,data = imap.list()
# print(data)
if rv == 'OK':
for mbox in data:
flags,separator,name = parse_mailbox(bytes.decode(mbox))
print(name)
if 'HasNoChildren' in flags and '2' in name:
name = name.replace('/ Inbox','Inbox')
rv2, data2 = imap.select('"'+name+'"')
print(rv2)
resp, items = imap.search(None, 'All')
email_ids = items[0].split()
# print(items)
print(email_ids)
# Assuming that you are moving the latest email.
# for mail in email_ids:
# if
print(len(email_ids))
for i in range(0,len(email_ids)):
print('NOUVEAU MESSAGE \n')
print(email_ids[i])
resp, data = imap.fetch(email_ids[i], "(UID)")
rv,sujet = imap.fetch(email_ids[i],("(RFC822)"))
varSubject= ""
for response_part in sujet:
if isinstance(response_part, tuple):
msg = email.message_from_bytes(response_part[1])
varSubject = msg['subject']
break
print(varSubject)
print(i)
print(type(data[0]))
print(data[0])
try :
msg_uid = parse_uid(data[0])
except AttributeError:
continue
num = ""
dossier = ""
if len(varSubject) <= 20:
dossier = 'INBOX/sujets_courts'
imap.create(dossier)
elif len(varSubject) > 20:
dossier = 'INBOX/sujets_longs'
imap.create(dossier)
print(varSubject+'******'+dossier)
print(msg_uid)
result = imap.uid('COPY', msg_uid, dossier)
# if result[0] == 'OK':
# mov, data = imap.uid('STORE', msg_uid , '+FLAGS', '(\Deleted)')
# imap.expunge()
disconnect(imap)
The email and the password are substituted by '****'
Thanks a lot :)
console output :
Traceback (most recent call last):
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1011, in _command_complete
typ, data = self._get_tagged_response(tag)
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1131, in _get_tagged_response
self._get_response()
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1039, in _get_response
resp = self._get_line()
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1143, in _get_line
raise self.abort('socket error: EOF')
imaplib.abort: socket error: EOF
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "movemail.py", line 44, in <module>
resp, data = imap.fetch(email_ids[i], "(UID)")
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 526, in fetch
typ, dat = self._simple_command(name, message_set, message_parts)
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1188, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1013, in _command_complete
raise self.abort('command: %s => %s' % (name, val))
imaplib.abort: command: FETCH => socket error: EOF
OK I figured out that the problem comes from the creation of an already-created folder, but I could not find any folder.exists() look-a-like method in the imaplib doc...
This is an app to search tweets and save the results to mongdb. This line is problematic
search_results = twitter_api.search.tweets(q=q, count=100, **kw)
I'm getting this error:
Traceback (most recent call last):
File "mongo.py", line 89, in
results = twitter_search(twitter_api,q,max_results=10)
File "mongo.py", line 21, in twitter_search
search_results = twitter_api.search.tweets(q=q, count=100, **kw)
File "/usr/local/lib/python2.7/dist-packages/twitter/api.py", line 312, in __call__
return self._handle_response(req, uri, arg_data, _timeout)
File "/usr/local/lib/python2.7/dist-packages/twitter/api.py", line 345, in _handle_response
raise TwitterHTTPError(e, uri, self.format, arg_data)
twitter.api.TwitterHTTPError: Twitter sent status 401 for URL: 1.1/search/tweets.json using parameters: (count=100&oauth_consumer_key= &oauth_nonce= &oauth_signature_method=HMAC-SHA1&oauth_timestamp=1457104474&oauth_token= &oauth_version=1.0&q=CrossFit&oauth_signature=6SMuzJBWfFUEmZHlu%2F4Hmt6Zu8k%3D)
details: {u'errors': [{u'message': u'Could not authenticate you.', u'code': 32}]}
Here is my code:
import json
import pymongo
import twitter
def oauth_login():
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
OAUTH_TOKEN = ''
OAUTH_TOKEN_SECRET = ''
auth = twitter.oauth.OAuth(OAUTH_TOKEN,OAUTH_TOKEN_SECRET,CONSUMER_KEY,CONSUMER_SECRET)
twitter_api = twitter.Twitter(auth=auth)
return twitter_api
def twitter_search(twitter_api, q, max_results=200, **kw):
search_results = twitter_api.search.tweets(q=q, count=100, **kw)
statuses = search_results['statuses']
max_results = min(1000, max_results)
for _ in range(10): # 10*100 = 1000
try:
next_results = search_results['search_metadata']['next_results']
except KeyError, e:
break
kwargs = dict([kv.split('=')
for kv in next_results[1:].split("&") ])
search_results = twitter_api.search.tweets(**kwargs)
statuses += search_results['statuses']
if len(statuses) > max_results:
break
return statuses
def save_to_mongo(data,mongo_db,mongo_db_coll,**mongo_conn_kw):
client = pymongo.MongoClient(**mongo_conn_kw)
db = client[mongo_db]
coll = db[mongo_db_coll]
return coll.insert(data)
def load_from_mongo(mongo_db,mongo_db_coll,return_cursor=False,criteria=None,projection=None,**mongo_conn_kw):
client = pymongo.MongoClient(**mongo_conn_kw)
db = client[mongo_db]
coll = db[mongo_db_coll]
if criteria is None:
criteria = {}
if projection is None:
cursor = coll.find(criteria)
else:
cursor = coll.find(criteria,projection)
#Returning a cursor is recommended for large amounts of data
if return_cursor:
return cursor
else:
return [item for item in cursor]
#Sample usage
q = 'CrossFit'
twitter_api = oauth_login()
results = twitter_search(twitter_api,q,max_results=10)
save_to_mongo(results,'search_results',q)
load_from_mongo('search_results',q)
I've setup simple server as described in Python Cookbook (ch.11)
# server.py
import cgi
def notfound_404(environ, start_response):
start_response('404 Not found', [('Content-type', 'text-plain')])
return [b'Not found']
class PathDispatcher:
def __init__(self):
self.pathmap = {}
def __call__(self, environ, start_response):
path = environ['PATH_INFO']
post_env = environ.copy()
post_env['QUERY_STRING'] = ''
params = cgi.FieldStorage(fp=environ['wsgi.input'], environ=post_env, keep_blank_values=True)
environ['params'] = {key: params.getvalue(key) for key in params}
method = environ['REQUEST_METHOD'].lower()
handler = self.pathmap.get((method, path), notfound_404)
return handler(environ, start_response)
def register(self, method, path, function):
self.pathmap[method.lower(), path] = function
return function
and
# app.py
def send_json(environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
params = environ['params']
result = ""
for key, param in params.iteritems():
result += str(key) + ' :: ' + str(param) + '\n'
yield result.encode('utf-8')
if __name__ == '__main__':
from server import PathDispatcher
from wsgiref.simple_server import make_server
dispatcher = PathDispatcher()
dispatcher.register('POST', '/send-json', send_json)
httpd = make_server('', 8080, dispatcher)
print('Listening on 8080...')
httpd.handle_request()
Simple agent sends some json data with python.requests
# agent.py
import requests
import json
json_data = {'some': 'data', 'moredata':[{1: 'one'}, {2: 'two'}]}
url = "http://localhost:8080/send-json"
headers = {'Content-Type': 'application/json'}
r = requests.post(url=url, data=json.dumps(json_data), headers=headers)
print r.text
Unfortunately, it produces errors like this
Traceback (most recent call last):
File "/usr/lib/python2.7/wsgiref/handlers.py", line 85, in run
self.result = application(self.environ, self.start_response)
File "/home/phux/PycharmProjects/printoscope_sql_injection/server.py", line 24, in __call__
environ['params'] = {key: params.getvalue(key) for key in params}
File "/usr/lib/python2.7/cgi.py", line 517, in __iter__
return iter(self.keys())
File "/usr/lib/python2.7/cgi.py", line 582, in keys
raise TypeError, "not indexable"
TypeError: not indexable
127.0.0.1 - - [23/Sep/2015 12:25:17] "POST /initial-scan HTTP/1.1" 500 59
Application cannot iterate over the received data and wsgi.FieldStorage doesn't contain MiniFieldStorage fields just raw json data
FieldStorage(None, None, '{"moredata": [{"1": "one"}, {"2": "two"}], "some": "data"}')
If I try to send data like this
r = requests.post(url=url, data=json_data)
everything works fine and FieldStorage looks fine
FieldStorage(None, None, [MiniFieldStorage('moredata', '1'), MiniFieldStorage('moredata', '2'), MiniFieldStorage('some', 'data')])
BUT I need to receive json data in the final application, so ...
Thanks in advance
Phux
--------------SOLUTION-------------
Just replace these lines in server.py
post_env = environ.copy()
post_env['QUERY_STRING'] = ''
params = cgi.FieldStorage(fp=environ['wsgi.input'], environ=post_env, keep_blank_values=True)
environ['params'] = {key: params.getvalue(key) for key in params}
With this
try:
request_body_size = int(environ.get('CONTENT_LENGTH', 0))
except ValueError:
request_body_size = 0
request_body = environ['wsgi.input'].read(request_body_size)
params = json.loads(request_body)
environ['params'] = {key: params[key] for key in params}
cgi.FieldStorage expects form and I don't send one ... and this is the root of the problem. Some slight modification in app.py is also needed, but this is not the case and can be easily adjusted.