Develop Reference

Python Flask types of showing responses because response is too big

This is my first time using Flask to build an API.
I have a CSV file that I'm reading and storing into a dataframe and then being converted to a dictionary in order to become a response for a Flask API using flask_restful to show a JSON output.
The CSV file has around 230000 rows, so when I try to connect to the endpoint it takes to much time to load the data. The data is only fully loaded if I use parameters to filter by date, so the dataframe can show less rows.
If I try to use Postman to test the endpoint I can see that the full load is 120mb of data (only if I change the settings to show more than 50mb), so Power BI or any other tool is not showing all the data as well.
This is the response I get with full data from the endpoint without parameters (it takes to much time to load):
So my question is, is there any other way to show the JSON output from Flask and connect in other tools with this amount of rows?
Endpoint with full data: https://masterdimensions.herokuapp.com/dimensioncalendar
Endpoint with parameters: https://masterdimensions.herokuapp.com/dimensioncalendar?start_dt=2020-01-01&end_dt=2021-12-31
My application:
# app.py
from flask import Flask
from flask_restful import Resource, Api, reqparse
import pandas as pd
import os
import pathlib
# Launching an app flask
app = Flask(__name__)
api = Api(app, default_mediatype='application/json')
# API class for Index page
class Index(Resource):
def get(self):
index_data = [
{
"href": "/dimensioncalendar",
"name": "Dimension Calendar",
"description": "Dimension Calendar endpoint with date attributes"
}
]
return (index_data), 200
# API class for DimensionCalendar page
class DimensionCalendar(Resource):
def get(self):
# Initialize reparse from Flask
parser = reqparse.RequestParser()
# Add arguments
parser.add_argument('start_dt')
parser.add_argument('end_dt')
parser.add_argument('lang')
args = parser.parse_args() # parse arguments to dictionary
# Read CSV file
data = pd.read_csv(os.path.join(os.path.sep, pathlib.Path(__file__).parent.resolve(), "static", "csv", "DimensionCalendar.csv"))
# If parameters on API is not null then filter dataframe by start date and end date
if args['start_dt'] is not None and args['end_dt'] is not None:
data = data.loc[(data['DT_SHORTDATE'] >= str(args['start_dt'])) & (data['DT_SHORTDATE'] <= str(args['end_dt']))]
elif args['start_dt'] is not None and args['end_dt'] is None:
data = data.loc[data['DT_SHORTDATE'] == str(args['start_dt'])]
if args['lang'] == 'pt-BR':
data.columns = ['DT_DATA', 'NR_ANO', 'NR_MES', 'NR_DIA', 'NM_MES', 'NM_DIADASEMANA', 'NR_MESANO', 'NM_MESANO', 'NM_DATALONGA', 'NR_DIADASEMANA', 'NR_DIADOANO', 'NR_SEMESTRE', 'NM_SEMESTRE', 'NR_QUADRIMESTRE', 'NR_TRIMESTRE', 'NM_TRIMESTRE', 'NR_BIMESTRE', 'NM_BIMESTRE', 'NR_ANORELATIVO', 'NR_MESRELATIVO', 'NR_DIARELATIVO', 'NR_DATA', 'NR_ANOMES', 'NR_ANOTRIMESTRE', 'IC_DIAUTIL', 'IC_FINALDESEMANA', 'IC_FERIADO', 'IC_ANOATUAL', 'IC_MESATUAL', 'IC_DATAATUAL', 'IC_ANOPASSADO', 'IC_MESPASSADO', 'IC_ONTEM', 'LINDATA', 'LINORIGEM']
# Convert dataframe to dictionary
data = data.to_dict()
# Return data and 200 OK code
return (data), 200
# Case the page is not found
#app.errorhandler(404)
def page_not_found(e):
return "<h1>404</h1><p>The resource could not be found.</p>", 404
# '/dimensioncalendar' is our entry point for DimensionCalendar
api.add_resource(DimensionCalendar, '/dimensioncalendar')
# '/Index' is our entry point for Index
api.add_resource(Index, '/')
# Run our Flask app
if __name__ == '__main__':
app.run()

How to assert JSON data using nose2? [duplicate]

How can I test that the response a Flask view generated is JSON?
from flask import jsonify
#app.route('/')
def index():
return jsonify(message='hello world')
c = app.app.test_client()
assert c.get('/').status_code == 200
# assert is json

As of Flask 1.0, response.get_json() will parse the response data as JSON or raise an error.
response = c.get("/")
assert response.get_json()["message"] == "hello world"
jsonify sets the content type to application/json. Additionally, you can try parsing the response data as JSON. If it fails to parse, your test will fail.
from flask import json
assert response.content_type == 'application/json'
data = json.loads(response.get_data(as_text=True))
assert data['message'] == 'hello world'
Typically, this test on its own doesn't make sense. You know it's JSON since jsonify returned without error, and jsonify is already tested by Flask. If it was not valid JSON, you would have received an error while serializing the data.

There is a python-library for it.
import json
#...
def checkJson(s):
try:
json.decode(s)
return True
except json.JSONDecodeError:
return False
If you also want to check if it is a valid string, check the boundaries for "s.
You can read the help here on pythons website https://docs.python.org/3.5/library/json.html .

Catch http-status code in Flask

I lately started using Flask in one of my projects to provide data via a simple route. So far I return a json file containing the data and some other information. When running my Flask app I see the status code of this request in terminal. I would like to return the status code as a part of my final json file. Is it possible to catch the same code I see in terminal?
Some simple might look like this
from flask import Flask
from flask import jsonify
app = Flask(__name__)
#app.route('/test/<int1>/<int2>/')
def test(int1,int2):
int_sum = int1 + int2
return jsonify({"result":int_sum})
if __name__ == '__main__':
app.run(port=8082)
And in terminal I get:

You are who set the response code (by default 200 on success response), you can't catch this value before the response is emited. But if you know the result of your operation you can put it on the final json.
#app.route('/test/<int1>/<int2>/')
def test(int1, int2):
int_sum = int1 + int2
response_data = {
"result": int_sum,
"sucess": True,
"status_code": 200
}
# make sure the status_code on your json and on the return match.
return jsonify(response_data), 200 # <- the status_code displayed code on console
By the way if you access this endpoint from a request library, on the response object you can find the status_code and all the http refered data plus the json you need.
Python requests library example
import requests
req = requests.get('your.domain/test/3/3')
print req.url # your.domain/test/3/3
print req.status_code # 200
print req.json() # {u'result': 6, u'status_code: 200, u'success': True}

You can send HTTP status code as follow:
#app.route('/test')
def test():
status_code = 200
return jsonify({'name': 'Nabin Khadka'}, status_code) # Notice second element of the return tuple(return)
This way you can control what status code to return to the client (typically to web browser.)

How to send and receive data from flask?

I am trying to send a python dictionary created from client.py to my webservice, have the webservice do something to the data, and return a boolean to client.py. This is the code I have so far for the server and client:
Server side (inside webservice.py):
from flask import Flask
from flask import request
import json
app = Flask(__name__)
#app.route('/determine_escalation',methods = ['POST'])
def determine_escalation():
jsondata = request.form['jsondata']
data = json.loads(jsondata)
#stuff happens here that involves data to obtain a result
result = {'escalate':'True'}
return json.dumps(result)
if __name__ == '__main__':
app.run(debug=True)
Client side (inside client.py):
import sys
import json
import requests
conv = [{'input': 'hi', 'topic': 'Greeting'}]
s = json.dumps(conv)
res = requests.post("http://127.0.0.1:5000/determine_escalation",data=s)
print res.text
But when I print out res.text, I get this:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<title>400 Bad Request</title>
<h1>Bad Request</h1>
<p>The browser (or proxy) sent a request that this server could not understand.</p>
What am I doing wrong, and how can I fix this? New to Flask and JSON stuff so any help is appreciated.

OK - a few issues here:
First, you can use requests.get_json() to retrieve your JSON data at the server end:
from flask import Flask
from flask import request
import json
app = Flask(__name__)
#app.route('/determine_escalation/', methods = ['POST'])
def determine_escalation():
jsondata = request.get_json()
data = json.loads(jsondata)
#stuff happens here that involves data to obtain a result
result = {'escalate': True}
return json.dumps(result)
if __name__ == '__main__':
app.run(debug=True)
Also, when you are putting your data together, rather than using "data=s" to send the request, use "json=s":
import sys
import json
import requests
conv = [{'input': 'hi', 'topic': 'Greeting'}]
s = json.dumps(conv)
res = requests.post("http://127.0.0.1:5000/determine_escalation/", json=s).json()
print(res['escalate'])
Please note that I have added trailing slashes at the end of the URL - this is just good practice :-)
I've also incorporated MarcelK's suggested changes - removing the quotes from the boolean 'True' (server side) and using .json() to parse the response on the client side - both of these are great suggestions.
I've tested this revised version (and re-revised version) and it works fine.

How to speed up JSON for a flask application?

I'm currently implementing a webapp in flask. It's an app that does a visualization of data gathered. Each page or section will always have a GET call and each call will return a JSON response which then will be processed into displayed data.
The current problem is that some calculation is needed before the function could return a JSON response. This causes some of the response to arrive slower than others and thus making the page loads a bit slow. How do I properly deal with this? I have read into caching in flask and wonder whether that is what the app need right now. I have also researched a bit into implementing a Redis-Queue. I'm not really sure which is the correct method.
Any help or insights would be appreciated. Thanks in advance

Here are some ideas:
If the source data that you use for your calculations is not likely to change often then you can run the calculations once and save the results. Then you can serve the results directly for as long as the source data remains the same.
You can save the results back to your database, or as you suggest, you can save them in a faster storage such as Redis. Based on your description I suspect the big performance gain will be in not doing calculations so often, the difference between storing in a regular database vs. Redis or similar is probably not significant in comparison.
If the data changes often then you will still need to do calculations frequently. For such a case an option that you have is to push the calculations to the client. Your Flask app can just return the source data in JSON format and then the browser can do the processing on the user's computer.
I hope this helps.

You can use
copy_current_request_context and Redis, Thread
It is helpful when you need long time to make JSON response.
The first request maybe slow, but next request will faster.
Example
from datetime import timedelta, datetime
from threading import Thread
from . import dbb, redis_client
from flask import Blueprint, request, jsonify, flash, after_this_request, copy_current_request_context, \
current_app, send_from_directory
from .models import Shop, Customers
def save_customer_json_to_redis(request):
response_json = {
"have_customer": False,
"status": False,
"anythingelse": None,
"message":"False, you have to check..."
}
#print(request.data)
headers = request.headers
Authorization = headers['Authorization']
token = Authorization.replace("Bearer", "")
phone = request.args.get('phone')
if phone is not None and phone != "":
print('token', token, "phone", phone)
now = datetime.utcnow() + timedelta(hours=7)
shop = Shop.query.filter(Shop.private_token == token, Shop.ended_date > now, Shop.shop_active == True).first()
customer = Customers.query.filter_by(shop_id=shop.id, phone=phone).first()
if customer:
redis_name = f'{shop.id}_api_v2_customer_phone_{phone}_customer_id_{customer.id}'
print(redis_name)
response_json["anythingelse"] = ...# do want you want, it need long time to do
response_json["status"] = True
response_json["message"] = "Successful"
redis_client.set(redis_name, json.dumps(response_json)) #Save JSON to Redis
#app.route('/api/v2/customer', methods=['GET'])
def api_customer():
#copy_current_request_context
def do_update_customer_to_redis():# this function to save JSON you want to response next time to Redis
save_customer_json_to_redis(request)
Thread(target=do_update_customer_to_redis).start()
response_json = {
"have_customer": False,
"status": False,
"anythingelse": {},
"message": "False, you have to check..."
}
#print(request.data)
headers = request.headers
Authorization = headers['Authorization']
token = Authorization.replace("Bearer", "")
phone = request.args.get('phone')
if phone is not None and phone != "":
print('token', token, "phone", phone)
now = datetime.utcnow() + timedelta(hours=7)
shop = Shop.query.filter(Shop.private_token == token, Shop.ended_date > now,Shop.shop_active == True).first()
customer = Customers.query.filter_by(shop_id=shop.id, phone=phone).first()
if customer:
redis_name = f'{shop.id}_api_v2_customer_phone_{phone}_customer_id_{customer.id}'
print(redis_name)
try:
response_json = json.loads(redis_client.get(redis_name)) # if have json from app
print("json.loads(redis_client.get(redis_name))")
except Exception as e:
print("json.loads(redis_client.get(redis_name))", e)
#do any thing you want to response json
response_json["anythingelse"] = ...# do want you want, it need long time to do
response_json["message"]= ...#do want you want
#redis_client.set(redis_name, json.dumps(response_json))
response_json["status"] = True
response_json["message"] = "Successful"
return jsonify(response_json)
In the init.py
from flask import Flask
from flask_cors import CORS
from flask_mail import Mail
from flask_sqlalchemy import SQLAlchemy
from redis import Redis
# init SQLAlchemy so we can use it later in our models
dbb = SQLAlchemy(session_options={"autoflush": False})
redis_client = Redis(
host='localhost',
port='6379',
password='your_redis_password'
)
def create_app():
app = Flask(__name__)
...........