I'm having some trouble with Mongodb and Python (Flask).
I have this api.py file, and I want all requests and responses to be in JSON, so I implement as such.
#
# Imports
#
from datetime import datetime
from flask import Flask
from flask import g
from flask import jsonify
from flask import json
from flask import request
from flask import url_for
from flask import redirect
from flask import render_template
from flask import make_response
import pymongo
from pymongo import Connection
from bson import BSON
from bson import json_util
#
# App Create
#
app = Flask(__name__)
app.config.from_object(__name__)
#
# Database
#
# connect
connection = Connection()
db = connection['storage']
units = db['storage']
#
# Request Mixins
#
#app.before_request
def before_request():
#before
return
#app.teardown_request
def teardown_request(exception):
#after
return
#
# Functions
#
def isInt(n):
try:
num = int(n)
return True
except ValueError:
return False
def isFloat(n):
try:
num = float(n)
return True
except ValueError:
return False
def jd(obj):
return json.dumps(obj, default=json_util.default)
def jl(obj):
return json.loads(obj, object_hook=json_util.object_hook)
#
# Response
#
def response(data={}, code=200):
resp = {
"code" : code,
"data" : data
}
response = make_response(jd(resp))
response.headers['Status Code'] = resp['code']
response.headers['Content-Type'] = "application/json"
return response
#
# REST API calls
#
# index
#app.route('/')
def index():
return response()
# search
#app.route('/search', methods=['POST'])
def search():
return response()
# add
#app.route('/add', methods=['POST'])
def add():
unit = request.json
_id = units.save(unit)
return response(_id)
# get
#app.route('/show', methods=['GET'])
def show():
import pdb; pdb.set_trace();
return response(db.units.find())
#
# Error handing
#
#app.errorhandler(404)
def page_not_found(error):
return response({},404)
#
# Run it!
#
if __name__ == '__main__':
app.debug = True
app.run()
The problem here is json encoding data coming to and from mongo. It seems I've been able to "hack" the add route by passing the request.json as the dictionary for save, so thats good... the problem is /show. This code does not work... When I do some logging I get
TypeError: <pymongo.cursor.Cursor object at 0x109bda150> is not JSON serializable
Any ideas? I also welcome any suggestions on the rest of the code, but the JSON is killing me.
Thanks in advance!
While #ErenGüven shows you a nice manual approach to solving this json serializing issue, pymongo comes with a utility to accomplish this for you. I use this in my own django mongodb project:
import json
from bson import json_util
json_docs = []
for doc in cursor:
json_doc = json.dumps(doc, default=json_util.default)
json_docs.append(json_doc)
Or simply:
json_docs = [json.dumps(doc, default=json_util.default) for doc in cursor]
And to get them back from json again:
docs = [json.loads(j_doc, object_hook=json_util.object_hook) for j_doc in json_docs]
The helper utilities tell json how to handle the custom mongodb objects.
When you pass db.units.find() to response you pass a pymongo.cursor.Cursor object to json.dumps ... and json.dumps doesn't know how to serialize it to JSON. Try getting the actual objects by iterating over the cursor to get its results:
[doc for doc in db.units.find()]
import json
from bson import json_util
docs_list = list(db.units.find())
return json.dumps(docs_list, default=json_util.default)
To encode MongoDB documents to JSON, I use a similar approach to the one below which covers bson.objectid.ObjectId and datetime.datetime types.
class CustomEncoder(json.JSONEncoder):
"""A C{json.JSONEncoder} subclass to encode documents that have fields of
type C{bson.objectid.ObjectId}, C{datetime.datetime}
"""
def default(self, obj):
if isinstance(obj, bson.objectid.ObjectId):
return str(obj)
elif isinstance(obj, datetime.datetime):
return obj.isoformat()
return json.JSONEncoder.default(self, obj)
enc = CustomEncoder()
enc.encode(doc)
As for the Cursor, you need to iterate it and get documents first.
Short answer: Its a cursor object. Iterate over it , and you get python dicts. Again to serialise do this:
import json
from bson import json_util
let's say this is my query:
details = mongo.db.details.find()
# this is cursor object
#iterate over to get a list of dicts
details_dicts = [doc for doc in details]
#serialize to json string
details_json_string = json.dumps(details_dicts,default=json_util.default)
If u wish to return the above , it will be just a string. Do this to return it as usable dict or json
return json.loads(details_json_string)
#return jsonified version rather than string without unwanted "\" in earlier json string!
Hope that helps! Happy Coding!
Yea plain old jsonify works, if your query doesn't contain some complex filed like Objectid , etc!
Just in case you are looking for a more plain python way of sending the response then create an empty list and append the values inside it and finally return this list in JSON format.
from flask import Flask, jsonify
#app.route('/view', methods=["GET'])
def viewFun():
data = units.find()
s = []
for i in data:
s.append(str(i))
return jsonify(s)
May be not the perfect one but it works!
if the document id is of no use, you can simply exclude it by
objects = collection.find({}, {'_id': False})
after that simply convert it to list of dictionaries by
list(objects)
Related
I get the following error when I try to access the 'data' variable from the endpoint '/'.
ValueError: [ValueError('dictionary update sequence element #0 has length 1; 2 is required'), TypeError('vars() argument must have __dict__ attribute')]
Code:
from fastapi import FastAPI
app = FastAPI()
data = {}
#app.on_event("startup")
def startup_event():
data[1] = [...] ...(numpy array)
data[2] = [...] ...(numpy array)
return data
#app.get("/")
def home():
return {'Data': data}
When I launch the endpoint I see 'Internal Server Error'. Nothing would display at the endpoint '/'. However, if I add this line -> 'print(data)' just above the return in home function for endpoint '/', it does print the values stored in the data dictionary, as specified in the startup function. How can I fix the issue, so that the data variable becomes visible when accessing the '/' endpoint?
FastAPI has no idea how it should decode numpy arrays to JSON; instead, do it explicitly yourself and then return the structure as native Python datatypes or as a raw JSON string. You can do this by using the custom Response objects in FastAPI.
return Response(content=numpy_as_json, media_type="application/json")
Or if you have it formatted as native datatypes:
return JSONResponse(content=numpy_as_native_datatypes)
You should convert (serialise) any numpy arrays into JSON before returning the data. For example:
data[1] = json.dumps(np.array([1, 2, 3, 4]).tolist())
data[2] = json.dumps(np.array([5, 6, 7, 8]).tolist())
Alternatively, you could serialise the whole dictionary object and return it in a custom Response, such as below:
json_data = json.dumps({k: v.tolist() for k, v in data.items()})
return Response(content=json_data, media_type="application/json")
or you could even use jsonable_encoder—which is used internally by FastAPI when returning a value, in order to convert objects that are not serializable into str—and then, return a JSONResponse, or a custom Response directly, which will return an application/json encoded response to the client, as described here.
from fastapi.responses import JSONResponse
from fastapi.encoders import jsonable_encoder
json_data = jsonable_encoder({k: v.tolist() for k, v in data.items()})
return JSONResponse(content=json_data)
I am trying to use tornado to do a simple get and post method. Quite new to tornado framework. For the post I would like to take in a json as input, use that input to feed into another function that I have to execute another part of code. However I can't get tornado post method to work even with a simple self.write().
For my get method I am reading from an SQL database to get the status of a sensor and write that in a json format. The get method works perfectly! When I go to localhost:port# it reads out my get json file. For my post method I would like to take in a simple json of just one key:value which is a float number. I want to take that float number that the user specified in the json and use it in my flowMKS.set() function that will change the setpoint parameter of the sensor. I am not sure how to input a json into the post method and read it into a variable. I have some #commented code below that I tried and didn't work. However I went back to the basics and just did a self.write("Hello World") to see if the post was working. I can't get self.write to work either. Keep getting a 500 error message when i go to localhost:port#/flow_post. The variable flow_status was used in my get method.
The intended result would be to take in a json {"setpoint":45.5} into the post method. Use the number and insert into my flowMKS method to change a parameter on the sensor.
How would you take in a json to a post method and take the number from the json input and store in a variable?
class Current(tornado.web.RequestHandler):
def get(self):
global flow_status
time = flow_status[0]
ip = flow_status[1]
rate = flow_status[2]
setp = flow_status[3]
tempc = flow_status[4]
status = {"flow_controller":{
"time":time,
"ip":ip,
"rate_sccm":rate,
"setpoint":setp,
"temperature_c":tempc,
}
}
self.write(status)
class Update(tornado.web.RequestHandler):
# def prepare(self):
# if self.request.haders["Content-Type"].startswith("application/json"):
# self.json_args = json.loads(self.request.body)
# else:
# self.json_args = None
def post(self):
# #expecting body data to contain JSON so we use json.loads to decrypt the JSON into a dict
# data = json.loads(self.request.body)
#
# #Getting what the setpoint should be
# setpoint = self.json_args["setpoint"]
#
# #making the input a float
# setpoint = float(setpoint)
#
# #setting up connection with sensor
# flowMKS = FlowController(flow_status[1])
#
# #sending setpoint to sensor
# flowMKS.set(setpoint)
self.write("Hello World")
if __name__ == '__main__':
# global flow_status
#Below is creating the Tornado based API for get and post methods
tornado.options.parse_command_line()
app = tornado.web.Application(
handlers=[(r'/',Current), (r'/flow_post', Update)])
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
#using PeriodicCallback to get info from the SQL database every 500 ms
PeriodicCallback(get_sql_status,500).start()
#starting the entire Tornado IOLoop
tornado.ioloop.IOLoop.current().start()
For uploading a file using Tornado you can use this function tornado.httputil.parse_body_arguments which will split the uploaded file content in a dictionary file_dict and other arguments in the FormData in the args_dict.
Sample code:
import tornado.httputil
import tornado.web
import tornado.escape
import json
import os
import sys
import traceback
class FileHandler(tornado.web.RequestHandler):
def _return_response(self, request, message_to_be_returned: dict, status_code):
"""
Returns formatted response back to client
"""
try:
request.set_header("Content-Type", "application/json; charset=UTF-8")
request.set_status(status_code)
#If dictionary is not empty then write the dictionary directly into
if(bool(message_to_be_returned)):
request.write(message_to_be_returned)
request.finish()
except Exception:
raise
def set_default_headers(self, *args, **kwargs):
self.set_header('Content-Type','text/csv')
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header("Access-Control-Allow-Methods", "*")
def post(self):
"""
This function reads an uploaded file
"""
try:
arg_dict = {}
file_dict = {}
tornado.httputil.parse_body_arguments(self.request.headers["Content-Type"], self.request.body, arg_dict, file_dict)
uploaded_file = file_dict['TestFile'][0]
if not uploaded_file:
return self._return_response(self, { 'message': 'No test file uploaded, please upload a test file' }, 400)
# File contents here
file_contents = str(uploaded_file['body'], "utf-8")
self.set_status(200)
self.finish()
except Exception as ex:
return self._return_response(self, { "message": 'Could not complete the request because of some error at the server!', "cause": ex.args[0], "stack_trace": traceback.format_exc(sys.exc_info()) }, 500)
You can alternatively use tornado.escape.json_decode to deserialize the request body into a dictionary and do something with it.
Sample code:
import tornado.gen
import tornado.web
import tornado.escape
import json
import os
import sys
import traceback
class JSONHandler(tornado.web.RequestHandler):
def _return_response(self, request, message_to_be_returned: dict, status_code):
"""
Returns formatted response back to client
"""
try:
request.set_header("Content-Type", "application/json; charset=UTF-8")
request.set_status(status_code)
#If dictionary is not empty then write the dictionary directly into
if(bool(message_to_be_returned)):
request.write(message_to_be_returned)
request.finish()
except Exception:
raise
def set_default_headers(self, *args, **kwargs):
self.set_header("Content-Type", "application/json")
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header("Access-Control-Allow-Methods", "*")
def post(self):
"""
This function parses the request body and does something
"""
try:
# Do something with request body
request_payload = tornado.escape.json_decode(self.request.body)
return self._return_response(self, request_payload, 200)
except json.decoder.JSONDecodeError:
return self._return_response(self, { "message": 'Cannot decode request body!' }, 400)
except Exception as ex:
return self._return_response(self, { "message": 'Could not complete the request because of some error at the server!', "cause": ex.args[0], "stack_trace": traceback.format_exc(sys.exc_info()) }, 500)
I am trying to retrieve data from url into json format using flask-restplus.
from flask import Flask, render_template
import requests
import json
from flask_restplus import Resource, Api, fields
from flask_sqlalchemy import SQLAlchemy
# config details
COLLECTION = 'indicators'
app = Flask(__name__)
api = Api(
app,
title='Akhil Jain',
description='Developing ' \
'a Flask-Restplus data service that allows a client to ' \
'read and store some publicly available economic indicator ' \
'data for countries around the world, and allow the consumers ' \
'to access the data through a REST API.'
)
#api.route('/indicators')
def get(self):
uri = 'http://api.worldbank.org/v2/indicators'
try:
res = requests.get(uri)
print(res)
return res.json()
except:
return False
if __name__ == '__main__':
app.run(debug=True)
but after trying out the GET, the response i am getting is False instead of the json data.
Could anyone tell me how to get the response data so that i can process it to sqllite db.
Thanks
you have to get the content from response
If you would want to save the xml data then.
try:
res = requests.get(uri)
print(res.content)
return res.content
except:
return False
If you want to save it as json, then install module xmltodict.
try:
res = requests.get(uri)
jsondata = xmltodict.parse(res.content)
print(jsondata)
return jsondata
except:
return False
You're trying to retrieve an XML producing endpoint. - Invoking json will raise an exception since the content retrieved is not json.
The ideal way of doing this is to build a function that parses the data as per your requirements; the example below can be treated as a reference.
import xml.etree.ElementTree as element_tree
xml = element_tree.fromstring(response.content)
for entry in xml.findall(path_here, namespace_here):
my_value = entry.find(attribute_here, namespace_here).text
You can opt to use lxml or other tools available. - The above is an example of how you can parse your xml response.
How can I test that the response a Flask view generated is JSON?
from flask import jsonify
#app.route('/')
def index():
return jsonify(message='hello world')
c = app.app.test_client()
assert c.get('/').status_code == 200
# assert is json
As of Flask 1.0, response.get_json() will parse the response data as JSON or raise an error.
response = c.get("/")
assert response.get_json()["message"] == "hello world"
jsonify sets the content type to application/json. Additionally, you can try parsing the response data as JSON. If it fails to parse, your test will fail.
from flask import json
assert response.content_type == 'application/json'
data = json.loads(response.get_data(as_text=True))
assert data['message'] == 'hello world'
Typically, this test on its own doesn't make sense. You know it's JSON since jsonify returned without error, and jsonify is already tested by Flask. If it was not valid JSON, you would have received an error while serializing the data.
There is a python-library for it.
import json
#...
def checkJson(s):
try:
json.decode(s)
return True
except json.JSONDecodeError:
return False
If you also want to check if it is a valid string, check the boundaries for "s.
You can read the help here on pythons website https://docs.python.org/3.5/library/json.html .
I have code within a Flask application that uses JSONs in the request, and I can get the JSON object like so:
Request = request.get_json()
This has been working fine, however I am trying to create unit tests using Python's unittest module and I'm having difficulty finding a way to send a JSON with the request.
response=self.app.post('/test_function',
data=json.dumps(dict(foo = 'bar')))
This gives me:
>>> request.get_data()
'{"foo": "bar"}'
>>> request.get_json()
None
Flask seems to have a JSON argument where you can set json=dict(foo='bar') within the post request, but I don't know how to do that with the unittest module.
Changing the post to
response=self.app.post('/test_function',
data=json.dumps(dict(foo='bar')),
content_type='application/json')
fixed it.
Thanks to user3012759.
Since Flask 1.0 release flask.testing.FlaskClient methods accepts json argument and Response.get_json method added, see pull request
with app.test_client() as c:
rv = c.post('/api/auth', json={
'username': 'flask', 'password': 'secret'
})
json_data = rv.get_json()
For Flask 0.x compatibility you may use receipt below:
from flask import Flask, Response as BaseResponse, json
from flask.testing import FlaskClient
class Response(BaseResponse):
def get_json(self):
return json.loads(self.data)
class TestClient(FlaskClient):
def open(self, *args, **kwargs):
if 'json' in kwargs:
kwargs['data'] = json.dumps(kwargs.pop('json'))
kwargs['content_type'] = 'application/json'
return super(TestClient, self).open(*args, **kwargs)
app = Flask(__name__)
app.response_class = Response
app.test_client_class = TestClient
app.testing = True