What is the proper way to handle response classes in Flask-RESTplus?
I am experimenting with a simple GET request seen below:
i_throughput = api.model('Throughput', {
'date': fields.String,
'value': fields.String
})
i_server = api.model('Server', {
'sessionId': fields.String,
'throughput': fields.Nested(i_throughput)
})
#api.route('/servers')
class Server(Resource):
#api.marshal_with(i_server)
def get(self):
servers = mongo.db.servers.find()
data = []
for x in servers:
data.append(x)
return data
I want to return my data in as part of a response object that looks like this:
{
status: // some boolean value
message: // some custom response message
error: // if there is an error store it here
trace: // if there is some stack trace dump throw it in here
data: // what was retrieved from DB
}
I am new to Python in general and new to Flask/Flask-RESTplus. There is a lot of tutorials out there and information. One of my biggest problems is that I'm not sure what to exactly search for to get the information I need. Also how does this work with marshalling? If anyone can post good documentation or examples of excellent API's, it would be greatly appreciated.
https://blog.miguelgrinberg.com/post/customizing-the-flask-response-class
from flask import Flask, Response, jsonify
app = Flask(__name__)
class CustomResponse(Response):
#classmethod
def force_type(cls, rv, environ=None):
if isinstance(rv, dict):
rv = jsonify(rv)
return super(MyResponse, cls).force_type(rv, environ)
app.response_class = CustomResponse
#app.route('/hello', methods=['GET', 'POST'])
def hello():
return {'status': 200, 'message': 'custom_message',
'error': 'error_message', 'trace': 'trace_message',
'data': 'input_data'}
result
import requests
response = requests.get('http://localhost:5000/hello')
print(response.text)
{
"data": "input_data",
"error": "error_message",
"message": "custom_message",
"status": 200,
"trace": "trace_message"
}
I am having a problem understanding how mock works and how to write unittests with mock objects. I wanted to mock an external api call every time when my model calls save() method.
My code:
models.py
from . import utils
class Book(Titleable, Isactiveable, Timestampable, IsVoidable, models.Model):
title
orig_author
orig_title
isbn
def save(self, *args, **kwargs):
if self.isbn:
google_data = utils.get_original_title_and_name(self.isbn)
if google_data:
self.original_author = google_data['author']
self.original_title = google_data['title']
super().save(*args, **kwargs)
utils.py
def get_original_title_and_name(isbn, **kawargs):
isbn_search_string = 'isbn:{}'.format(isbn)
payload = {
'key': GOOGLE_API_KEY,
'q': isbn_search_string,
'printType': 'books',
}
r = requests.get(GOOGLE_API_URL, params=payload)
response = r.json()
if 'items' in response.keys():
title = response['items'][THE_FIRST_INDEX]['volumeInfo']['title']
author = response['items'][THE_FIRST_INDEX]['volumeInfo']['authors'][THE_FIRST_INDEX]
return {
'title': title,
'author': author
}
else:
return None
I began read docs and write test:
test.py:
from unittest import mock
from django.test import TestCase
from rest_framework import status
from .constants import THE_FIRST_INDEX, GOOGLE_API_URL, GOOGLE_API_KEY
class BookModelTestCase(TestCase):
#mock.patch('requests.get')
def test_get_original_title_and_name_from_google_api(self, mock_get):
# Define new Mock object
mock_response = mock.Mock()
# Define response data from Google API
expected_dict = {
'kind': 'books#volumes',
'totalItems': 1,
'items': [
{
'kind': 'books#volume',
'id': 'IHxXBAAAQBAJ',
'etag': 'B3N9X8vAMWg',
'selfLink': 'https://www.googleapis.com/books/v1/volumes/IHxXBAAAQBAJ',
'volumeInfo': {
'title': "Alice's Adventures in Wonderland",
'authors': [
'Lewis Carroll'
]
}
}
]
}
# Define response data for my Mock object
mock_response.json.return_value = expected_dict
mock_response.status_code = 200
# Define response for the fake API
mock_get.return_value = mock_response
The first of all, I can't write target for the #mock.patch correct. If a define target as utuls.get_original_title_and_name.requests.get, I get ModuleNotFoundError. Also I can't understand how to make fake-call to external API and verify recieved data (whether necessarly its, if I've already define mock_response.json.return_value = expected_dict?) and verify that my save() method work well?
How do I write test for this cases? Could anyone explain me this case?
You should mock the direct collaborators of the code under test. For Book that would be utils. For utils that would be requests.
So for the BookModelTestCase:
class BookModelTestCase(TestCase):
#mock.patch('app.models.utils')
def test_save_book_calls_google_api(self, mock_utils):
mock_utils.get_original_title_and_name.return_value = {
'title': 'Google title',
'author': 'Google author'
}
book = Book(
title='Some title',
isbn='12345'
)
book.save()
self.assertEqual(book.title, 'Google title')
self.assertEqual(book.author, 'Google author')
mock_utils.get_original_title_and_name.assert_called_once_with('12345')
And then you can create a separate test case to test get_original_title_and_name:
class GetOriginalTitleAndNameTestCase(TestCase):
#mock.patch('app.utils.requests.get')
def test_get_original_title_and_name_from_google_api(self, mock_get):
mock_response = mock.Mock()
# Define response data from Google API
expected_dict = {
'kind': 'books#volumes',
'totalItems': 1,
'items': [
{
'kind': 'books#volume',
'id': 'IHxXBAAAQBAJ',
'etag': 'B3N9X8vAMWg',
'selfLink': 'https://www.googleapis.com/books/v1/volumes/IHxXBAAAQBAJ',
'volumeInfo': {
'title': "Alice's Adventures in Wonderland",
'authors': [
'Lewis Carroll'
]
}
}
]
}
# Define response data for my Mock object
mock_response.json.return_value = expected_dict
mock_response.status_code = 200
# Define response for the fake API
mock_get.return_value = mock_response
# Call the function
result = get_original_title_and_name(12345)
self.assertEqual(result, {
'title': "Alice's Adventures in Wonderland",
'author': 'Lewis Carroll'
})
mock_get.assert_called_once_with(GOOGLE_API_URL, params={
'key': GOOGLE_API_KEY,
'q': 'isbn:12345',
'printType': 'books',
})
There is a Flask-Appbuilder app with a custom SecurityManager that looks up the user token that it gets from a browser. We get the client credentials on the start of the app. And it works fine till the moment the credentials rotate.
Is there an extension point where I can implement requesting customer_id and customer_secret from an external resource?
SecurityManager implementation:
class MySecurityManager(SecurityManager):
TOKENINFO_URL = "..."
USERINFO_URL = ".../{}"
def __init__(self, appbuilder):
super(MySecurityManager, self).__init__(appbuilder)
def get_oauth_user_info(self, provider, resp=None):
"""
We authenticate users against Our OAuth provider
"""
if provider == 'MyProvider':
tokeninfo = self.appbuilder.sm.oauth_remotes[provider].get(self.TOKENINFO_URL)
uid = tokeninfo.data.get('uid')
user = self.appbuilder.sm.oauth_remotes[provider].get(self.USERINFO_URL.format(uid))
log.debug("Token info: {0}".format(tokeninfo.data))
log.debug("User info: {0}".format(user.data))
return {'username': tokeninfo.data.get('uid', ''),
'email': user.data.get('email', ''),
'first_name': user.data.get('name', '').split(" ")[0],
'last_name': user.data.get('name', '').split(" ")[-1]}
else:
return super(MySecurityManager, self).get_oauth_user_info(provider, resp=None)
config.py:
OAUTH_PROVIDERS = [
{
'name': 'MyProvider',
'icon': ...,
'token_key': ...,
'remote_app': {
'base_url': ...,
'consumer_key': SUPERSET_OAUTH_CONSUMER_KEY,
'consumer_secret': SUPERSET_OAUTH_CONSUMER_SECRET,
'request_token_params': {
'scope': ...,
},
'request_token_url': ...,
'access_token_url': ...,
'authorize_url': ...,
}
}
]
I solved it overriding the get oauth_providers form https://github.com/dpgaspar/Flask-AppBuilder/blob/master/flask_appbuilder/security/manager.py#L306.
Here an example:
#property
def oauth_providers(self):
providers = self.appbuilder.get_app.config['OAUTH_PROVIDERS']
for provider in providers:
if provider['name'] == 'XXXX':
# rotate logic here
provider['remote_app']['consumer_key'] = xxxxx
provider['remote_app']['consumer_secret'] = xxxx
return providers
Whenever I execute an update statement using a session with SqlAlchemy and then call commit(), it will rarely update the database.
Here is my environment:
I have two servers running. One is for my database the other is for my python server.
Database Server:
Postgres v9.6 - On Amazon's RDS
Server with Python
Linux 3.13.0-65-generic x86_64 - On an Amazon EC2 Instance
SqlAlchemy v1.1.5
Python v3.4.3
Flask 0.11.1
Also, I use pgAdmin 4 for querying my table.
The files of importance:
server/models/category.py
from sqlalchemy.orm import backref
from .. import db
from flask import jsonify
class Category(db.Model):
__tablename__ = "categories"
id = db.Column(db.Integer, primary_key=True)
cat_name = db.Column(db.String(80))
includes = db.Column(db.ARRAY(db.String), default=[])
excludes = db.Column(db.ARRAY(db.String), default=[])
parent_id = db.Column(db.ForeignKey('categories.id', ondelete='SET NULL'), nullable=True, default=None)
subcategories = db.relationship('Category', backref=backref(
'categories',
remote_side=[id],
single_parent=True,
cascade="all, delete-orphan"
))
assigned_user = db.Column(db.String(80), nullable=True, default=None)
def to_dict(self):
return dict(
id=self.id,
cat_name=self.cat_name,
parent_id=self.parent_id,
includes=self.includes,
excludes=self.excludes,
assigned_user=self.assigned_user,
)
def json(self):
return jsonify(self.to_dict())
def __repr__(self):
return "<%s %r>" % (self.__class__, self.to_dict())
class CategoryOperations:
...
#staticmethod
def update_category(category):
return """
UPDATE categories
SET cat_name='{0}',
parent_id={1},
includes='{2}',
excludes='{3}',
assigned_user={4}
WHERE id={5}
RETURNING cat_name, parent_id, includes, excludes, assigned_user
""".format(
category.cat_name,
category.parent_id if category.parent_id is not None else 'null',
"{" + ",".join(category.includes) + "}",
"{" + ",".join(category.excludes) + "}",
"'" + category.assigned_user + "'" if category.assigned_user is not None else 'null',
category.id
)
#staticmethod
def update(category, session):
print("Updating category with id: " + str(category.id))
stmt = CategoryOperations.update_category(category)
print(stmt)
row_updated = session.execute(stmt).fetchone()
return Category(
id=category.id,
cat_name=row_updated[0],
parent_id=row_updated[1],
includes=row_updated[2],
excludes=row_updated[3],
assigned_user=row_updated[4]
)
...
server/api/category.py
from flask import jsonify, request
import json
from .api_utils.utils import valid_request as is_valid_request
from . import api
from ..models.category import Category, CategoryOperations
from ..models.users_categories import UsersCategoriesOperations, UsersCategories
from ..models.listener_item import ListenerItemOperations, ListenerItem
from ..models.user import UserOperations
from ..schemas.category import category_schema
from .. import get_session
...
#api.route('/categories/<int:id>', methods=['PUT'])
def update_category(id):
category_json = request.json
if category_json is None:
return "Bad Request: Request not sent as json", 400
valid_json, json_err = is_valid_request(category_json, ['cat_name', 'parent_id', 'includes', 'excludes', 'assigned_user'], "and")
if not valid_json:
return json_err, 400
category = Category(
id=id,
cat_name=category_json['cat_name'],
parent_id=category_json['parent_id'],
includes=category_json['includes'],
excludes=category_json['excludes'],
assigned_user=category_json['assigned_user'],
)
session = get_session()
try:
updated_category = CategoryOperations.update(category, session)
session.commit()
print(updated_category.to_dict())
return jsonify(updated_category.to_dict()), 200
except Exception as e:
print("ROLLBACK")
print(e)
session.rollback()
return str(e), 500
...
There is one more file that will probably be useful in this case:
server/__init__.py
import sqlalchemy as sa
from flask import Flask
from flask_marshmallow import Marshmallow
from flask_sqlalchemy import SQLAlchemy
from config import config
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from flask_cors import CORS, cross_origin
from .db_config import CONFIG
db = SQLAlchemy()
ma = Marshmallow()
Engine = sa.create_engine(
CONFIG.POSTGRES_URL,
client_encoding='utf8',
pool_size=20,
max_overflow=0
)
Session = sessionmaker(bind=Engine)
conn = Engine.connect()
def get_session():
return Session(bind=conn)
def create_app(config_name):
app = Flask(__name__, static_url_path="/app", static_folder="static")
app_config = config[config_name]()
print(app_config)
app.config.from_object(app_config)
from .api import api as api_blueprint
app.register_blueprint(api_blueprint, url_prefix='/api')
from .api.routes import routes
routes(app)
from .auth import authentication
authentication(app)
db.init_app(app)
ma.init_app(app)
CORS(app)
...
return app
To explain a little more with the environment and files I have given, let's say I have a row in my categories table like so:
{
"assigned_user": null,
"cat_name": "Category Name Before",
"excludes": [
"exclude1",
"excludeBefore"
],
"id": 2,
"includes": [
"include1",
"include2"
],
"parent_id": null
}
When I do a PUT request to /api/categories/2 with the body as:
{
"assigned_user": null,
"cat_name": "Category Name 1",
"excludes": [
"exclude1",
"exclude2"
],
"id": 2,
"includes": [
"include1",
"include2"
],
"parent_id": null
}
During the request, I print out the SQL Statement that my PUT request created (for testing) and I get this:
UPDATE categories
SET cat_name='Category Name 1',
parent_id=null,
includes='{include1,include2}',
excludes='{exclude1,exclude2}',
assigned_user=null
WHERE id=2
RETURNING cat_name, parent_id, includes, excludes, assigned_user
After it commits the UPDATE statement, it then returns the response. I get the updated object back like so:
{
"assigned_user": null,
"cat_name": "Category Name 1",
"excludes": [
"exclude1",
"exclude2"
],
"id": 2,
"includes": [
"include1",
"include2"
],
"parent_id": null
}
When I do a GET request with this URL: /api/categories/2 and I get the same object too like so:
{
"assigned_user": null,
"cat_name": "Category Name 1",
"excludes": [
"exclude1",
"exclude2"
],
"id": 2,
"includes": [
"include1",
"include2"
],
"parent_id": null
}
However, when I run the SQL command below in pgAdmin, I get the old version (it didn't update the row in the database):
SELECT * FROM categories WHERE id=2
Here is the object I get:
{
"assigned_user": null,
"cat_name": "Category Name Before",
"excludes": [
"exclude1",
"excludeBefore"
],
"id": 2,
"includes": [
"include1",
"include2"
],
"parent_id": null
}
This is the object I had before doing the PUT request. If I restart my python server and do the GET request, then I get the old object. It feels like in the session, it is storing the data, but for some reason it's not propagating to the database.
It might be good to know that if I run the update command in pgAdmin, it updates the row just fine.
UPDATE: I have also used these methods (as talked about here) to update, but still the same problem:
# using the session to update
session.query(Category).filter_by(id=category.id).update({
"cat_name": category.id,
"assigned_user": category.assigned_user,
"includes": category.includes,
"excludes": category.excludes,
"parent_id": category.parent_id
})
# using the category object to edit, then commit
category_from_db = session.query(Category).filter_by(id=category.id).first()
category_from_db.cat_name = category_json['cat_name']
category_from_db.assigned_user = category_json['assigned_user']
category_from_db.excludes = category_json['excludes']
category_from_db.includes = category_json['includes']
category_from_db.parent_id = category_json['parent_id']
session.commit()
Any ideas?
It turns out that each time I called get_session, I was creating a new session. And I was not closing the session after every HTTP request.
Here is what the server/api/category.py PUT request looks like:
#api.route('/categories/<int:id>', methods=['PUT'])
def update_category(id):
category_json = request.json
if category_json is None:
return "Bad Request: Request not sent as json", 400
valid_json, json_err = is_valid_request(category_json, ['cat_name', 'parent_id', 'includes', 'excludes', 'assigned_user'], "and")
if not valid_json:
return json_err, 400
category = Category(
id=id,
cat_name=category_json['cat_name'],
parent_id=category_json['parent_id'],
includes=category_json['includes'],
excludes=category_json['excludes'],
assigned_user=category_json['assigned_user'],
)
session = get_session()
try:
updated_category = CategoryOperations.update(category, session)
session.commit()
print(updated_category.to_dict())
return jsonify(updated_category.to_dict()), 200
except Exception as e:
print("ROLLBACK")
print(e)
session.rollback()
return str(e), 500
finally: #
session.close() # <== The fix
Once I closed every session I opened after I was done with it, the problem was solved.
Hope this helps someone.
On production, a soon as I open a channel with the javascript, it disconnects a seccond after.
Everything works super fine on devserver. The callback works on the server but not on the client. We are using flask, backbone, requirejs and sourcemap.
Client code:
window.channel = new goog.appengine.Channel(window.PLAY_SETTINGS.CHANNEL_TOKEN);
window.gae_websocket = window.channel.open({
onopen: function() {
return console.log('onopen');
},
onclose: function() {
return console.log('onclose');
},
onerror: function() {
return console.log('onerror');
},
onmessage: function() {
return console.log('onmessage');
}
});
Server code:
class Connection(ndb.Model):
user_key = ndb.KeyProperty()
scope = ndb.IntegerProperty(indexed=True, choices=range(0, 2))
target_key = ndb.KeyProperty(indexed=True) # Event ou debate
channel_id = ndb.StringProperty(indexed=True)
#staticmethod
def open_channel():
channel_id = str(uuid4())
channel_token = channel.create_channel(client_id=channel_id, duration_minutes=480)
return channel_token, channel_id
Logs from the appengine production console.
The client callbacks (js) dont works. These are the server callbacks that create the logs:
#app.route('/_ah/channel/disconnected/', methods=['POST'])
def channel_disconnection():
client_id = request.form.get('from')
ndb.delete_multi(Connection.query(Connection.channel_id == client_id).fetch(keys_only=True))
logging.info("Channel closed : %s" % client_id)
return make_response('ok', '200')
#app.route('/_ah/channel/connected/', methods=['POST'])
def channel_connection():
client_id = request.form.get('from')
logging.info("Channel open : %s" % client_id)
return make_response('ok', '200')