I'm trying to unit test my RESTful API. Here's my API:
class BaseHandler(tornado.web.RequestHandler):
def __init__(self, *args, **kwargs):
tornado.web.RequestHandler.__init__(self, *args, **kwargs)
self.log = self.application.log
self.db = self.application.db
class ProductHandler(BaseHandler):
#tornado.web.removeslash
def put(self, id = None, *args, **kwargs):
try:
self.log.info("Handling PUT request")
if not id:
raise Exception('Object Id Required')
id = { '_id' : id }
new_values = dict()
name = self.get_argument('name', None)
description = self.get_argument('description', None)
if name:
new_values['name'] = name
if description:
new_values['description'] = description
self.db.products.update(id, new_values, safe = True)
except:
self.log.error("".join(tb.format_exception(*sys.exc_info())))
raise
class Application(tornado.web.Application):
def __init__(self, config_path, test = False, *args, **kwargs):
handlers = [
(r"/product/?(.*)", ProductHandler),
]
settings = dict(debug=True)
tornado.web.Application.__init__(self, handlers, **settings)
self.log = logging.getLogger(__name__)
self.config = ConfigParser()
self.config.read(config_path)
self.mongo_connection = Connection(
host = self.config.get('mongo','host'),
port = self.config.getint('mongo','port'),
)
if test:
db_name = self.config.get('test', 'mongo.db')
else:
db_name = self.config.get('mongo', 'db')
self.log.debug("Using db: %s" % db_name)
self.db = self.mongo_connection[db_name]
But, here's my problem: the handler isn't seeing the name or description arguments. :(
Any suggestions?
As a work-around, I found them in the request.body and parsed the encoded parameters manually. It was kindof annoying, but it works.
new_values = urlparse.parse_qs(self.request.body)
# values show as lists with only one item
for k in new_values:
new_values[k] = new_values[k][0]
Say if you are using jQuery to send this PUT request:
$.ajax({
type: "PUT",
url: "/yourURL",
data: JSON.stringify({'json':'your json here'),
dataType: 'json'
})
The data should not be like:
data: {'json': 'your json here'}, because it will automatically be encoded into query string, which needs to be parsed by parse_qs
Then in Tornado
def put(self, pid):
d = json.loads(self.request.body)
print d
put handler will parse request.body, if request had proper content-type header (application/x-www-form-urlencoded), for example if you are using tornado http client:
headers = HTTPHeaders({'content-type': 'application/x-www-form-urlencoded'})
http_client.fetch(
HTTPRequest(url, 'PUT', body=urllib.urlencode(body), headers=headers))
Have you tried using a get method instead? Because depending on how you test your program, if you test it via your browser like Firefox or Chrome, they might be able to do it. Doing a HTTP PUT from a browser
If I were you I would write get instead of put. Cause then you can definitely test it in your browser.
For example, instead of:
def put ...
Try:
def get ...
Or Actually in your:
name = self.get_argument('name', None)
description = self.get_argument('description', None)
Why is the None there? According to the documentation:
RequestHandler.get_argument(name, default=[], strip=True)
...
If default is not provided, the argument is considered to be required,
and we throw an HTTP 400 exception if it is missing.
So in your case because you are not providing a proper default, therefore your app is returning HTTP 400. Miss out the default! (i.e.)
name = self.get_argument('name')
description = self.get_argument('description')
Related
Here is my code.
import webapp2
import json
from google.appengine.ext import ndb
class Email(ndb.Model):
email = ndb.StringProperty()
subscribed = ndb.BooleanProperty()
#staticmethod
def create(email):
ekey = ndb.Key("Email", email)
entity = Email.get_or_insert(ekey)
if entity.email: ###
# This email already exists
return None
entity.email = email
entity.subscribed = True
entity.put()
return entity
class Subscribe(webapp2.RequestHandler):
def post(self):
add = Email.create(self.request.get('email'))
success = add is not None
self.response.headers['Content-Type'] = 'application/json'
obj = {
'success': success
}
self.response.out.write(json.dumps(obj))
app = webapp2.WSGIApplication([
webapp2.Route(r'/newsletter/new', Subscribe),
], debug=True)
Here is my error.
File "/Users/nick/google-cloud-sdk/platform/google_appengine/google/appengine/ext/ndb/model.py", line 3524, in _get_or_insert_async
raise TypeError('name must be a string; received %r' % name) TypeError: name must be a string; received Key('Email', 'test#test.com')
What am I missing?
The error is caused by passing ekey (which is an ndb.Key) as arg to get_or_insert() (which expects a string):
ekey = ndb.Key("Email", email)
entity = Email.get_or_insert(ekey)
Since it appears you want to use the user's email as a unique key ID you should directly pass the email string to get_or_insert():
entity = Email.get_or_insert(email)
I've got a cherrypy app and I'm trying to change response header Content-type. I'm trying to do that with cherrypy.response.header['Content-Type'] = 'text/plain'. Unfortunately I'm still getting 'text/html'. I want to have set one content type for ok request and another content type for error message. The only way how can I change content type is with my decorator. But this set type for the method and I need to change it. Do you know where could be a problem?
My config:
config = {
'/': {
'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
'tools.response_headers.on': True,
'tools.response_headers.headers': [('Content-Type', 'text/html')],
}
}
def GET(self, id):
cherrypy.response.headers['Content-Type'] = 'application/x-download'
somecode
if res < None:
cherrypy.response.headers['Content-Type'] = 'text/plain'
cherrypy.response.status=404
GET._cp_config = {'response.stream': True}
def stream():
def decorate(func):
def wrapper(*args, **kwargs):
name = time.strftime("%Y%m%d-%H%M%S")
cherrypy.response.headers['Content-Type'] = 'application/x-download'
cherrypy.response.headers['Content-Disposition'] = 'attachment; filename="' + name + '.txt"'
cherrypy.response.headers['Transfer-Encoding'] = 'chunked'
return func(*args, **kwargs)
return wrapper
return decorate
#cherrypy.expose
class Network:
#stream()
def GET(self, id):
source = my_generator()
for i in source:
if res < None:
cherrypy.response.headers['Content-Type'] = 'text/plain'
cherrypy.response.status=404
break
yield bytes(i)
GET._cp_config = {'response.stream': True}
Ok, there is more complex code, config of cherrypy is in the previous comment. I have a generator, which yields me some data and this is streaming that data in file to the client. I know, there are definitely better solutions. Imagine that in res variable there is result from saving to db. The problem is, that this completely ignores my settings in the if condition. It's still returning a file (empty one). The decorator is the only way how to set content type, that's why it's there
I am working off of a Miguel Grinberg tutorial on social authentication.
On the homepage template I have this code, and I removed the twitter portion from the tutorial:
<h2>I don't know you!</h2>
<p>Login with Facebook</p>
{% endif %}
So when you click that link, you pass Facebook as the provider through this view function:
#app.route('/authorize/<provider>')
def oauth_authorize(provider):
if not current_user.is_anonymous():
return redirect(url_for('index'))
oauth = OAuthSignIn.get_provider(provider)
return oauth.authorize()
Now, in a different file, oauth.py, I have the following and my issue is this. I keep getting an error when I click the Facebook link UNLESS the TwitterSignIn class is removed. I guess I am curious as to why the TwitterSignIn class needs to be removed for this to work, because no data is being passed to it, right? Even if Facebook wasn't the only option, why would clicking the Facebook sign-in link pass any data to the TwitterSignIn class?
from rauth import OAuth1Service, OAuth2Service
from flask import current_app, url_for, request, redirect, session
class OAuthSignIn(object):
providers = None
def __init__(self, provider_name):
self.provider_name = provider_name
credentials = current_app.config['OAUTH_CREDENTIALS'][provider_name]
self.consumer_id = credentials['id']
self.consumer_secret = credentials['secret']
def authorize(self):
pass
def callback(self):
pass
def get_callback_url(self):
return url_for('oauth_callback', provider=self.provider_name,
_external=True)
#classmethod
def get_provider(self, provider_name):
if self.providers is None:
self.providers = {}
for provider_class in self.__subclasses__():
provider = provider_class()
self.providers[provider.provider_name] = provider
return self.providers[provider_name]
class FacebookSignIn(OAuthSignIn):
def __init__(self):
super(FacebookSignIn, self).__init__('facebook')
self.service = OAuth2Service(
name='facebook',
client_id=self.consumer_id,
client_secret=self.consumer_secret,
authorize_url='https://graph.facebook.com/oauth/authorize',
access_token_url='https://graph.facebook.com/oauth/access_token',
base_url='https://graph.facebook.com/'
)
def authorize(self):
return redirect(self.service.get_authorize_url(
scope='email',
response_type='code',
redirect_uri=self.get_callback_url())
)
def callback(self):
if 'code' not in request.args:
return None, None, None
oauth_session = self.service.get_auth_session(
data={'code': request.args['code'],
'grant_type': 'authorization_code',
'redirect_uri': self.get_callback_url()}
)
me = oauth_session.get('me').json()
return (
'facebook$' + me['id'],
me.get('email').split('#')[0], # Facebook does not provide
# username, so the email's user
# is used instead
me.get('email')
)
class TwitterSignIn(OAuthSignIn):
def __init__(self):
super(TwitterSignIn, self).__init__('twitter')
self.service = OAuth1Service(
name='twitter',
consumer_key=self.consumer_id,
consumer_secret=self.consumer_secret,
request_token_url='https://api.twitter.com/oauth/request_token',
authorize_url='https://api.twitter.com/oauth/authorize',
access_token_url='https://api.twitter.com/oauth/access_token',
base_url='https://api.twitter.com/1.1/'
)
def authorize(self):
request_token = self.service.get_request_token(
params={'oauth_callback': self.get_callback_url()}
)
session['request_token'] = request_token
return redirect(self.service.get_authorize_url(request_token[0]))
def callback(self):
request_token = session.pop('request_token')
if 'oauth_verifier' not in request.args:
return None, None, None
oauth_session = self.service.get_auth_session(
request_token[0],
request_token[1],
data={'oauth_verifier': request.args['oauth_verifier']}
)
me = oauth_session.get('account/verify_credentials.json').json()
social_id = 'twitter$' + str(me.get('id'))
username = me.get('screen_name')
return social_id, username, None # Twitter does not provide email
Some additional information-
The specific error is this:
File "/Users/metersky/code/mylastapt/app/oauth.py", line 29, in get_provider
provider = provider_class()
File "/Users/metersky/code/mylastapt/app/oauth.py", line 73, in __init__
super(TwitterSignIn, self).__init__('twitter')
File "/Users/metersky/code/mylastapt/app/oauth.py", line 10, in __init__
credentials = current_app.config['OAUTH_CREDENTIALS'][provider_name]
KeyError: 'twitter'
And this is where the I think the issue might be happening:
app.config['OAUTH_CREDENTIALS'] = {
'facebook': {
'id': 'XXX',
'secret': 'XXXX'
}
}
The problem is in OAuthSignIn.get_provider.
#classmethod
def get_provider(self, provider_name):
if self.providers is None:
self.providers = {}
for provider_class in self.__subclasses__():
provider = provider_class()
self.providers[provider.provider_name] = provider
return self.providers[provider_name]
The first time you call it from within your view
oauth = OAuthSignIn.get_provider(provider)
the method caches the providers you've defined. It does this by checking for all of OAuthSignIn's subclasses.
for provider_class in self.__subclasses__():
When you include TwitterSignIn, it will be included as a subclass. You'll then instantiate an instance of the class
provider = provider_class()
Inside OAuthSignIn.__init__, you load the provider's settings with current_app.config['OAUTH_CREDENTIALS'][provider_name]. Since Twitter isn't included, you get the KeyError.
If you don't want to support Twitter, just remove the class. If you want to protect your application a little more so that providers can be removed from your settings without updating code, you'll need to check for the exception. You could do the check inside OAuthSignIn.__init__, but there probably isn't much value to including an unsupported provider in OAuthSignIn.providers. You're better off putting the check in OAuthSignIn.get_provider.
#classmethod
def get_provider(cls, provider_name):
if cls.providers is None:
cls.providers = {}
for provider_class in cls.__subclassess__():
try:
provider = provider_class()
except KeyError:
pass # unsupported provider
else:
cls.providers[provider.provider_name] = provider
return cls.providers[provider_name]
In my function I read user's data from session and store them in a dictionary. Next I'm sending it to 'register' function from registration.backend but the function somehow get's it empty and a KeyError is thrown. Where are my data gone ? The code from function calling 'register' function :
data = request.session['temp_data']
email = data['email']
logging.debug(email)
password1 = data['password1']
userdata = {'email': email, 'password1': password1}
logging.debug(userdata)
backend = request.session['backend']
logging.debug(backend)
user = backend.register(userdata)
And the register function (whole source here : http://bitbucket.org/ubernostrum/django-registration/src/tip/registration/backends/default/init.py ) :
class DefaultBackend(object):
def register(self, request, **kwargs):
logging.debug("backend.register")
logging.debug(kwargs)
username, email, password = kwargs['email'], kwargs['email'], kwargs['password1']
Debug after invoking them :
2010-07-09 19:24:35,020 DEBUG my#email.com
2010-07-09 19:24:35,020 DEBUG {'password1': u'a', 'email': u'my#email.com'}
2010-07-09 19:24:35,020 DEBUG <registration.backends.default.DefaultBackend object at 0x15c6090>
2010-07-09 19:24:35,021 DEBUG backend.register
2010-07-09 19:24:35,021 DEBUG {}
Why the data could be missing ? Am I doing something wrong ?
#edit for Silent-Ghost
register() takes exactly 2 arguments (3 given)
112. backend = request.session['backend']
113. logging.debug(backend)
114. user = backend.register(request, userdata)
No need to mess with ** in register method. What you want to do is simply pass dictionary to register method:
user = backend.register( request, userdata ) # you need to pass request as definition says
def register( self, request, userdata ): # note lack of **
logging.debug("backend.register")
logging.debug( userdata ) # should work as expected
username, email, password = userdata['email'], userdata['email'], userdata['password1']
Judging by the method's signature:
you need to unpack your dictionary
you need to pass relevant request variable
Something like this:
backend.register(request, **userdata)
Assuming register is a method on backend instance.
this perfectly work
class Logging():
def debug(self,f):
print f
class DefaultBackend(object):
def register(self, request, **kwargs):
logging.debug("backend.register")
logging.debug(kwargs)
username, email, password = kwargs['email'], kwargs['email'], kwargs['password1']
class Request:
def __init__(self):
self.session = {}
request = Request()
logging=Logging()
request.session['temp_data']={'password1': u'a', 'email': u'my#email.com'}
request.session['backend']=DefaultBackend()
data = request.session['temp_data']
email = data['email']
logging.debug(email)
password1 = data['password1']
userdata = {'email': email, 'password1': password1}
logging.debug(userdata)
backend = request.session['backend']
logging.debug(backend)
user = backend.register(request,**userdata)
Is there a better pattern for input validation than I'm using in this function?
https://github.com/nathancahill/clearbit-intercom/blob/133e4df0cfd1a146cedb3c749fc1b4fac85a6e1b/server.py#L71
Here's the same function without any validation. It's much more readable, it's short and to the point (9 LoC vs 53 LoC).
def webhook(clearbitkey, appid, intercomkey):
event = request.get_json()
id = event['data']['item']['id']
email = event['data']['item']['email']
person = requests.get(CLEARBIT_USER_ENDPOINT.format(email=email), auth=(clearbitkey, '')).json()
domain = person['employment']['domain']
company = requests.get(CLEARBIT_COMPANY_ENDPOINT.format(domain=domain), auth=(clearbitkey, '')).json()
note = create_note(person, company)
res = requests.post(INTERCOM_ENDPOINT,
json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey))
return jsonify(note=res.json())
However, it doesn't handle any of these errors:
dict KeyError's (especially nested dicts)
HTTP errors
Invalid JSON
Unexpected responses
Is there a better pattern to follow? I looked into using a data validation library like voluptous but it seems like I'd still have the same problem of verbosity.
Your original code on github seems fine to me. It's a little over complicated, but also handle all cases of error. You can try to improve readability by abstract things.
Just for demonstration, I may write code like this:
class ValidationError(Exception):
"Raises when data validation fails"
pass
class CallExternalApiError(Exception):
"Raises when calling external api fails"
pass
def get_user_from_event(event):
"""Get user profile from event
:param dict event: request.get_json() result
:returns: A dict of user profile
"""
try:
event_type = event['data']['item']['type']
except KeyError:
raise ValidationError('Unexpected JSON format.')
if event_type != 'user':
return ValidationError('Event type is not supported.')
try:
id = event['data']['item']['id']
email = event['data']['item']['email']
except KeyError:
return ValidationError('User object missing fields.')
return {'id': id, 'email': email}
def call_json_api(request_function, api_name, *args, **kwargs):
"""An simple wrapper for sending request
:param request_function: function used for sending request
:param str api_name: name for this api call
"""
try:
res = request_function(*args, **kwargs)
except:
raise CallExternalApiError('API call failed to %s.' % api_name)
try:
return res.json()
except:
raise CallExternalApiError('Invalid response from %s.' % api_name)
#app.route('/<clearbitkey>+<appid>:<intercomkey>', methods=['POST'])
def webhook(clearbitkey, appid, intercomkey):
"""
Webhook endpoint for Intercom.io events. Uses this format for Clearbit and
Intercom.io keys:
/<clearbitkey>+<appid>:<intercomkey>
:clearbitkey: Clearbit API key.
:appid: Intercom.io app id.
:intercomkey: Intercom.io API key.
Supports User events, specifically designed for the User Created event.
Adds a note to the user with their employment and company metrics.
"""
event = request.get_json()
try:
return handle_event(event, clearbitkey, appid, intercomkey)
except (ValidationError, CallExternalApiError) as e:
# TODO: include **res_objs in response
return jsonify(error=str(e))
def handle_event(event):
"""Handle the incoming event
"""
user = get_user_from_event(event)
res_objs = dict(event=event)
person = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_USER_ENDPOINT.format(email=user['email']),
auth=(clearbitkey, '')
)
res_objs['person'] = person
if 'error' in person:
raise CallExternalApiError('Error response from Clearbit.')
domain = person['employment']['domain']
company = None
if domain:
try:
company = call_json_api(
requests.get,
'Clearbit',
CLEARBIT_COMPANY_ENDPOINT.format(domain=domain),
auth=(clearbitkey, ''))
)
if 'error' in company:
company = None
except:
company = None
res_objs['company'] = company
try:
note = create_note(person, company)
except:
return jsonify(error='Failed to generate note for user.', **res_objs)
result = call_json_api(
requests.post,
'Intercom',
(INTERCOM_ENDPOINT, json=dict(user=dict(id=id), body=note),
headers=dict(accept='application/json'),
auth=(appid, intercomkey)
)
return jsonify(note=result, **res_objs)
I hope it helps.