I have a custom HTTP request handler that can be simplified to something like this:
# Python 3:
from http import server
class MyHandler(server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
# Here's where all the complicated logic is done to generate HTML.
# For clarity here, replace with a simple stand-in:
html = "<html><p>hello world</p></html>"
self.wfile.write(html.encode())
I'd like to unit-test this handler (i.e. make sure that my do_GET executes without an exception) without actually starting a web server. Is there any lightweight way to mock the SimpleHTTPServer so that I can test this code?
Expanding on the answer from jakevdp, I managed to be able to check the output, too:
try:
import unittest2 as unittest
except ImportError:
import unittest
try:
from io import BytesIO as IO
except ImportError:
from StringIO import StringIO as IO
from server import MyHandlerSSL # My BaseHTTPRequestHandler child
class TestableHandler(MyHandlerSSL):
# On Python3, in socketserver.StreamRequestHandler, if this is
# set it will use makefile() to produce the output stream. Otherwise,
# it will use socketserver._SocketWriter, and we won't be able to get
# to the data
wbufsize = 1
def finish(self):
# Do not close self.wfile, so we can read its value
self.wfile.flush()
self.rfile.close()
def date_time_string(self, timestamp=None):
""" Mocked date time string """
return 'DATETIME'
def version_string(self):
""" mock the server id """
return 'BaseHTTP/x.x Python/x.x.x'
class MockSocket(object):
def getsockname(self):
return ('sockname',)
class MockRequest(object):
_sock = MockSocket()
def __init__(self, path):
self._path = path
def makefile(self, *args, **kwargs):
if args[0] == 'rb':
return IO(b"GET %s HTTP/1.0" % self._path)
elif args[0] == 'wb':
return IO(b'')
else:
raise ValueError("Unknown file type to make", args, kwargs)
class HTTPRequestHandlerTestCase(unittest.TestCase):
maxDiff = None
def _test(self, request):
handler = TestableHandler(request, (0, 0), None)
return handler.wfile.getvalue()
def test_unauthenticated(self):
self.assertEqual(
self._test(MockRequest(b'/')),
b"""HTTP/1.0 401 Unauthorized\r
Server: BaseHTTP/x.x Python/x.x.x\r
Date: DATETIME\r
WWW-Authenticate: Basic realm="MyRealm", charset="UTF-8"\r
Content-type: text/html\r
\r
<html><head><title>Authentication Failed</title></html><body><h1>Authentication Failed</h1><p>Authentication Failed. Authorised Personnel Only.</p></body></html>"""
)
def main():
unittest.main()
if __name__ == "__main__":
main()
The code I am testing returns a 401 Unauthorised for "/". Change the response as appopriate for your test case.
Here's one approach I came up with to mock the server. Note that this should be compatible with both Python 2 and python 3. The only issue is that I can't find a way to access the result of the GET request, but at least the test will catch any exceptions it comes across!
try:
# Python 2.x
import BaseHTTPServer as server
from StringIO import StringIO as IO
except ImportError:
# Python 3.x
from http import server
from io import BytesIO as IO
class MyHandler(server.BaseHTTPRequestHandler):
"""Custom handler to be tested"""
def do_GET(self):
# print just to confirm that this method is being called
print("executing do_GET") # just to confirm...
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
# Here's where all the complicated logic is done to generate HTML.
# For clarity here, replace with a simple stand-in:
html = "<html><p>hello world</p></html>"
self.wfile.write(html.encode())
def test_handler():
"""Test the custom HTTP request handler by mocking a server"""
class MockRequest(object):
def makefile(self, *args, **kwargs):
return IO(b"GET /")
class MockServer(object):
def __init__(self, ip_port, Handler):
handler = Handler(MockRequest(), ip_port, self)
# The GET request will be sent here
# and any exceptions will be propagated through.
server = MockServer(('0.0.0.0', 8888), MyHandler)
test_handler()
So this is a little tricky depending on how "deep" you want to go into the BaseHTTPRequestHandler behavior to define your unit test. At the most basic level I think you can use this example from the mock library:
>>> from mock import MagicMock
>>> thing = ProductionClass()
>>> thing.method = MagicMock(return_value=3)
>>> thing.method(3, 4, 5, key='value')
3
>>> thing.method.assert_called_with(3, 4, 5, key='value')
So if you know which methods in the BaseHTTPRequestHandler your class is going to call you could mock the results of those methods to be something acceptable. This can of course get pretty complex depending on how many different types of server responses you want to test.
Related
I tried to mock a specific URL as shown in this example:
How can I mock requests and the response?
to test my own function:
class URLStatus():
#staticmethod
def check(url, redirects):
try:
session = requests.Session()
session.max_redirects = redirects
urlcheck = session.get(url)
return urlcheck.status_code
The issue is that it never takes the mocked url, but instead only takes real ones.
import requests
from unittest import TestCase, mock
from unittest.mock import patch
from lib.checks.url_status import URLStatus
def mocked_requests_get(*args, **kwargs):
class MockResponse:
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
if args[0] == 'http://someurl.com/test.json':
return MockResponse({"key1": "value1"}, 200)
elif args[0] == 'http://someotherurl.com/anothertest.json':
return MockResponse({"key2": "value2"}, 200)
return MockResponse(None, 404)
class URLStatusTestCase(TestCase):
#mock.patch('lib.checks.url_status.requests.get', side_effect=mocked_requests_get)
def test_check(self, mock_get):
url_status = URLStatus()
test_data = url_status.check('http://someurl.com/test.json', 5)
self.assertEqual(test_data, 200)
if __name__ == '__main__':
unittest.main()
This one, for example, fails because it sees 'http://someurl.com/test.json' as a 404, not a 200. I have no idea why this is happening.
How do I make it take the mocked URL?
You are mocking the wrong function. requests.get is a convenience function that creates its own, one-use Session, then uses its get method to provide the result. Your check method is using its own Session object; you need to at least mock that object's get method.
Given that you aren't reusing this session elsewhere, it would probably be simplest to change its implementation to take advantage of requests.get:
class URLStatus():
#staticmethod
def check(url, redirects):
return requests.get(url, max_redirects=redirects).status_code
I have a class that is using a suds client in several places to make some xml calls to another server. No problems when the code is running, however, I cannot figure out how to mock the suds client creation in the class constructor such that it will create a mocked object and not use a real socket. We have tried multiple permutations of mock.patch. mocker.patch, etc; but the ones that run result in a socket error and the rest result in AttributeError or ImportError.
This is a dumbed down version of the class:
from suds.client import Client
from suds.transport.https import HttpAuthenticated
class MYClass(object):
def __init__(self, host, usern, passw, provisioning_timeout=90):
wsdl_url = 'https://{host}/server/GetWsdl?wsdl'.format(host=host)
transport = CustomTransport()
try:
self.client = Client(wsdl_url, transport=transport, timeout=5)
def run_wsdl(self, data):
result = self.client.service.testwsdl(data=data)
return result
And this is what I am trying to run through a unit test
from me import my_class
from mock import patch
#These are some of the many permutations we've tried
#patch('my_class.MYClass.suds.client.Client') #ImportError, no suds
#patch('my_class.Client') #Socket use error in __init__
#patch('my_class.suds.client.Client') #ImportError, no suds
def test_sip_stuff(mock_client):
with patch.object(mock_client.client.service, 'testwsdl') as mockwsdl:
mockwsdl.return_value = good_wsdl_data
test_instance = my_class.MYClass(
host='10.10.10.20',
usern='user',
passw='pass'
)
return_value = test_instance.run_wsdl(data='something')
assert return_value == good_wsdl_data
I tried to make proxy server by tornado in Python. The simple http proxy server has worked well, but the https proxy has some problem.
Part of my programs which might have problem are below.
import tornado.ioloop
from tornado.web import RequestHandler, Application
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
from tornado.httpserver import HTTPServer
class HTTPSHandler(RequestHandler):
#tornado.web.asynchronous
def get(self):
print self.request.host, self.request.method
def handle_request(response):
if response.error and not isinstance(response.error, tornado.httpclient.HTTPError):
print "Error:", response.error
else:
self.write(response.body)
self.finish(" ")#in case of response.body == None
request = self.request
req = HTTPRequest(url=request.uri, method=request.method,
headers=request.headers, body=request.body,
allow_nonstandard_methods = True, follow_redirects = False,
validate_cert=True)
http_client = AsyncHTTPClient()
try:
http_client.fetch(req, handle_request)
except Exception as e:
print e
#tornado.web.asynchronous
def post(self):
return self.get()
#tornado.web.asynchronous
def head(self):
return self.get()
#tornado.web.asynchronous
def delete(self):
return self.get()
#tornado.web.asynchronous
def patch(self):
return self.get()
#tornado.web.asynchronous
def put(self):
return self.get()
#tornado.web.asynchronous
def options(self):
return self.get()
if __name__ == "__main__":
app2 = Application([(r"https:.*", HTTPSHandler),])
httpsServer = HTTPServer(app2, ssl_options = {
"certfile": "./server.crt",
"keyfile": "./server.key",
})
app2.listen(444)
tornado.ioloop.IOLoop.instance().start()
It outputs a WARNING like below (when I access to https://www.google.com and https://github.com)
WARNING:tornado.access:405 CONNECT www.google.co.jp:443 (127.0.0.1) 0.69ms
WARNING:tornado.access:405 CONNECT github.com:443 (127.0.0.1) 0.58ms
Finally, web pages which use https protocol could not be displayed with browser error.
ERR_TUNNEL_CONNECTION_FAILED
I guess, this is caused by the tornado’s requestHandler because it does not support CONNECT method.
My question is how can I use the CONNECT method?
I have noticed the way to fix this problem.
At first, SUPPORTED_METHOD should be written in HTTPSHandler class.
This can solve the 405 WARNING and browser error.
class HTTPSHandler(RequestHandler):
SUPPORTED_METHODS = ("CONNECT", "GET", "HEAD", "POST", "DELETE", "PATCH", "PUT", "OPTIONS")
#tornado.web.asynchronous
def get(self):
This is written in official document as follows.
If you want to support more methods than the standard GET/HEAD/POST, you
should override the class variable ``SUPPORTED_METHODS`` in your
`RequestHandler` subclass.
Moreover, to handle and process the CONNECT method request, additional method is needed in HTTPSHandler class.
#tornado.web.asynchronous
def connect(self):
print "some specific processings here"
Finally, I took stupid mistakes in regular expressions and ssl_option.
app2 = Application([(r"https:.*", HTTPSHandler),]) # not correct
app2 = Application([(r".*", HTTPSHandler),]) # correct
httpServer = HTTPServer(app2) # ssl_options is not needed
In theory you should be able to implement the CONNECT method in the same way that WebSocketHandler works, by hijacking the underlying connection's IOStream. But be warned that this is uncharted territory; the HTTP proxy protocol has some differences from plain HTTP and I don't know how well it will work to implement a proxy on top of a normal application-level HTTP service.
I have the following web app:
import bottle
app = bottle.Bottle()
#app.route('/ping')
def ping():
print 'pong'
return 'pong'
#app.hook('after_request')
def after():
print 'foo'
print bottle.response.body
if __name__ == "__main__":
app.run(host='0.0.0.0', port='9999', server='cherrypy')
Is there a way to access the response body before sending the response back?
If I start the app and I query /ping, I can see in the console that the ping() and the after() function run in the right sequence
$ python bottle_after_request.py
Bottle v0.11.6 server starting up (using CherryPyServer())...
Listening on http://0.0.0.0:9999/
Hit Ctrl-C to quit.
pong
foo
but when in after() I try to access response.body, I don't have anything.
In Flask the after_request decorated functions take in input the response object so it's easy to access it. How can I do the same in Bottle?
Is there something I'm missing?
Is there a way to access the response body before sending the response back?
You could write a simple plugin, which (depending on what you're actually trying to do with the response) might be all you need.
Here's an example from the Bottle plugin docs, which sets a request header. It could just as easily manipulate body.
from bottle import response, install
import time
def stopwatch(callback):
def wrapper(*args, **kwargs):
start = time.time()
body = callback(*args, **kwargs)
end = time.time()
response.headers['X-Exec-Time'] = str(end - start)
return body
return wrapper
install(stopwatch)
Hope that works for your purposes.
You can use plugin approach, this is what i did
from bottle import response
class BottlePlugin(object):
name = 'my_custom_plugin'
api = 2
def __init__(self, debug=False):
self.debug = debug
self.app = None
def setup(self, app):
"""Handle plugin install"""
self.app = app
def apply(self, callback):
"""Handle route callbacks"""
def wrapper(*a, **ka):
"""Encapsulate the result in the expected api structure"""
# Check if the client wants a different format
# output depends what you are returning from view
# in my case its dict with keys ("data")
output = callback(*a, **ka)
data = output["data"]
paging = output.get("paging", {})
response_data = {
data: data,
paging: paging
}
# in case if you want to update response
# e.g response code
response.status = 200
return response_data
return wrapper
I've got a piece of code that I can't figure out how to unit test! The module pulls content from external XML feeds (twitter, flickr, youtube, etc.) with urllib2. Here's some pseudo-code for it:
params = (url, urlencode(data),) if data else (url,)
req = Request(*params)
response = urlopen(req)
#check headers, content-length, etc...
#parse the response XML with lxml...
My first thought was to pickle the response and load it for testing, but apparently urllib's response object is unserializable (it raises an exception).
Just saving the XML from the response body isn't ideal, because my code uses the header information too. It's designed to act on a response object.
And of course, relying on an external source for data in a unit test is a horrible idea.
So how do I write a unit test for this?
urllib2 has a functions called build_opener() and install_opener() which you should use to mock the behaviour of urlopen()
import urllib2
from StringIO import StringIO
def mock_response(req):
if req.get_full_url() == "http://example.com":
resp = urllib2.addinfourl(StringIO("mock file"), "mock message", req.get_full_url())
resp.code = 200
resp.msg = "OK"
return resp
class MyHTTPHandler(urllib2.HTTPHandler):
def http_open(self, req):
print "mock opener"
return mock_response(req)
my_opener = urllib2.build_opener(MyHTTPHandler)
urllib2.install_opener(my_opener)
response=urllib2.urlopen("http://example.com")
print response.read()
print response.code
print response.msg
It would be best if you could write a mock urlopen (and possibly Request) which provides the minimum required interface to behave like urllib2's version. You'd then need to have your function/method which uses it able to accept this mock urlopen somehow, and use urllib2.urlopen otherwise.
This is a fair amount of work, but worthwhile. Remember that python is very friendly to ducktyping, so you just need to provide some semblance of the response object's properties to mock it.
For example:
class MockResponse(object):
def __init__(self, resp_data, code=200, msg='OK'):
self.resp_data = resp_data
self.code = code
self.msg = msg
self.headers = {'content-type': 'text/xml; charset=utf-8'}
def read(self):
return self.resp_data
def getcode(self):
return self.code
# Define other members and properties you want
def mock_urlopen(request):
return MockResponse(r'<xml document>')
Granted, some of these are difficult to mock, because for example I believe the normal "headers" is an HTTPMessage which implements fun stuff like case-insensitive header names. But, you might be able to simply construct an HTTPMessage with your response data.
Build a separate class or module responsible for communicating with your external feeds.
Make this class able to be a test double. You're using python, so you're pretty golden there; if you were using C#, I'd suggest either in interface or virtual methods.
In your unit test, insert a test double of the external feed class. Test that your code uses the class correctly, assuming that the class does the work of communicating with your external resources correctly. Have your test double return fake data rather than live data; test various combinations of the data and of course the possible exceptions urllib2 could throw.
Aand... that's it.
You can't effectively automate unit tests that rely on external sources, so you're best off not doing it. Run an occasional integration test on your communication module, but don't include those tests as part of your automated tests.
Edit:
Just a note on the difference between my answer and #Crast's answer. Both are essentially correct, but they involve different approaches. In Crast's approach, you use a test double on the library itself. In my approach, you abstract the use of the library away into a separate module and test double that module.
Which approach you use is entirely subjective; there's no "correct" answer there. I prefer my approach because it allows me to build more modular, flexible code, something I value. But it comes at a cost in terms of additional code to write, something that may not be valued in many agile situations.
You can use pymox to mock the behavior of anything and everything in the urllib2 (or any other) package. It's 2010, you shouldn't be writing your own mock classes.
I think the easiest thing to do is to actually create a simple web server in your unit test. When you start the test, create a new thread that listens on some arbitrary port and when a client connects just returns a known set of headers and XML, then terminates.
I can elaborate if you need more info.
Here's some code:
import threading, SocketServer, time
# a request handler
class SimpleRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(102400) # token receive
senddata = file(self.server.datafile).read() # read data from unit test file
self.request.send(senddata)
time.sleep(0.1) # make sure it finishes receiving request before closing
self.request.close()
def serve_data(datafile):
server = SocketServer.TCPServer(('127.0.0.1', 12345), SimpleRequestHandler)
server.datafile = datafile
http_server_thread = threading.Thread(target=server.handle_request())
To run your unit test, call serve_data() then call your code that requests a URL that looks like http://localhost:12345/anythingyouwant.
Why not just mock a website that returns the response you expect? then start the server in a thread in setup and kill it in the teardown. I ended up doing this for testing code that would send email by mocking an smtp server and it works great. Surely something more trivial could be done for http...
from smtpd import SMTPServer
from time import sleep
import asyncore
SMTP_PORT = 6544
class MockSMTPServer(SMTPServer):
def __init__(self, localaddr, remoteaddr, cb = None):
self.cb = cb
SMTPServer.__init__(self, localaddr, remoteaddr)
def process_message(self, peer, mailfrom, rcpttos, data):
print (peer, mailfrom, rcpttos, data)
if self.cb:
self.cb(peer, mailfrom, rcpttos, data)
self.close()
def start_smtp(cb, port=SMTP_PORT):
def smtp_thread():
_smtp = MockSMTPServer(("127.0.0.1", port), (None, 0), cb)
asyncore.loop()
return Thread(None, smtp_thread)
def test_stuff():
#.......snip noise
email_result = None
def email_back(*args):
email_result = args
t = start_smtp(email_back)
t.start()
sleep(1)
res.form["email"]= self.admin_email
res = res.form.submit()
assert res.status_int == 302,"should've redirected"
sleep(1)
assert email_result is not None, "didn't get an email"
Trying to improve a bit on #john-la-rooy answer, I've made a small class allowing simple mocking for unit tests
Should work with python 2 and 3
try:
import urllib.request as urllib
except ImportError:
import urllib2 as urllib
from io import BytesIO
class MockHTTPHandler(urllib.HTTPHandler):
def mock_response(self, req):
url = req.get_full_url()
print("incomming request:", url)
if url.endswith('.json'):
resdata = b'[{"hello": "world"}]'
headers = {'Content-Type': 'application/json'}
resp = urllib.addinfourl(BytesIO(resdata), header, url, 200)
resp.msg = "OK"
return resp
raise RuntimeError('Unhandled URL', url)
http_open = mock_response
#classmethod
def install(cls):
previous = urllib._opener
urllib.install_opener(urllib.build_opener(cls))
return previous
#classmethod
def remove(cls, previous=None):
urllib.install_opener(previous)
Used like this:
class TestOther(unittest.TestCase):
def setUp(self):
previous = MockHTTPHandler.install()
self.addCleanup(MockHTTPHandler.remove, previous)