Develop Reference

Mock entire client class with pytest

I have a class that inherits from another class in which we build a client:
class Client(ClientLibrary):
def __init__(self, hosts=[{'host':<HOST_ADDRESS>, 'port':<PORT>}], **kwargs):
''' alternative constructor, where i'd pass in some defaults to simplify connection'''
super().__init__(hosts, *args, **kwargs)
def some_method(self):
...
I want to test this class, and already have a test server set up that I want to connect to for testing. My initial approach was to create a MockClient that inherits from the original Client but swaps out the hosts parameter for the test host like so:
# I create a mock client that inherits from the original `Client` class, but passes in the host and port of the test server.
class MockClient(Client):
def __init__(self, hosts=[{'host':MOCK_HOST, 'port':MOCK_PORT}]):
super().__init__(hosts=hosts)
The idea was then that i'd use this mock client in the tests, however I have faced a lot of issues where I am testing functions that encapsulate the original Client class. I have tried patching it but keep on running into issues.
Is there a better way to approach this? And can this be done using pytest fixtures?
I want to be able to perform the following sorts of tests:
class TestFunctionThatUtilisesClient:
def test_in_which_class_is_constructed_explicitly(self):
client = Client()
r = client.some_method()
assert r == 'something'
def test_in_which_class_is_constructed_implicitly(self):
r = another_method() # Client() is called somewhere in here
assert r == 'something else'

Is there any way to Call python soap spyne server with c++ Gsoap client?

Actually i have a python script and soap server running using soap spyne and i need to call that soap api using c++ gsoap client so that the python script will run and get the output as a response to client
i am able to call the api using SOAP UI and python zeep client but when i try to call the client using gsoap it gave me error
DEBUG:spyne.protocol.soap.soap11:ValueError: Deserializing from unicode strings with encoding declaration is not supported by lxml
the generated wsdl file of both gsoap and soap spyne have different namespace also
```python
from spyne import Application, rpc, ServiceBase, Integer, Unicode,String
from spyne.protocol.soap import Soap11
from spyne.server.wsgi import WsgiApplication
from spyne.model.complex import ComplexModel
from spyne.model.complex import Array
from twisted.internet import reactor
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.python import log
import sys
sys.path.insert(1,'../cloud-client/slicing')
import speech as t
class caps__CSoapReqBuf(ComplexModel):
stringarray=String
size=Integer
class caps__CSoapResponse(ComplexModel):
__namespace__ = "spyne.examples.hello.soap"
nRetCode=Integer
strResponseData=String
class caps__CSoapRequest(ComplexModel):
__namespace__ = "spyne.examples.hello.soap"
nRequestType = Integer
wstrRequestParam= String
class caps_CCallbackData(ComplexModel):
__namespace__ = "spyne.examples.hello.soap"
nPort=Integer
strFunction = String
class TranscriptionService(ServiceBase):
#rpc(String, String, caps_CCallbackData, caps__CSoapResponse, _returns=Integer)
def caps__SoapRequestString(ctx, function_name, SoapRequest, CallbackData, SoapResponse):
parameters = SoapRequest
list = parameters.split('|')
d = dict(s.split(':') for s in list)
filename = d['path']
samplerate = int(d['sr'])
outputpath = d['outputpath']
# samplerate=parameters.samplerate
if(function_name=='gettranscription'):
print("gettranscription")
out=t.main(filename,samplerate)
SoapResponse.nRetCode=1
SoapResponse.wstrResponseData=out
return 0
elif(function_name=='getocr'):
return "Do OCR"
else:
return "error"
#rpc(caps__CSoapResponse,_returns=Unicode)
def caps_SoapResponseString(ctx,caps__CSoapResponse):
response = caps__CSoapResponse.wstrResponseData
return response
application = Application([TranscriptionService], 'spyne.examples.hello.soap',
in_protocol=Soap11(validator='lxml'),
out_protocol=Soap11())
wsgi_application = WsgiApplication(application)
if __name__ == '__main__':
import logging
from wsgiref.simple_server import make_server
ip = '192.168.0.103'
port = 8090
resource = WSGIResource(reactor, reactor, wsgi_application)
site = Site(resource)
reactor.listenTCP(port, site,interface=ip)
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
logging.info("listening to "+ ip +":"+str(port))
reactor.run()
```
Updated
After updating the code following errors follows in response.
``` Response
'<soap11env:Envelope
xmlns:soap11env="http://schemas.xmlsoap.org/soap/envelope/">\n
<soap11env:Body>\n <soap11env:Fault>\n
<faultcode>soap11env:Client.ResourceNotFound</faultcode>\n
<faultstring>Requested resource
\'{http://tempuri.org/caps.xsd/Service.wsdl}\' not
found</faultstring>\n <faultactor></faultactor>\n
</soap11env:Fault>\n </soap11env:Body>\n</soap11env:Envelope>\n'
DEBUG:spyne:gc.collect() took around 40ms.
```

On that front, lxml is a bit strict on the kind of stream it thinks it can process.
I'm persuaded that the right thing to do to solve this conflict is to strip the encoding declaration from the xml string.
You can add a handler for the 'wsgi_call' event to implement this. See the events example to see what the relevant API looks like.

My solution to this is to let lxml fail and parse the payload manually using WebOb and then place it in the user defined context. You can later access ctx.udc to access the data. So:
...
from webob import Request as WebObRequest
...
HOST = '0.0.0.0'
PORT = 8000
# parse payload manually
def _on_wsgi_call(ctx):
req = WebObRequest(ctx.transport.req)
decoded = req.body.decode('utf-8')
envelope = Etree.fromstring(decoded)
# depending on your data, you may need to update this step
element = next(envelope.iter())
ctx.udc = element.text
if __name__ == '__main__':
application = initialize([YourService])
wsgi_application = WsgiApplication(application)
wsgi_application.event_manager.add_listener('wsgi_call', _on_wsgi_call)
resource = WSGIResource(reactor, reactor, wsgi_application)
site = Site(resource)
reactor.listenTCP(PORT, site, interface=HOST)
logging.info('listening on: %s:%d' % (HOST, PORT))
logging.info('wsdl is at: http://%s:%d/?wsdl' % (HOST, PORT))
sys.exit(reactor.run())

Calling a python class method from an instance of a class doesn't work as I expect

I'm new to Python, and I think I'm trying to do something simple. However, I am confused with the results I am getting. I am declaring a class that has 2 class methods, add and remove, which in my simple example add or remove a client from a list class variable. Here's my code:
Service.py
from Client import Client
class Service:
clients = []
#classmethod
def add(cls, client):
cls.clients.append(client)
#classmethod
def remove(cls, client):
if client in cls.clients:
cls.clients.remove(client)
if __name == '__main__'
a = Client()
b = Client()
c = Client()
Service.add(a)
Service.add(b)
Service.add(c)
print(Service.clients)
c.kill()
print(Service.clients)
Service.remove(c)
print(Service.clients)
Client.py
class Client:
def kill(self):
from Service import Service
Service.remove(self)
I would expect calling c.kill() would remove the instance from the clients list.
However, when I evaluate the clients list, it is showing 0 items. when I call Service.remove(c), it shows the correct list, and removes it as expected. I am not sure what I am missing here.
If it matters, I am currently using PyCharm with my code running in a Virtualenv with Python 3.6.5.

Your current code is using circular imports, as both files utilize each other. Also, instead of relying on the client to destroy the connections, use a contextmanager to facilitate the updating of clients, and at the end of the procedure, empty clients:
import contextlib
class Client:
pass
class Service:
clients = []
#classmethod
def add(cls, client):
cls.clients.append(client)
#classmethod
#contextlib.contextmanager
def thread(cls):
yield cls
cls.clients = []
with Service.thread() as t:
t.add(Client())
t.add(Client())

Python: How to unit test a custom HTTP request Handler?

I have a custom HTTP request handler that can be simplified to something like this:
# Python 3:
from http import server
class MyHandler(server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
# Here's where all the complicated logic is done to generate HTML.
# For clarity here, replace with a simple stand-in:
html = "<html><p>hello world</p></html>"
self.wfile.write(html.encode())
I'd like to unit-test this handler (i.e. make sure that my do_GET executes without an exception) without actually starting a web server. Is there any lightweight way to mock the SimpleHTTPServer so that I can test this code?

Expanding on the answer from jakevdp, I managed to be able to check the output, too:
try:
import unittest2 as unittest
except ImportError:
import unittest
try:
from io import BytesIO as IO
except ImportError:
from StringIO import StringIO as IO
from server import MyHandlerSSL # My BaseHTTPRequestHandler child
class TestableHandler(MyHandlerSSL):
# On Python3, in socketserver.StreamRequestHandler, if this is
# set it will use makefile() to produce the output stream. Otherwise,
# it will use socketserver._SocketWriter, and we won't be able to get
# to the data
wbufsize = 1
def finish(self):
# Do not close self.wfile, so we can read its value
self.wfile.flush()
self.rfile.close()
def date_time_string(self, timestamp=None):
""" Mocked date time string """
return 'DATETIME'
def version_string(self):
""" mock the server id """
return 'BaseHTTP/x.x Python/x.x.x'
class MockSocket(object):
def getsockname(self):
return ('sockname',)
class MockRequest(object):
_sock = MockSocket()
def __init__(self, path):
self._path = path
def makefile(self, *args, **kwargs):
if args[0] == 'rb':
return IO(b"GET %s HTTP/1.0" % self._path)
elif args[0] == 'wb':
return IO(b'')
else:
raise ValueError("Unknown file type to make", args, kwargs)
class HTTPRequestHandlerTestCase(unittest.TestCase):
maxDiff = None
def _test(self, request):
handler = TestableHandler(request, (0, 0), None)
return handler.wfile.getvalue()
def test_unauthenticated(self):
self.assertEqual(
self._test(MockRequest(b'/')),
b"""HTTP/1.0 401 Unauthorized\r
Server: BaseHTTP/x.x Python/x.x.x\r
Date: DATETIME\r
WWW-Authenticate: Basic realm="MyRealm", charset="UTF-8"\r
Content-type: text/html\r
\r
<html><head><title>Authentication Failed</title></html><body><h1>Authentication Failed</h1><p>Authentication Failed. Authorised Personnel Only.</p></body></html>"""
)
def main():
unittest.main()
if __name__ == "__main__":
main()
The code I am testing returns a 401 Unauthorised for "/". Change the response as appopriate for your test case.

Here's one approach I came up with to mock the server. Note that this should be compatible with both Python 2 and python 3. The only issue is that I can't find a way to access the result of the GET request, but at least the test will catch any exceptions it comes across!
try:
# Python 2.x
import BaseHTTPServer as server
from StringIO import StringIO as IO
except ImportError:
# Python 3.x
from http import server
from io import BytesIO as IO
class MyHandler(server.BaseHTTPRequestHandler):
"""Custom handler to be tested"""
def do_GET(self):
# print just to confirm that this method is being called
print("executing do_GET") # just to confirm...
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
# Here's where all the complicated logic is done to generate HTML.
# For clarity here, replace with a simple stand-in:
html = "<html><p>hello world</p></html>"
self.wfile.write(html.encode())
def test_handler():
"""Test the custom HTTP request handler by mocking a server"""
class MockRequest(object):
def makefile(self, *args, **kwargs):
return IO(b"GET /")
class MockServer(object):
def __init__(self, ip_port, Handler):
handler = Handler(MockRequest(), ip_port, self)
# The GET request will be sent here
# and any exceptions will be propagated through.
server = MockServer(('0.0.0.0', 8888), MyHandler)
test_handler()

So this is a little tricky depending on how "deep" you want to go into the BaseHTTPRequestHandler behavior to define your unit test. At the most basic level I think you can use this example from the mock library:
>>> from mock import MagicMock
>>> thing = ProductionClass()
>>> thing.method = MagicMock(return_value=3)
>>> thing.method(3, 4, 5, key='value')
3
>>> thing.method.assert_called_with(3, 4, 5, key='value')
So if you know which methods in the BaseHTTPRequestHandler your class is going to call you could mock the results of those methods to be something acceptable. This can of course get pretty complex depending on how many different types of server responses you want to test.

How do I unit test a module that relies on urllib2?

I've got a piece of code that I can't figure out how to unit test! The module pulls content from external XML feeds (twitter, flickr, youtube, etc.) with urllib2. Here's some pseudo-code for it:
params = (url, urlencode(data),) if data else (url,)
req = Request(*params)
response = urlopen(req)
#check headers, content-length, etc...
#parse the response XML with lxml...
My first thought was to pickle the response and load it for testing, but apparently urllib's response object is unserializable (it raises an exception).
Just saving the XML from the response body isn't ideal, because my code uses the header information too. It's designed to act on a response object.
And of course, relying on an external source for data in a unit test is a horrible idea.
So how do I write a unit test for this?

urllib2 has a functions called build_opener() and install_opener() which you should use to mock the behaviour of urlopen()
import urllib2
from StringIO import StringIO
def mock_response(req):
if req.get_full_url() == "http://example.com":
resp = urllib2.addinfourl(StringIO("mock file"), "mock message", req.get_full_url())
resp.code = 200
resp.msg = "OK"
return resp
class MyHTTPHandler(urllib2.HTTPHandler):
def http_open(self, req):
print "mock opener"
return mock_response(req)
my_opener = urllib2.build_opener(MyHTTPHandler)
urllib2.install_opener(my_opener)
response=urllib2.urlopen("http://example.com")
print response.read()
print response.code
print response.msg

It would be best if you could write a mock urlopen (and possibly Request) which provides the minimum required interface to behave like urllib2's version. You'd then need to have your function/method which uses it able to accept this mock urlopen somehow, and use urllib2.urlopen otherwise.
This is a fair amount of work, but worthwhile. Remember that python is very friendly to ducktyping, so you just need to provide some semblance of the response object's properties to mock it.
For example:
class MockResponse(object):
def __init__(self, resp_data, code=200, msg='OK'):
self.resp_data = resp_data
self.code = code
self.msg = msg
self.headers = {'content-type': 'text/xml; charset=utf-8'}
def read(self):
return self.resp_data
def getcode(self):
return self.code
# Define other members and properties you want
def mock_urlopen(request):
return MockResponse(r'<xml document>')
Granted, some of these are difficult to mock, because for example I believe the normal "headers" is an HTTPMessage which implements fun stuff like case-insensitive header names. But, you might be able to simply construct an HTTPMessage with your response data.

Build a separate class or module responsible for communicating with your external feeds.
Make this class able to be a test double. You're using python, so you're pretty golden there; if you were using C#, I'd suggest either in interface or virtual methods.
In your unit test, insert a test double of the external feed class. Test that your code uses the class correctly, assuming that the class does the work of communicating with your external resources correctly. Have your test double return fake data rather than live data; test various combinations of the data and of course the possible exceptions urllib2 could throw.
Aand... that's it.
You can't effectively automate unit tests that rely on external sources, so you're best off not doing it. Run an occasional integration test on your communication module, but don't include those tests as part of your automated tests.
Edit:
Just a note on the difference between my answer and #Crast's answer. Both are essentially correct, but they involve different approaches. In Crast's approach, you use a test double on the library itself. In my approach, you abstract the use of the library away into a separate module and test double that module.
Which approach you use is entirely subjective; there's no "correct" answer there. I prefer my approach because it allows me to build more modular, flexible code, something I value. But it comes at a cost in terms of additional code to write, something that may not be valued in many agile situations.

You can use pymox to mock the behavior of anything and everything in the urllib2 (or any other) package. It's 2010, you shouldn't be writing your own mock classes.

I think the easiest thing to do is to actually create a simple web server in your unit test. When you start the test, create a new thread that listens on some arbitrary port and when a client connects just returns a known set of headers and XML, then terminates.
I can elaborate if you need more info.
Here's some code:
import threading, SocketServer, time
# a request handler
class SimpleRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(102400) # token receive
senddata = file(self.server.datafile).read() # read data from unit test file
self.request.send(senddata)
time.sleep(0.1) # make sure it finishes receiving request before closing
self.request.close()
def serve_data(datafile):
server = SocketServer.TCPServer(('127.0.0.1', 12345), SimpleRequestHandler)
server.datafile = datafile
http_server_thread = threading.Thread(target=server.handle_request())
To run your unit test, call serve_data() then call your code that requests a URL that looks like http://localhost:12345/anythingyouwant.

Why not just mock a website that returns the response you expect? then start the server in a thread in setup and kill it in the teardown. I ended up doing this for testing code that would send email by mocking an smtp server and it works great. Surely something more trivial could be done for http...
from smtpd import SMTPServer
from time import sleep
import asyncore
SMTP_PORT = 6544
class MockSMTPServer(SMTPServer):
def __init__(self, localaddr, remoteaddr, cb = None):
self.cb = cb
SMTPServer.__init__(self, localaddr, remoteaddr)
def process_message(self, peer, mailfrom, rcpttos, data):
print (peer, mailfrom, rcpttos, data)
if self.cb:
self.cb(peer, mailfrom, rcpttos, data)
self.close()
def start_smtp(cb, port=SMTP_PORT):
def smtp_thread():
_smtp = MockSMTPServer(("127.0.0.1", port), (None, 0), cb)
asyncore.loop()
return Thread(None, smtp_thread)
def test_stuff():
#.......snip noise
email_result = None
def email_back(*args):
email_result = args
t = start_smtp(email_back)
t.start()
sleep(1)
res.form["email"]= self.admin_email
res = res.form.submit()
assert res.status_int == 302,"should've redirected"
sleep(1)
assert email_result is not None, "didn't get an email"

Trying to improve a bit on #john-la-rooy answer, I've made a small class allowing simple mocking for unit tests
Should work with python 2 and 3
try:
import urllib.request as urllib
except ImportError:
import urllib2 as urllib
from io import BytesIO
class MockHTTPHandler(urllib.HTTPHandler):
def mock_response(self, req):
url = req.get_full_url()
print("incomming request:", url)
if url.endswith('.json'):
resdata = b'[{"hello": "world"}]'
headers = {'Content-Type': 'application/json'}
resp = urllib.addinfourl(BytesIO(resdata), header, url, 200)
resp.msg = "OK"
return resp
raise RuntimeError('Unhandled URL', url)
http_open = mock_response
#classmethod
def install(cls):
previous = urllib._opener
urllib.install_opener(urllib.build_opener(cls))
return previous
#classmethod
def remove(cls, previous=None):
urllib.install_opener(previous)
Used like this:
class TestOther(unittest.TestCase):
def setUp(self):
previous = MockHTTPHandler.install()
self.addCleanup(MockHTTPHandler.remove, previous)