Develop Reference

side_effect function of PropertyMock gets called only once

I have two questions regarding the mocking of a property with unittest.mock.PropertyMock (code will follow below):
Why does the test test_prop output "text_0, text_0" and not "text_0, text_1" as the test test_func? The output indicates, that the function side_effect_func in test_prop gets called only once, while I would have expected it to get called twice, like in test_func.
How can I specify a side_effect function for a property, that gets called every time the property is accessed?
My use case is that I would like to have a mock that returns a different name (which is a property) depending on how often it was called. This would "simulate" two different instances of Class1 to Class2 in the following minimal example.
The code:
File dut.py:
class Class1():
def __init__(self, name):
self.__name = name
#property
def name(self):
return self.__name
def name_func(self):
return self.__name
class Class2():
def __init__(self, name, class1):
self.__name = name
self.__class1 = class1
#property
def name(self):
return self.__name
#property
def class1(self):
return self.__class1
File test\test_dut.py (the second with-statement produces the exact same behavior when swapped with the first one):
import dut
import unittest
from unittest.mock import patch, PropertyMock
class TestClass2(unittest.TestCase):
def test_func(self):
side_effect_counter = -1
def side_effect_func(_):
nonlocal side_effect_counter
side_effect_counter += 1
return f'text_{side_effect_counter}'
c2_1 = dut.Class2('class2', dut.Class1('class1'))
c2_2 = dut.Class2('class2_2', dut.Class1('class1_2'))
with patch('test_dut.dut.Class1.name_func', side_effect=side_effect_func, autospec=True):
print(f'{c2_2.class1.name_func()}, {c2_1.class1.name_func()}')
def test_prop(self):
side_effect_counter = -1
def side_effect_func():
nonlocal side_effect_counter
side_effect_counter += 1
return f'text_{side_effect_counter}'
c2_1 = dut.Class2('class2', dut.Class1('class1'))
c2_2 = dut.Class2('class2_2', dut.Class1('class1_2'))
with patch.object(dut.Class1, 'name', new_callable=PropertyMock(side_effect=side_effect_func)):
# with patch('test_dut.dut.Class1.name', new_callable=PropertyMock(side_effect=side_effect_func)):
print(f'{c2_2.class1.name}, {c2_1.class1.name}')
Call from command line: pytest -rP test\test_dut.py
This leads to the following output (problematic line marked by me):
============================================================================================== test session starts ==============================================================================================
platform win32 -- Python 3.9.12, pytest-7.1.2, pluggy-1.0.0
rootdir: C:\Users\klosemic\Documents\playground_mocks
plugins: hypothesis-6.46.5, cov-3.0.0, forked-1.4.0, html-3.1.1, metadata-2.0.1, xdist-2.5.0
collected 2 items
test\test_dut.py .. [100%]
==================================================================================================== PASSES =====================================================================================================
_____________________________________________________________________________________________ TestClass2.test_func ______________________________________________________________________________________________
--------------------------------------------------------------------------------------------- Captured stdout call ----------------------------------------------------------------------------------------------
text_0, text_1
_____________________________________________________________________________________________ TestClass2.test_prop ______________________________________________________________________________________________
--------------------------------------------------------------------------------------------- Captured stdout call ----------------------------------------------------------------------------------------------
text_0, text_0 <<<<<< HERE IS THE PROBLEM
=============================================================================================== 2 passed in 0.46s ===============================================================================================

The issue has to do with how you are instantiating the PropertyMock. To answer your first question about why the second test prints test_0 for both calls, you instantiate the PropertyMock class during your with patch.object(get_events.Class1, 'name', new_callable=PropertyMock(side_effect=side_effect_func)) call.
Since you instantiate the class, it gets called immediately at the end of that line due to the logic of the __enter__ method in the patch object. You can see that logic in this line and then this one. Due to that the value of your side_effect immediately becomes a string, which is essentially the output of the first call to the PropertyMock. This can be confirmed by changing your function to the following and observing the output:
with patch.object(dut.Class1, 'name', new_callable=PropertyMock(side_effect=side_effect_func)) as mock_prop:
# print(f'{c2_2.class1.name}, {c2_1.class1.name}')
print(mock_prop)
You will notice that this prints text_0 in the console, confirming what has been mentioned above.
To answer your second question, the way to use PropertyMock in this case would be to change the second test to the following:
with patch.object(dut.Class1, 'name', new_callable=PropertyMock) as mock_prop:
mock_prop.side_effect = side_effect_func
print(f'{c2_2.class1.name}, {c2_1.class1.name}')
Then when you run the tests you get the correct output as shown below.
============================================================= test session starts =============================================================
platform darwin -- Python 3.8.9, pytest-7.0.1, pluggy-1.0.0
rootdir: ***
plugins: asyncio-0.18.3, mock-3.7.0
asyncio: mode=strict
collected 2 items
tests/test_dut.py .. [100%]
=================================================================== PASSES ====================================================================
____________________________________________________________ TestClass2.test_func _____________________________________________________________
------------------------------------------------------------ Captured stdout call -------------------------------------------------------------
text_0, text_1
____________________________________________________________ TestClass2.test_prop _____________________________________________________________
------------------------------------------------------------ Captured stdout call -------------------------------------------------------------
text_0, text_1
============================================================== 2 passed in 0.01s ==============================================================

Mocking a method which calls another method

Suppose I have a test class TestSuite with a method test_database_inaccessible(). I want to test a method run() in another class, AA_database.run() calls is_primary(). I can mock is_primary to return True.
I have tried
with patch.object(AADatabase, "is_primary") as is_primary_mocked:
self.dbsize = 2.1e10
self.returncode = 2
is_primary_mocked.return_value = True
self.AADatabase.run()
but I get
AttributeError: 'TestSuite' object has no attribute 'AADatabase'
Previously, I had tried simply
with patch.object(AADatabase, "is_primary") as is_primary_mocked:
self.dbsize = 2.1e10
self.returncode = 2
is_primary_mocked.return_value = True
AADatabase.run()
But I got a different error message then.
If patch is the wrong tool here, I don't mind switching to a different one. I have tried quite a few different methods.

Here is the unit test solution based on the code you provided.
main.py:
class AADatabase:
#classmethod
def is_primary(cls):
return False
#classmethod
def run(cls):
return cls.is_primary()
test_main.py:
import unittest
from main import AADatabase
from unittest.mock import patch
class TestAADatabase(unittest.TestCase):
def test_database_inaccessible(self):
with patch.object(AADatabase, 'is_primary') as is_primary_mocked:
is_primary_mocked.return_value = True
res = AADatabase.run()
self.assertTrue(res)
if __name__ == '__main__':
unittest.main()
Unit test result with coverage report:
.
----------------------------------------------------------------------
Ran 1 test in 0.001s
OK
Name Stmts Miss Cover Missing
-----------------------------------------------------------------------
src/stackoverflow/58862981/main.py 5 1 80% 4
src/stackoverflow/58862981/test_main.py 11 0 100%
-----------------------------------------------------------------------
TOTAL 16 1 94%
Source code: https://github.com/mrdulin/python-codelab/tree/master/src/stackoverflow/58862981

What is the best practice for testing a function with a list of inputs and outputs with python's unittest module? [duplicate]

I have some kind of test data and want to create a unit test for each item. My first idea was to do it like this:
import unittest
l = [["foo", "a", "a",], ["bar", "a", "b"], ["lee", "b", "b"]]
class TestSequence(unittest.TestCase):
def testsample(self):
for name, a,b in l:
print "test", name
self.assertEqual(a,b)
if __name__ == '__main__':
unittest.main()
The downside of this is that it handles all data in one test. I would like to generate one test for each item on the fly. Any suggestions?

This is called "parametrization".
There are several tools that support this approach. E.g.:
pytest's decorator
parameterized
The resulting code looks like this:
from parameterized import parameterized
class TestSequence(unittest.TestCase):
#parameterized.expand([
["foo", "a", "a",],
["bar", "a", "b"],
["lee", "b", "b"],
])
def test_sequence(self, name, a, b):
self.assertEqual(a,b)
Which will generate the tests:
test_sequence_0_foo (__main__.TestSequence) ... ok
test_sequence_1_bar (__main__.TestSequence) ... FAIL
test_sequence_2_lee (__main__.TestSequence) ... ok
======================================================================
FAIL: test_sequence_1_bar (__main__.TestSequence)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python2.7/site-packages/parameterized/parameterized.py", line 233, in <lambda>
standalone_func = lambda *a: func(*(a + p.args), **p.kwargs)
File "x.py", line 12, in test_sequence
self.assertEqual(a,b)
AssertionError: 'a' != 'b'
For historical reasons I'll leave the original answer circa 2008):
I use something like this:
import unittest
l = [["foo", "a", "a",], ["bar", "a", "b"], ["lee", "b", "b"]]
class TestSequense(unittest.TestCase):
pass
def test_generator(a, b):
def test(self):
self.assertEqual(a,b)
return test
if __name__ == '__main__':
for t in l:
test_name = 'test_%s' % t[0]
test = test_generator(t[1], t[2])
setattr(TestSequense, test_name, test)
unittest.main()

Using unittest (since 3.4)
Since Python 3.4, the standard library unittest package has the subTest context manager.
See the documentation:
26.4.7. Distinguishing test iterations using subtests
subTest
Example:
from unittest import TestCase
param_list = [('a', 'a'), ('a', 'b'), ('b', 'b')]
class TestDemonstrateSubtest(TestCase):
def test_works_as_expected(self):
for p1, p2 in param_list:
with self.subTest():
self.assertEqual(p1, p2)
You can also specify a custom message and parameter values to subTest():
with self.subTest(msg="Checking if p1 equals p2", p1=p1, p2=p2):
Using nose
The nose testing framework supports this.
Example (the code below is the entire contents of the file containing the test):
param_list = [('a', 'a'), ('a', 'b'), ('b', 'b')]
def test_generator():
for params in param_list:
yield check_em, params[0], params[1]
def check_em(a, b):
assert a == b
The output of the nosetests command:
> nosetests -v
testgen.test_generator('a', 'a') ... ok
testgen.test_generator('a', 'b') ... FAIL
testgen.test_generator('b', 'b') ... ok
======================================================================
FAIL: testgen.test_generator('a', 'b')
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/lib/python2.5/site-packages/nose-0.10.1-py2.5.egg/nose/case.py", line 203, in runTest
self.test(*self.arg)
File "testgen.py", line 7, in check_em
assert a == b
AssertionError
----------------------------------------------------------------------
Ran 3 tests in 0.006s
FAILED (failures=1)

This can be solved elegantly using Metaclasses:
import unittest
l = [["foo", "a", "a",], ["bar", "a", "b"], ["lee", "b", "b"]]
class TestSequenceMeta(type):
def __new__(mcs, name, bases, dict):
def gen_test(a, b):
def test(self):
self.assertEqual(a, b)
return test
for tname, a, b in l:
test_name = "test_%s" % tname
dict[test_name] = gen_test(a,b)
return type.__new__(mcs, name, bases, dict)
class TestSequence(unittest.TestCase):
__metaclass__ = TestSequenceMeta
if __name__ == '__main__':
unittest.main()

As of Python 3.4, subtests have been introduced to unittest for this purpose. See the documentation for details. TestCase.subTest is a context manager which allows one to isolate asserts in a test so that a failure will be reported with parameter information, but it does not stop the test execution. Here's the example from the documentation:
class NumbersTest(unittest.TestCase):
def test_even(self):
"""
Test that numbers between 0 and 5 are all even.
"""
for i in range(0, 6):
with self.subTest(i=i):
self.assertEqual(i % 2, 0)
The output of a test run would be:
======================================================================
FAIL: test_even (__main__.NumbersTest) (i=1)
----------------------------------------------------------------------
Traceback (most recent call last):
File "subtests.py", line 32, in test_even
self.assertEqual(i % 2, 0)
AssertionError: 1 != 0
======================================================================
FAIL: test_even (__main__.NumbersTest) (i=3)
----------------------------------------------------------------------
Traceback (most recent call last):
File "subtests.py", line 32, in test_even
self.assertEqual(i % 2, 0)
AssertionError: 1 != 0
======================================================================
FAIL: test_even (__main__.NumbersTest) (i=5)
----------------------------------------------------------------------
Traceback (most recent call last):
File "subtests.py", line 32, in test_even
self.assertEqual(i % 2, 0)
AssertionError: 1 != 0
This is also part of unittest2, so it is available for earlier versions of Python.

load_tests is a little known mechanism introduced in 2.7 to dynamically create a TestSuite. With it, you can easily create parametrized tests.
For example:
import unittest
class GeneralTestCase(unittest.TestCase):
def __init__(self, methodName, param1=None, param2=None):
super(GeneralTestCase, self).__init__(methodName)
self.param1 = param1
self.param2 = param2
def runTest(self):
pass # Test that depends on param 1 and 2.
def load_tests(loader, tests, pattern):
test_cases = unittest.TestSuite()
for p1, p2 in [(1, 2), (3, 4)]:
test_cases.addTest(GeneralTestCase('runTest', p1, p2))
return test_cases
That code will run all the TestCases in the TestSuite returned by load_tests. No other tests are automatically run by the discovery mechanism.
Alternatively, you can also use inheritance as shown in this ticket: http://bugs.python.org/msg151444

It can be done by using pytest. Just write the file test_me.py with content:
import pytest
#pytest.mark.parametrize('name, left, right', [['foo', 'a', 'a'],
['bar', 'a', 'b'],
['baz', 'b', 'b']])
def test_me(name, left, right):
assert left == right, name
And run your test with command py.test --tb=short test_me.py. Then the output will look like:
=========================== test session starts ============================
platform darwin -- Python 2.7.6 -- py-1.4.23 -- pytest-2.6.1
collected 3 items
test_me.py .F.
================================= FAILURES =================================
_____________________________ test_me[bar-a-b] _____________________________
test_me.py:8: in test_me
assert left == right, name
E AssertionError: bar
==================== 1 failed, 2 passed in 0.01 seconds ====================
It is simple! Also pytest has more features like fixtures, mark, assert, etc.

Use the ddt library. It adds simple decorators for the test methods:
import unittest
from ddt import ddt, data
from mycode import larger_than_two
#ddt
class FooTestCase(unittest.TestCase):
#data(3, 4, 12, 23)
def test_larger_than_two(self, value):
self.assertTrue(larger_than_two(value))
#data(1, -3, 2, 0)
def test_not_larger_than_two(self, value):
self.assertFalse(larger_than_two(value))
This library can be installed with pip. It doesn't require nose, and works excellent with the standard library unittest module.

You would benefit from trying the TestScenarios library.
testscenarios provides clean dependency injection for python unittest style tests. This can be used for interface testing (testing many implementations via a single test suite) or for classic dependency injection (provide tests with dependencies externally to the test code itself, allowing easy testing in different situations).

There's also Hypothesis which adds fuzz or property based testing.
This is a very powerful testing method.

This is effectively the same as parameterized as mentioned in a previous answer, but specific to unittest:
def sub_test(param_list):
"""Decorates a test case to run it as a set of subtests."""
def decorator(f):
#functools.wraps(f)
def wrapped(self):
for param in param_list:
with self.subTest(**param):
f(self, **param)
return wrapped
return decorator
Example usage:
class TestStuff(unittest.TestCase):
#sub_test([
dict(arg1='a', arg2='b'),
dict(arg1='x', arg2='y'),
])
def test_stuff(self, arg1, arg2):
...

You can use the nose-ittr plugin (pip install nose-ittr).
It's very easy to integrate with existing tests, and minimal changes (if any) are required. It also supports the nose multiprocessing plugin.
Note that you can also have a customize setup function per test.
#ittr(number=[1, 2, 3, 4])
def test_even(self):
assert_equal(self.number % 2, 0)
It is also possible to pass nosetest parameters like with their built-in plugin attrib. This way you can run only a specific test with specific parameter:
nosetest -a number=2

I use metaclasses and decorators for generate tests. You can check my implementation python_wrap_cases. This library doesn't require any test frameworks.
Your example:
import unittest
from python_wrap_cases import wrap_case
#wrap_case
class TestSequence(unittest.TestCase):
#wrap_case("foo", "a", "a")
#wrap_case("bar", "a", "b")
#wrap_case("lee", "b", "b")
def testsample(self, name, a, b):
print "test", name
self.assertEqual(a, b)
Console output:
testsample_u'bar'_u'a'_u'b' (tests.example.test_stackoverflow.TestSequence) ... test bar
FAIL
testsample_u'foo'_u'a'_u'a' (tests.example.test_stackoverflow.TestSequence) ... test foo
ok
testsample_u'lee'_u'b'_u'b' (tests.example.test_stackoverflow.TestSequence) ... test lee
ok
Also you may use generators. For example this code generate all possible combinations of tests with arguments a__list and b__list
import unittest
from python_wrap_cases import wrap_case
#wrap_case
class TestSequence(unittest.TestCase):
#wrap_case(a__list=["a", "b"], b__list=["a", "b"])
def testsample(self, a, b):
self.assertEqual(a, b)
Console output:
testsample_a(u'a')_b(u'a') (tests.example.test_stackoverflow.TestSequence) ... ok
testsample_a(u'a')_b(u'b') (tests.example.test_stackoverflow.TestSequence) ... FAIL
testsample_a(u'b')_b(u'a') (tests.example.test_stackoverflow.TestSequence) ... FAIL
testsample_a(u'b')_b(u'b') (tests.example.test_stackoverflow.TestSequence) ... ok

I came across ParamUnittest the other day when looking at the source code for radon (example usage on the GitHub repository). It should work with other frameworks that extend TestCase (like Nose).
Here is an example:
import unittest
import paramunittest
#paramunittest.parametrized(
('1', '2'),
#(4, 3), <---- Uncomment to have a failing test
('2', '3'),
(('4', ), {'b': '5'}),
((), {'a': 5, 'b': 6}),
{'a': 5, 'b': 6},
)
class TestBar(TestCase):
def setParameters(self, a, b):
self.a = a
self.b = b
def testLess(self):
self.assertLess(self.a, self.b)

import unittest
def generator(test_class, a, b):
def test(self):
self.assertEqual(a, b)
return test
def add_test_methods(test_class):
# The first element of list is variable "a", then variable "b", then name of test case that will be used as suffix.
test_list = [[2,3, 'one'], [5,5, 'two'], [0,0, 'three']]
for case in test_list:
test = generator(test_class, case[0], case[1])
setattr(test_class, "test_%s" % case[2], test)
class TestAuto(unittest.TestCase):
def setUp(self):
print 'Setup'
pass
def tearDown(self):
print 'TearDown'
pass
_add_test_methods(TestAuto) # It's better to start with underscore so it is not detected as a test itself
if __name__ == '__main__':
unittest.main(verbosity=1)
RESULT:
>>>
Setup
FTearDown
Setup
TearDown
.Setup
TearDown
.
======================================================================
FAIL: test_one (__main__.TestAuto)
----------------------------------------------------------------------
Traceback (most recent call last):
File "D:/inchowar/Desktop/PyTrash/test_auto_3.py", line 5, in test
self.assertEqual(a, b)
AssertionError: 2 != 3
----------------------------------------------------------------------
Ran 3 tests in 0.019s
FAILED (failures=1)

Just use metaclasses, as seen here;
class DocTestMeta(type):
"""
Test functions are generated in metaclass due to the way some
test loaders work. For example, setupClass() won't get called
unless there are other existing test methods, and will also
prevent unit test loader logic being called before the test
methods have been defined.
"""
def __init__(self, name, bases, attrs):
super(DocTestMeta, self).__init__(name, bases, attrs)
def __new__(cls, name, bases, attrs):
def func(self):
"""Inner test method goes here"""
self.assertTrue(1)
func.__name__ = 'test_sample'
attrs[func.__name__] = func
return super(DocTestMeta, cls).__new__(cls, name, bases, attrs)
class ExampleTestCase(TestCase):
"""Our example test case, with no methods defined"""
__metaclass__ = DocTestMeta
Output:
test_sample (ExampleTestCase) ... OK

I'd been having trouble with a very particular style of parameterized tests. All our Selenium tests can run locally, but they also should be able to be run remotely against several platforms on SauceLabs. Basically, I wanted to take a large amount of already-written test cases and parameterize them with the fewest changes to code possible. Furthermore, I needed to be able to pass the parameters into the setUp method, something which I haven't seen any solutions for elsewhere.
Here's what I've come up with:
import inspect
import types
test_platforms = [
{'browserName': "internet explorer", 'platform': "Windows 7", 'version': "10.0"},
{'browserName': "internet explorer", 'platform': "Windows 7", 'version': "11.0"},
{'browserName': "firefox", 'platform': "Linux", 'version': "43.0"},
]
def sauce_labs():
def wrapper(cls):
return test_on_platforms(cls)
return wrapper
def test_on_platforms(base_class):
for name, function in inspect.getmembers(base_class, inspect.isfunction):
if name.startswith('test_'):
for platform in test_platforms:
new_name = '_'.join(list([name, ''.join(platform['browserName'].title().split()), platform['version']]))
new_function = types.FunctionType(function.__code__, function.__globals__, new_name,
function.__defaults__, function.__closure__)
setattr(new_function, 'platform', platform)
setattr(base_class, new_name, new_function)
delattr(base_class, name)
return base_class
With this, all I had to do was add a simple decorator #sauce_labs() to each regular old TestCase, and now when running them, they're wrapped up and rewritten, so that all the test methods are parameterized and renamed. LoginTests.test_login(self) runs as LoginTests.test_login_internet_explorer_10.0(self), LoginTests.test_login_internet_explorer_11.0(self), and LoginTests.test_login_firefox_43.0(self), and each one has the parameter self.platform to decide what browser/platform to run against, even in LoginTests.setUp, which is crucial for my task since that's where the connection to SauceLabs is initialized.
Anyway, I hope this might be of help to someone looking to do a similar "global" parameterization of their tests!

You can use TestSuite and custom TestCase classes.
import unittest
class CustomTest(unittest.TestCase):
def __init__(self, name, a, b):
super().__init__()
self.name = name
self.a = a
self.b = b
def runTest(self):
print("test", self.name)
self.assertEqual(self.a, self.b)
if __name__ == '__main__':
suite = unittest.TestSuite()
suite.addTest(CustomTest("Foo", 1337, 1337))
suite.addTest(CustomTest("Bar", 0xDEAD, 0xC0DE))
unittest.TextTestRunner().run(suite)

I have found that this works well for my purposes, especially if I need to generate tests that do slightly difference processes on a collection of data.
import unittest
def rename(newName):
def renamingFunc(func):
func.__name__ == newName
return func
return renamingFunc
class TestGenerator(unittest.TestCase):
TEST_DATA = {}
#classmethod
def generateTests(cls):
for dataName, dataValue in TestGenerator.TEST_DATA:
for func in cls.getTests(dataName, dataValue):
setattr(cls, "test_{:s}_{:s}".format(func.__name__, dataName), func)
#classmethod
def getTests(cls):
raise(NotImplementedError("This must be implemented"))
class TestCluster(TestGenerator):
TEST_CASES = []
#staticmethod
def getTests(dataName, dataValue):
def makeTest(case):
#rename("{:s}".format(case["name"]))
def test(self):
# Do things with self, case, data
pass
return test
return [makeTest(c) for c in TestCluster.TEST_CASES]
TestCluster.generateTests()
The TestGenerator class can be used to spawn different sets of test cases like TestCluster.
TestCluster can be thought of as an implementation of the TestGenerator interface.

This solution works with unittest and nose for Python 2 and Python 3:
#!/usr/bin/env python
import unittest
def make_function(description, a, b):
def ghost(self):
self.assertEqual(a, b, description)
print(description)
ghost.__name__ = 'test_{0}'.format(description)
return ghost
class TestsContainer(unittest.TestCase):
pass
testsmap = {
'foo': [1, 1],
'bar': [1, 2],
'baz': [5, 5]}
def generator():
for name, params in testsmap.iteritems():
test_func = make_function(name, params[0], params[1])
setattr(TestsContainer, 'test_{0}'.format(name), test_func)
generator()
if __name__ == '__main__':
unittest.main()

Meta-programming is fun, but it can get in the way. Most solutions here make it difficult to:
selectively launch a test
point back to the code given the test's name
So, my first suggestion is to follow the simple/explicit path (works with any test runner):
import unittest
class TestSequence(unittest.TestCase):
def _test_complex_property(self, a, b):
self.assertEqual(a,b)
def test_foo(self):
self._test_complex_property("a", "a")
def test_bar(self):
self._test_complex_property("a", "b")
def test_lee(self):
self._test_complex_property("b", "b")
if __name__ == '__main__':
unittest.main()
Since we shouldn't repeat ourselves, my second suggestion builds on Javier's answer: embrace property based testing. Hypothesis library:
is "more relentlessly devious about test case generation than us mere humans"
will provide simple count-examples
works with any test runner
has many more interesting features (statistics, additional test output, ...)
class TestSequence(unittest.TestCase):
#given(st.text(), st.text())
def test_complex_property(self, a, b):
self.assertEqual(a,b)
To test your specific examples, just add:
#example("a", "a")
#example("a", "b")
#example("b", "b")
To run only one particular example, you can comment out the other examples (provided example will be run first). You may want to use #given(st.nothing()). Another option is to replace the whole block by:
#given(st.just("a"), st.just("b"))
OK, you don't have distinct test names. But maybe you just need:
a descriptive name of the property under test.
which input leads to failure (falsifying example).
Funnier example

I had trouble making these work for setUpClass.
Here's a version of Javier's answer that gives setUpClass access to dynamically allocated attributes.
import unittest
class GeneralTestCase(unittest.TestCase):
#classmethod
def setUpClass(cls):
print ''
print cls.p1
print cls.p2
def runTest1(self):
self.assertTrue((self.p2 - self.p1) == 1)
def runTest2(self):
self.assertFalse((self.p2 - self.p1) == 2)
def load_tests(loader, tests, pattern):
test_cases = unittest.TestSuite()
for p1, p2 in [(1, 2), (3, 4)]:
clsname = 'TestCase_{}_{}'.format(p1, p2)
dct = {
'p1': p1,
'p2': p2,
}
cls = type(clsname, (GeneralTestCase,), dct)
test_cases.addTest(cls('runTest1'))
test_cases.addTest(cls('runTest2'))
return test_cases
Outputs
1
2
..
3
4
..
----------------------------------------------------------------------
Ran 4 tests in 0.000s
OK

The metaclass-based answers still work in Python 3, but instead of the __metaclass__ attribute, one has to use the metaclass parameter, as in:
class ExampleTestCase(TestCase,metaclass=DocTestMeta):
pass

import unittest
def generator(test_class, a, b,c,d,name):
def test(self):
print('Testexecution=',name)
print('a=',a)
print('b=',b)
print('c=',c)
print('d=',d)
return test
def add_test_methods(test_class):
test_list = [[3,3,5,6, 'one'], [5,5,8,9, 'two'], [0,0,5,6, 'three'],[0,0,2,3,'Four']]
for case in test_list:
print('case=',case[0], case[1],case[2],case[3],case[4])
test = generator(test_class, case[0], case[1],case[2],case[3],case[4])
setattr(test_class, "test_%s" % case[4], test)
class TestAuto(unittest.TestCase):
def setUp(self):
print ('Setup')
pass
def tearDown(self):
print ('TearDown')
pass
add_test_methods(TestAuto)
if __name__ == '__main__':
unittest.main(verbosity=1)

Following is my solution. I find this useful when:
Should work for unittest.Testcase and unittest discover
Have a set of tests to be run for different parameter settings.
Very simple and no dependency on other packages
import unittest
class BaseClass(unittest.TestCase):
def setUp(self):
self.param = 2
self.base = 2
def test_me(self):
self.assertGreaterEqual(5, self.param+self.base)
def test_me_too(self):
self.assertLessEqual(3, self.param+self.base)
class Child_One(BaseClass):
def setUp(self):
BaseClass.setUp(self)
self.param = 4
class Child_Two(BaseClass):
def setUp(self):
BaseClass.setUp(self)
self.param = 1

Besides using setattr, we can use load_tests with Python 3.2 and later.
class Test(unittest.TestCase):
pass
def _test(self, file_name):
open(file_name, 'r') as f:
self.assertEqual('test result',f.read())
def _generate_test(file_name):
def test(self):
_test(self, file_name)
return test
def _generate_tests():
for file in files:
file_name = os.path.splitext(os.path.basename(file))[0]
setattr(Test, 'test_%s' % file_name, _generate_test(file))
test_cases = (Test,)
def load_tests(loader, tests, pattern):
_generate_tests()
suite = TestSuite()
for test_class in test_cases:
tests = loader.loadTestsFromTestCase(test_class)
suite.addTests(tests)
return suite
if __name__ == '__main__':
_generate_tests()
unittest.main()

How to run unittest test cases in the order they are declared

I fully realize that the order of unit tests should not matter. But these unit tests are as much for instructional use as for actual unit testing, so I would like the test output to match up with the test case source code.
I see that there is a way to set the sort order by setting the sortTestMethodsUsing attribute on the test loader. The default is a simple cmp() call to lexically compare names. So I tried writing a cmp-like function that would take two names, find their declaration line numbers and them return the cmp()-equivalent of them:
import unittest
class TestCaseB(unittest.TestCase):
def test(self):
print("running test case B")
class TestCaseA(unittest.TestCase):
def test(self):
print("running test case A")
import inspect
def get_decl_line_no(cls_name):
cls = globals()[cls_name]
return inspect.getsourcelines(cls)[1]
def sgn(x):
return -1 if x < 0 else 1 if x > 0 else 0
def cmp_class_names_by_decl_order(cls_a, cls_b):
a = get_decl_line_no(cls_a)
b = get_decl_line_no(cls_b)
return sgn(a - b)
unittest.defaultTestLoader.sortTestMethodsUsing = cmp_class_names_by_decl_order
unittest.main()
When I run this, I get this output:
running test case A
.running test case B
.
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
indicating that the test cases are not running in the declaration order.
My sort function is just not being called, so I suspect that main() is building a new test loader, which is wiping out my sort function.

The solution is to create a TestSuite explicitly, instead of letting unittest.main() follow all its default test discovery and ordering behavior. Here's how I got it to work:
import unittest
class TestCaseB(unittest.TestCase):
def runTest(self):
print("running test case B")
class TestCaseA(unittest.TestCase):
def runTest(self):
print("running test case A")
import inspect
def get_decl_line_no(cls):
return inspect.getsourcelines(cls)[1]
# get all test cases defined in this module
test_case_classes = list(filter(lambda c: c.__name__ in globals(),
unittest.TestCase.__subclasses__()))
# sort them by decl line no
test_case_classes.sort(key=get_decl_line_no)
# make into a suite and run it
suite = unittest.TestSuite(cls() for cls in test_case_classes)
unittest.TextTestRunner().run(suite)
This gives the desired output:
running test case B
.running test case A
.
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
It is important to note that the test method in each class must be named runTest.

You can manually build a TestSuite where your TestCases and all tests inside them run by line number:
# Python 3.8.3
import unittest
import sys
import inspect
def isTestClass(x):
return inspect.isclass(x) and issubclass(x, unittest.TestCase)
def isTestFunction(x):
return inspect.isfunction(x) and x.__name__.startswith("test")
class TestB(unittest.TestCase):
def test_B(self):
print("Running test_B")
self.assertEqual((2+2), 4)
def test_A(self):
print("Running test_A")
self.assertEqual((2+2), 4)
def setUpClass():
print("TestB Class Setup")
class TestA(unittest.TestCase):
def test_A(self):
print("Running test_A")
self.assertEqual((2+2), 4)
def test_B(self):
print("Running test_B")
self.assertEqual((2+2), 4)
def setUpClass():
print("TestA Class Setup")
def suite():
# get current module object
module = sys.modules[__name__]
# get all test className,class tuples in current module
testClasses = [
tup for tup in
inspect.getmembers(module, isTestClass)
]
# sort classes by line number
testClasses.sort(key=lambda t: inspect.getsourcelines(t[1])[1])
testSuite = unittest.TestSuite()
for testClass in testClasses:
# get list of testFunctionName,testFunction tuples in current class
classTests = [
tup for tup in
inspect.getmembers(testClass[1], isTestFunction)
]
# sort TestFunctions by line number
classTests.sort(key=lambda t: inspect.getsourcelines(t[1])[1])
# create TestCase instances and add to testSuite;
for test in classTests:
testSuite.addTest(testClass[1](test[0]))
return testSuite
if __name__ == '__main__':
runner = unittest.TextTestRunner()
runner.run(suite())
Output:
TestB Class Setup
Running test_B
.Running test_A
.TestA Class Setup
Running test_A
.Running test_B
.
----------------------------------------------------------------------
Ran 4 tests in 0.000s
OK

As stated in the name, sortTestMethodsUsing is used to sort test methods. It is not used to sort classes. (It is not used to sort methods in different classes either; separate classes are handled separately.)
If you had two test methods in the same class, sortTestMethodsUsing would be used to determine their order. (At that point, you would get an exception because your function expects class names.)

Outputting data from unit test in Python

If I'm writing unit tests in Python (using the unittest module), is it possible to output data from a failed test, so I can examine it to help deduce what caused the error?
I am aware of the ability to create a customized message, which can carry some information, but sometimes you might deal with more complex data, that can't easily be represented as a string.
For example, suppose you had a class Foo, and were testing a method bar, using data from a list called testdata:
class TestBar(unittest.TestCase):
def runTest(self):
for t1, t2 in testdata:
f = Foo(t1)
self.assertEqual(f.bar(t2), 2)
If the test failed, I might want to output t1, t2 and/or f, to see why this particular data resulted in a failure. By output, I mean that the variables can be accessed like any other variables, after the test has been run.

We use the logging module for this.
For example:
import logging
class SomeTest( unittest.TestCase ):
def testSomething( self ):
log= logging.getLogger( "SomeTest.testSomething" )
log.debug( "this= %r", self.this )
log.debug( "that= %r", self.that )
self.assertEqual( 3.14, pi )
if __name__ == "__main__":
logging.basicConfig( stream=sys.stderr )
logging.getLogger( "SomeTest.testSomething" ).setLevel( logging.DEBUG )
unittest.main()
That allows us to turn on debugging for specific tests which we know are failing and for which we want additional debugging information.
My preferred method, however, isn't to spend a lot of time on debugging, but spend it writing more fine-grained tests to expose the problem.

In Python 2.7 you could use an additional parameter, msg, to add information to the error message like this:
self.assertEqual(f.bar(t2), 2, msg='{0}, {1}'.format(t1, t2))
The official documentation is here.

You can use simple print statements, or any other way of writing to standard output. You can also invoke the Python debugger anywhere in your tests.
If you use nose to run your tests (which I recommend), it will collect the standard output for each test and only show it to you if the test failed, so you don't have to live with the cluttered output when the tests pass.
nose also has switches to automatically show variables mentioned in asserts, or to invoke the debugger on failed tests. For example, -s (--nocapture) prevents the capture of standard output.

I don't think this is quite what you're looking for. There's no way to display variable values that don't fail, but this may help you get closer to outputting the results the way you want.
You can use the TestResult object returned by the TestRunner.run() for results analysis and processing. Particularly, TestResult.errors and TestResult.failures
About the TestResults Object:
http://docs.python.org/library/unittest.html#id3
And some code to point you in the right direction:
>>> import random
>>> import unittest
>>>
>>> class TestSequenceFunctions(unittest.TestCase):
... def setUp(self):
... self.seq = range(5)
... def testshuffle(self):
... # make sure the shuffled sequence does not lose any elements
... random.shuffle(self.seq)
... self.seq.sort()
... self.assertEqual(self.seq, range(10))
... def testchoice(self):
... element = random.choice(self.seq)
... error_test = 1/0
... self.assert_(element in self.seq)
... def testsample(self):
... self.assertRaises(ValueError, random.sample, self.seq, 20)
... for element in random.sample(self.seq, 5):
... self.assert_(element in self.seq)
...
>>> suite = unittest.TestLoader().loadTestsFromTestCase(TestSequenceFunctions)
>>> testResult = unittest.TextTestRunner(verbosity=2).run(suite)
testchoice (__main__.TestSequenceFunctions) ... ERROR
testsample (__main__.TestSequenceFunctions) ... ok
testshuffle (__main__.TestSequenceFunctions) ... FAIL
======================================================================
ERROR: testchoice (__main__.TestSequenceFunctions)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<stdin>", line 11, in testchoice
ZeroDivisionError: integer division or modulo by zero
======================================================================
FAIL: testshuffle (__main__.TestSequenceFunctions)
----------------------------------------------------------------------
Traceback (most recent call last):
File "<stdin>", line 8, in testshuffle
AssertionError: [0, 1, 2, 3, 4] != [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
----------------------------------------------------------------------
Ran 3 tests in 0.031s
FAILED (failures=1, errors=1)
>>>
>>> testResult.errors
[(<__main__.TestSequenceFunctions testMethod=testchoice>, 'Traceback (most recent call last):\n File "<stdin>"
, line 11, in testchoice\nZeroDivisionError: integer division or modulo by zero\n')]
>>>
>>> testResult.failures
[(<__main__.TestSequenceFunctions testMethod=testshuffle>, 'Traceback (most recent call last):\n File "<stdin>
", line 8, in testshuffle\nAssertionError: [0, 1, 2, 3, 4] != [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n')]
>>>

The method I use is really simple. I just log it as a warning so it will actually show up.
import logging
class TestBar(unittest.TestCase):
def runTest(self):
#this line is important
logging.basicConfig()
log = logging.getLogger("LOG")
for t1, t2 in testdata:
f = Foo(t1)
self.assertEqual(f.bar(t2), 2)
log.warning(t1)

In these cases I use a log.debug() with some messages in my application. Since the default logging level is WARNING, such messages don't show in the normal execution.
Then, in the unit test I change the logging level to DEBUG, so that such messages are shown while running them.
import logging
log.debug("Some messages to be shown just when debugging or unit testing")
In the unit tests:
# Set log level
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel)
See a full example:
This is daikiri.py, a basic class that implements a daikiri with its name and price. There is method make_discount() that returns the price of that specific daikiri after applying a given discount:
import logging
log = logging.getLogger(__name__)
class Daikiri(object):
def __init__(self, name, price):
self.name = name
self.price = price
def make_discount(self, percentage):
log.debug("Deducting discount...") # I want to see this message
return self.price * percentage
Then, I create a unit test, test_daikiri.py, that checks its usage:
import unittest
import logging
from .daikiri import Daikiri
class TestDaikiri(unittest.TestCase):
def setUp(self):
# Changing log level to DEBUG
loglevel = logging.DEBUG
logging.basicConfig(level=loglevel)
self.mydaikiri = Daikiri("cuban", 25)
def test_drop_price(self):
new_price = self.mydaikiri.make_discount(0)
self.assertEqual(new_price, 0)
if __name__ == "__main__":
unittest.main()
So when I execute it I get the log.debug messages:
$ python -m test_daikiri
DEBUG:daikiri:Deducting discount...
.
----------------------------------------------------------------------
Ran 1 test in 0.000s
OK

Another option - start a debugger where the test fails.
Try running your tests with Testoob (it will run your unit test suite without changes), and you can use the '--debug' command line switch to open a debugger when a test fails.
Here's a terminal session on Windows:
C:\work> testoob tests.py --debug
F
Debugging for failure in test: test_foo (tests.MyTests.test_foo)
> c:\python25\lib\unittest.py(334)failUnlessEqual()
-> (msg or '%r != %r' % (first, second))
(Pdb) up
> c:\work\tests.py(6)test_foo()
-> self.assertEqual(x, y)
(Pdb) l
1 from unittest import TestCase
2 class MyTests(TestCase):
3 def test_foo(self):
4 x = 1
5 y = 2
6 -> self.assertEqual(x, y)
[EOF]
(Pdb)

I think I might have been overthinking this. One way I've come up with that does the job, is simply to have a global variable that accumulates the diagnostic data.
Something like this:
log1 = dict()
class TestBar(unittest.TestCase):
def runTest(self):
for t1, t2 in testdata:
f = Foo(t1)
if f.bar(t2) != 2:
log1("TestBar.runTest") = (f, t1, t2)
self.fail("f.bar(t2) != 2")

Use logging:
import unittest
import logging
import inspect
import os
logging_level = logging.INFO
try:
log_file = os.environ["LOG_FILE"]
except KeyError:
log_file = None
def logger(stack=None):
if not hasattr(logger, "initialized"):
logging.basicConfig(filename=log_file, level=logging_level)
logger.initialized = True
if not stack:
stack = inspect.stack()
name = stack[1][3]
try:
name = stack[1][0].f_locals["self"].__class__.__name__ + "." + name
except KeyError:
pass
return logging.getLogger(name)
def todo(msg):
logger(inspect.stack()).warning("TODO: {}".format(msg))
def get_pi():
logger().info("sorry, I know only three digits")
return 3.14
class Test(unittest.TestCase):
def testName(self):
todo("use a better get_pi")
pi = get_pi()
logger().info("pi = {}".format(pi))
todo("check more digits in pi")
self.assertAlmostEqual(pi, 3.14)
logger().debug("end of this test")
pass
Usage:
# LOG_FILE=/tmp/log python3 -m unittest LoggerDemo
.
----------------------------------------------------------------------
Ran 1 test in 0.047s
OK
# cat /tmp/log
WARNING:Test.testName:TODO: use a better get_pi
INFO:get_pi:sorry, I know only three digits
INFO:Test.testName:pi = 3.14
WARNING:Test.testName:TODO: check more digits in pi
If you do not set LOG_FILE, logging will got to stderr.

You can use logging module for that.
So in the unit test code, use:
import logging as log
def test_foo(self):
log.debug("Some debug message.")
log.info("Some info message.")
log.warning("Some warning message.")
log.error("Some error message.")
By default warnings and errors are outputted to /dev/stderr, so they should be visible on the console.
To customize logs (such as formatting), try the following sample:
# Set-up logger
if args.verbose or args.debug:
logging.basicConfig( stream=sys.stdout )
root = logging.getLogger()
root.setLevel(logging.INFO if args.verbose else logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO if args.verbose else logging.DEBUG)
ch.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(name)s: %(message)s'))
root.addHandler(ch)
else:
logging.basicConfig(stream=sys.stderr)

inspect.trace will let you get local variables after an exception has been thrown. You can then wrap the unit tests with a decorator like the following one to save off those local variables for examination during the post mortem.
import random
import unittest
import inspect
def store_result(f):
"""
Store the results of a test
On success, store the return value.
On failure, store the local variables where the exception was thrown.
"""
def wrapped(self):
if 'results' not in self.__dict__:
self.results = {}
# If a test throws an exception, store local variables in results:
try:
result = f(self)
except Exception as e:
self.results[f.__name__] = {'success':False, 'locals':inspect.trace()[-1][0].f_locals}
raise e
self.results[f.__name__] = {'success':True, 'result':result}
return result
return wrapped
def suite_results(suite):
"""
Get all the results from a test suite
"""
ans = {}
for test in suite:
if 'results' in test.__dict__:
ans.update(test.results)
return ans
# Example:
class TestSequenceFunctions(unittest.TestCase):
def setUp(self):
self.seq = range(10)
#store_result
def test_shuffle(self):
# make sure the shuffled sequence does not lose any elements
random.shuffle(self.seq)
self.seq.sort()
self.assertEqual(self.seq, range(10))
# should raise an exception for an immutable sequence
self.assertRaises(TypeError, random.shuffle, (1,2,3))
return {1:2}
#store_result
def test_choice(self):
element = random.choice(self.seq)
self.assertTrue(element in self.seq)
return {7:2}
#store_result
def test_sample(self):
x = 799
with self.assertRaises(ValueError):
random.sample(self.seq, 20)
for element in random.sample(self.seq, 5):
self.assertTrue(element in self.seq)
return {1:99999}
suite = unittest.TestLoader().loadTestsFromTestCase(TestSequenceFunctions)
unittest.TextTestRunner(verbosity=2).run(suite)
from pprint import pprint
pprint(suite_results(suite))
The last line will print the returned values where the test succeeded and the local variables, in this case x, when it fails:
{'test_choice': {'result': {7: 2}, 'success': True},
'test_sample': {'locals': {'self': <__main__.TestSequenceFunctions testMethod=test_sample>,
'x': 799},
'success': False},
'test_shuffle': {'result': {1: 2}, 'success': True}}

Catch the exception that gets generated from the assertion failure. In your catch block you could output the data however you wanted to wherever. Then when you were done you could rethrow the exception. The test runner probably wouldn't know the difference.
Disclaimer: I haven't tried this with Python's unit test framework, but I have with other unit test frameworks.

Expanding on Facundo Casco's answer, this works quite well for me:
class MyTest(unittest.TestCase):
def messenger(self, message):
try:
self.assertEqual(1, 2, msg=message)
except AssertionError as e:
print "\nMESSENGER OUTPUT: %s" % str(e),