Develop Reference

How to delete test files when python unittest fails

I'm using python unittest for functions that write data to JSON. I use tearDownClass to delete the output test files so they don't clutter the local repo. Ground truths are also stored as JSON files.
I do want to store the output test files when tests fail, so its easier for troubleshooting.
My current implementation is to use a global boolean keep_file = False. When the unittest fails the assertion, it modifies keep_file = True. tearDownClass only deletes the files when keep_file == False. I don't like the idea of modifying global variables and the try exception blocks for each assert.
import json
import os
import unittest
from src.mymodule import foo1, foo2
# These are defined outside the class on purpose so the classmethods can access them
FILE_1 = "unittest.file1.json"
EXPECTED_FILE_1 = "expected.file1.json"
FILE_2 = "unittest.file2.json"
EXPECTED_FILE_2 = "expected.file2.json"
keep_files = False
class TestRhaPostPayload(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.get_file1()
cls.get_file2()
#classmethod
def get_file1(cls):
output1 = foo1()
with open(FILE_1, "w") as f:
f.write(output1)
#classmethod
def get_file2(cls):
output2 = foo1()
with open(FILE_2, "w") as f:
f.write(output2)
#classmethod
def tearDownClass(cls):
if not keep_files:
os.remove(FILE_1)
os.remove(FILE_2)
def test_foo1(self):
# code that reads in file1 and expected_file_1
try:
self.assert(expected_output1, output1)
except AssertionError:
global keep_files
keep_files = True
raise
def test_foo2(self):
# code that reads in file2 and expected_file_2
try:
self.assert(expected_output2, output2)
except AssertionError:
global keep_files
keep_files = True
raise

You could simply check, if there were any errors/failures in your test case during tear-down and only delete the files, if there were none.
How to perform this check was explained in this post.
This check is done on a TestCase instance so tearDownClass won't work. But you are using different files in different tests anyway, so you might as well use normal setUp/tearDown to remove the current file.
Here is a working example:
from pathlib import Path
from typing import Optional
from unittest import TestCase
class Test(TestCase):
def all_tests_passed(self) -> bool:
"""Returns `True` if no errors/failures occurred at the time of calling."""
outcome = getattr(self, "_outcome")
if hasattr(outcome, "errors"): # Python <=3.10
result = self.defaultTestResult()
getattr(self, "_feedErrorsToResult")(result, outcome.errors)
else: # Python >=3.11
result = outcome.result
return all(test != self for test, _ in result.errors + result.failures)
def setUp(self) -> None:
super().setUp()
self.test_file: Optional[Path] = None
def tearDown(self) -> None:
super().tearDown()
if self.test_file and self.all_tests_passed():
self.test_file.unlink()
def test_foo(self) -> None:
self.test_file = Path("foo.txt")
self.test_file.touch()
self.assertTrue(True)
def test_bar(self) -> None:
self.test_file = Path("bar.txt")
self.test_file.touch()
self.assertTrue(False)
Running this test case leaves bar.txt in the current working directory, whereas foo.txt is gone.

unit test of method of class with mocking data in python

I have some class SomeClass in file prog.py
class SomeClass:
def __init__(self, file_name):
self.file_name = file_name
self.data_dict = {}
def load(self):
"""
"""
if not os.path.exists(self.file_name):
return False
with open(self.file_name, "r", encoding='iso-8859-1') as file:
skip_lines(4, file)
reader = csv.DictReader(file)
for line in reader:
if line['Region'] == "":#to remove Copiright and disclaimer text at the end
break
label=re.sub('\s+/\s+',"/",line['Region']) #remove spaces around /
if label in self.data_dict.keys():
#if line['Region'] in self.data_dict.keys():
self.data_dict[label][line['Range']] = int(line['All people'])
#self.data_dict[line['Region']][line['Range']] = line['All people']
else:
self.data_dict[label] = {line['Range']:int(line['All people'])}
#self.data_dict[line['Region']] = {line['Range']:line['All people']}
return True
def regions(self):
return list(self.data_dict.keys())
I need to:
Create a file named test_someclass_data.py.
Use import to import the prog module.
In the test_someclass_data.py file, create a unit test class named
TestSomeClass that inherits from unittest.TestCase.
The unit test class should include member functions to test the
regions member function of the SomeClass class.
My code of test_someclass_data.py:
import unittest
import prog
class TestSomeClass(unittest.TestCase):
def test_regions(self):
"""
Test regions function of the SomeClass class.
"""
prog.SomeClass.data_dict ={"key1":"val1","key2":"val2"}
test_output = prog.SomeClass.regions()
expected_output =["key1", "key2"]
self.assertEqual(test_output, expected_output)
I need to test a function of the SomeClass, so I have to create an instance of the class, call method call to store data, then I can compare outputs. Could you advise me, please, how to avoid this and test functions through mocking? Thank you.

Reset class and class variables for each test in Python via pytest

I created a class to make my life easier while doing some integration tests involving workers and their contracts. The code looks like this:
class ContractID(str):
contract_counter = 0
contract_list = list()
def __new__(cls):
cls.contract_counter += 1
new_entry = super().__new__(cls, f'Some_internal_name-{cls.contract_counter:10d}')
cls.contract_list.append(new_entry)
return new_entry
#classmethod
def get_contract_no(cls, worker_number):
return cls.contract_list[worker_number-1] # -1 so WORKER1 has contract #1 and not #0 etc.
When I'm unit-testing the class, I'm using the following code:
from test_helpers import ContractID
#pytest.fixture
def get_contract_numbers():
test_string_1 = ContractID()
test_string_2 = ContractID()
test_string_3 = ContractID()
return test_string_1, test_string_2, test_string_3
def test_contract_id(get_contract_numbers):
assert get_contract_ids[0] == 'Some_internal_name-0000000001'
assert get_contract_ids[1] == 'Some_internal_name-0000000002'
assert get_contract_ids[2] == 'Some_internal_name-0000000003'
def test_contract_id_get_contract_no(get_contract_numbers):
assert ContractID.get_contract_no(1) == 'Some_internal_name-0000000001'
assert ContractID.get_contract_no(2) == 'Some_internal_name-0000000002'
assert ContractID.get_contract_no(3) == 'Some_internal_name-0000000003'
with pytest.raises(IndexError) as py_e:
ContractID.get_contract_no(4)
assert py_e.type == IndexError
However, when I try to run these tests, the second one (test_contract_id_get_contract_no) fails, because it does not raise the error as there are more than three values. Furthermore, when I try to run all my tests in my folder test/, it fails even the first test (test_contract_id), which is probably because I'm trying to use this function in other tests that run before this test.
After reading this book, my understanding of fixtures was that it provides objects as if they were never called before, which is obviously not the case here. Is there a way how to tell the tests to use the class as if it hasn't been used before anywhere else?

If I understand that correctly, you want to run the fixture as setup code, so that your class has exactly 3 instances. If the fixture is function-scoped (the default) it is indeed run before each test, which will each time create 3 new instances for your class. If you want to reset your class after the test, you have to do this yourself - there is no way pytest can guess what you want to do here.
So, a working solution would be something like this:
#pytest.fixture(autouse=True)
def get_contract_numbers():
test_string_1 = ContractID()
test_string_2 = ContractID()
test_string_3 = ContractID()
yield
ContractID.contract_counter = 0
ContractID.contract_list.clear()
def test_contract_id():
...
Note that I did not yield the test strings, as you don't need them in the shown tests - if you need them, you can yield them, of course. I also added autouse=True, which makes sense if you need this for all tests, so you don't have to reference the fixture in each test.
Another possibility would be to use a session-scoped fixture. In this case the setup would be done only once. If that is what you need, you can use this instead:
#pytest.fixture(autouse=True, scope="session")
def get_contract_numbers():
test_string_1 = ContractID()
test_string_2 = ContractID()
test_string_3 = ContractID()
yield

How to run unittest test cases in the order they are declared

I fully realize that the order of unit tests should not matter. But these unit tests are as much for instructional use as for actual unit testing, so I would like the test output to match up with the test case source code.
I see that there is a way to set the sort order by setting the sortTestMethodsUsing attribute on the test loader. The default is a simple cmp() call to lexically compare names. So I tried writing a cmp-like function that would take two names, find their declaration line numbers and them return the cmp()-equivalent of them:
import unittest
class TestCaseB(unittest.TestCase):
def test(self):
print("running test case B")
class TestCaseA(unittest.TestCase):
def test(self):
print("running test case A")
import inspect
def get_decl_line_no(cls_name):
cls = globals()[cls_name]
return inspect.getsourcelines(cls)[1]
def sgn(x):
return -1 if x < 0 else 1 if x > 0 else 0
def cmp_class_names_by_decl_order(cls_a, cls_b):
a = get_decl_line_no(cls_a)
b = get_decl_line_no(cls_b)
return sgn(a - b)
unittest.defaultTestLoader.sortTestMethodsUsing = cmp_class_names_by_decl_order
unittest.main()
When I run this, I get this output:
running test case A
.running test case B
.
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
indicating that the test cases are not running in the declaration order.
My sort function is just not being called, so I suspect that main() is building a new test loader, which is wiping out my sort function.

The solution is to create a TestSuite explicitly, instead of letting unittest.main() follow all its default test discovery and ordering behavior. Here's how I got it to work:
import unittest
class TestCaseB(unittest.TestCase):
def runTest(self):
print("running test case B")
class TestCaseA(unittest.TestCase):
def runTest(self):
print("running test case A")
import inspect
def get_decl_line_no(cls):
return inspect.getsourcelines(cls)[1]
# get all test cases defined in this module
test_case_classes = list(filter(lambda c: c.__name__ in globals(),
unittest.TestCase.__subclasses__()))
# sort them by decl line no
test_case_classes.sort(key=get_decl_line_no)
# make into a suite and run it
suite = unittest.TestSuite(cls() for cls in test_case_classes)
unittest.TextTestRunner().run(suite)
This gives the desired output:
running test case B
.running test case A
.
----------------------------------------------------------------------
Ran 2 tests in 0.000s
OK
It is important to note that the test method in each class must be named runTest.

You can manually build a TestSuite where your TestCases and all tests inside them run by line number:
# Python 3.8.3
import unittest
import sys
import inspect
def isTestClass(x):
return inspect.isclass(x) and issubclass(x, unittest.TestCase)
def isTestFunction(x):
return inspect.isfunction(x) and x.__name__.startswith("test")
class TestB(unittest.TestCase):
def test_B(self):
print("Running test_B")
self.assertEqual((2+2), 4)
def test_A(self):
print("Running test_A")
self.assertEqual((2+2), 4)
def setUpClass():
print("TestB Class Setup")
class TestA(unittest.TestCase):
def test_A(self):
print("Running test_A")
self.assertEqual((2+2), 4)
def test_B(self):
print("Running test_B")
self.assertEqual((2+2), 4)
def setUpClass():
print("TestA Class Setup")
def suite():
# get current module object
module = sys.modules[__name__]
# get all test className,class tuples in current module
testClasses = [
tup for tup in
inspect.getmembers(module, isTestClass)
]
# sort classes by line number
testClasses.sort(key=lambda t: inspect.getsourcelines(t[1])[1])
testSuite = unittest.TestSuite()
for testClass in testClasses:
# get list of testFunctionName,testFunction tuples in current class
classTests = [
tup for tup in
inspect.getmembers(testClass[1], isTestFunction)
]
# sort TestFunctions by line number
classTests.sort(key=lambda t: inspect.getsourcelines(t[1])[1])
# create TestCase instances and add to testSuite;
for test in classTests:
testSuite.addTest(testClass[1](test[0]))
return testSuite
if __name__ == '__main__':
runner = unittest.TextTestRunner()
runner.run(suite())
Output:
TestB Class Setup
Running test_B
.Running test_A
.TestA Class Setup
Running test_A
.Running test_B
.
----------------------------------------------------------------------
Ran 4 tests in 0.000s
OK

As stated in the name, sortTestMethodsUsing is used to sort test methods. It is not used to sort classes. (It is not used to sort methods in different classes either; separate classes are handled separately.)
If you had two test methods in the same class, sortTestMethodsUsing would be used to determine their order. (At that point, you would get an exception because your function expects class names.)

Is there a way to pass/inject test parameters from outside

Lets say I have the following test class in a tests.py:
class MyTest(unittest.TestCase):
#classmethod
def setUpClass(cls, ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"):
cls.driver = get_remote_webdriver(ip, port, browserType)
cls.driver.implicitly_wait(30)
cls.h5_client_url = h5_client_url
cls.h5_username = h5_username
cls.h5_password = h5_password
#classmethod
def tearDownClass(cls):
cls.driver.quit()
def test_01(self):
# test code
def test_02(self):
# test code
...
def test_N(self):
# test code
All my tests (test_01 to test_N) use the parameters, provided in the setUpClass. Those parameters have default values:
ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"
So I wonder if I can inject new values for those parameters. And I want to do it from another python script so there will be no changes or just minor changes to code of the tests.
Note: I want to run my tests by a batch/shell command and save the output of the test to a log file (to redirect the standard output to that log file)
One think I did was to create a function decorator, that passes a dictionary with key=parameter_name and value=parameter_new_value, but I had to write to much additional code in the tests.py:
I defined the function_decorator logic
I put that #function_decorator annotation above every function I want to decorate
That function decorator needs that dictionary as a parameter, so I made a main, that looks something like that:
if __name__ == '__main__':
# terminal command to run tests should look like this /it is executed by the run-test PARROT command/
# python [this_module_name] [dictionary_containing_parameters] [log_file.log] *[tests]
parser = argparse.ArgumentParser()
# add testbeds_folder as scripts' first parameter, test_log_file as second and tests as the rest
parser.add_argument('dictionary_containing_parameters')
parser.add_argument('test_log_file')
parser.add_argument('unittest_args', nargs='*')
args = parser.parse_args()
dictionary_containing_parameters = sys.argv[1]
test_log_file = sys.argv[2]
# removes the "dictionary_containing_parameters" and "test_log_file" from sys.args - otherwise an error occurs unittest TestRunner
sys.argv[1:] = args.unittest_args
# executes the test/tests and save the output to the test_log_file
with open(test_log_file, "w") as f:
runner = unittest.TextTestRunner(f)
unittest.main(defaultTest=sys.argv[1:], exit=False, testRunner=runner)

Here is one possible solution:
You run your test from a different module this way:
if __name__ == '__main__':
testbed_dict = {"ip": "11.111.111.112",
"browserType": "Chrome",
"port": "4444",
"h5_client_url": "https://new_somelink.com/",
"h5_username": "new_username",
"h5_password": "new_pass"}
sys.argv.append(testbed_dict)
from your_tests_module import *
with open("test.log", "w") as f:
runner = unittest.TextTestRunner(f)
unittest.main(argv=[sys.argv[0]], defaultTest='test_class.test_name', exit=False, testRunner=runner)
You can nottice that argv=[sys.argv[0]] in unittest.main(argv=[sys.argv[0]], defaultTest='test_class.test_name', exit=False, testRunner=runner). Doing that you change the unittests argument to one (no error occurs) to a list with your real arguments. Note that at the end of this list is the dictionary with the new values of test parameters.
Ok, now you write an function decorator, that should look like this:
def load_params(system_arguments_list):
def decorator(func_to_decorate):
#wraps(func_to_decorate)
def wrapper(self, *args, **kwargs):
kwargs = system_arguments_list[-1]
return func_to_decorate(self, **kwargs)
return wrapper
return decorator
And use this decorator this way:
#classmethod
#load_params(sys.argv)
def setUpClass(cls, ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"):
cls.driver = get_remote_webdriver(ip, port, browserType)
cls.driver.implicitly_wait(30)
cls.h5_client_url = h5_client_url
cls.h5_username = h5_username