I'm using python unittest for functions that write data to JSON. I use tearDownClass to delete the output test files so they don't clutter the local repo. Ground truths are also stored as JSON files.
I do want to store the output test files when tests fail, so its easier for troubleshooting.
My current implementation is to use a global boolean keep_file = False. When the unittest fails the assertion, it modifies keep_file = True. tearDownClass only deletes the files when keep_file == False. I don't like the idea of modifying global variables and the try exception blocks for each assert.
import json
import os
import unittest
from src.mymodule import foo1, foo2
# These are defined outside the class on purpose so the classmethods can access them
FILE_1 = "unittest.file1.json"
EXPECTED_FILE_1 = "expected.file1.json"
FILE_2 = "unittest.file2.json"
EXPECTED_FILE_2 = "expected.file2.json"
keep_files = False
class TestRhaPostPayload(unittest.TestCase):
#classmethod
def setUpClass(cls):
cls.get_file1()
cls.get_file2()
#classmethod
def get_file1(cls):
output1 = foo1()
with open(FILE_1, "w") as f:
f.write(output1)
#classmethod
def get_file2(cls):
output2 = foo1()
with open(FILE_2, "w") as f:
f.write(output2)
#classmethod
def tearDownClass(cls):
if not keep_files:
os.remove(FILE_1)
os.remove(FILE_2)
def test_foo1(self):
# code that reads in file1 and expected_file_1
try:
self.assert(expected_output1, output1)
except AssertionError:
global keep_files
keep_files = True
raise
def test_foo2(self):
# code that reads in file2 and expected_file_2
try:
self.assert(expected_output2, output2)
except AssertionError:
global keep_files
keep_files = True
raise
You could simply check, if there were any errors/failures in your test case during tear-down and only delete the files, if there were none.
How to perform this check was explained in this post.
This check is done on a TestCase instance so tearDownClass won't work. But you are using different files in different tests anyway, so you might as well use normal setUp/tearDown to remove the current file.
Here is a working example:
from pathlib import Path
from typing import Optional
from unittest import TestCase
class Test(TestCase):
def all_tests_passed(self) -> bool:
"""Returns `True` if no errors/failures occurred at the time of calling."""
outcome = getattr(self, "_outcome")
if hasattr(outcome, "errors"): # Python <=3.10
result = self.defaultTestResult()
getattr(self, "_feedErrorsToResult")(result, outcome.errors)
else: # Python >=3.11
result = outcome.result
return all(test != self for test, _ in result.errors + result.failures)
def setUp(self) -> None:
super().setUp()
self.test_file: Optional[Path] = None
def tearDown(self) -> None:
super().tearDown()
if self.test_file and self.all_tests_passed():
self.test_file.unlink()
def test_foo(self) -> None:
self.test_file = Path("foo.txt")
self.test_file.touch()
self.assertTrue(True)
def test_bar(self) -> None:
self.test_file = Path("bar.txt")
self.test_file.touch()
self.assertTrue(False)
Running this test case leaves bar.txt in the current working directory, whereas foo.txt is gone.
Related
I would like to store that specific tests failed, then pass that info via API when test class is finished.
I tried sth like that:
fails = []
#pytest.fixture(scope='function')
def something(request):
yield
if request.session.testsfailed:
print("I failed")
fails.append(request.node.name)
print('FAILED', fails)
class TestLala:
#pytest.mark.order(1)
def test_test1(self, something):
assert False
#pytest.mark.order(3)
def test_test1(self, something):
assert True
#pytest.mark.order(3)
def test_test3(self, something):
assert 4 == 4
but instead of failed tests I am still getting each test name added to the list.
The pytest_runtest_makereport hook should solve it for you. Add this to your conftest.py
import pytest
pytest.failed_nodes = []
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item):
output = yield
report = output.get_result()
if report.failed:
node_id = report.nodeid
pytest.failed_nodes.append(node_id)
##EDIT##
The answer #Teejay Bruno gave shows how you can avoid having the push to your API run after every function. Just send the data to a list of dict after every test. Then call the send_data when the test is over to send to your API
I have done something similar in my work. By using conftest with make_report you are able to capture the test results and other meta data, then do as you please with that data (like send to a database or API).
test.py
import pytest
class TestLala:
#pytest.mark.order(1)
def test_test1(self):
assert False
#pytest.mark.order(3)
def test_test1(self):
assert True
#pytest.mark.order(3)
def test_test3(self):
assert 4 == 4
conftest.py
import pytest
test = None
status_tag = None
line = None
duration = None
exception = None
#pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
global test, status_tag, line, duration, exception
report = yield
result = report.get_result()
if result.when == 'call':
(filename, line, name) = item.location
test = item.nodeid
status_tag = result.outcome
line = line
duration = call.duration
exception = call.excinfo
#pytest.fixture(scope='function', autouse=True)
def send_data(pytestconfig):
yield
global test, status_tag, line, duration, exception
# This is where you can send the data to your API
# This will run after every test so if you dont want to send the data as it comes in, you will need to change
# how this function and the one above work a little
print(f"TEST: {test}")
print(f"STATUS_TAG: {status_tag}")
print(f"LINE: {line}")
print(f"DURATION: {duration}")
print(f"EXCEPTION: {exception}")
test = None
status_tag = None
line = None
duration = None
exception = None
If you have not worked with conftest before see the below link:
https://docs.pytest.org/en/6.2.x/fixture.html
Search for the section titled -> "conftest.py: sharing fixtures across multiple files"
Sorry if this is a basic concept in pytest but I'm a beginner in pytest and I have yet to find an answer for this. I have a python file that has several functions (code_to_be_tested.py). Output of each function is needed by the next function to work. I want to use pytest to test these functions (test_code.py). I want to use each test function in test_code.py to test 1 function in code_to_be_tested.py but I cannot figure out a way to provide output of each function inside a test function to next test function.
I'll try to explain what I mean by below code examples,
# code_to_be_tested.py
def func1(nm=""):
file_name = "".join(["file", nm, ".txt"])
open(file_name, "w")
return file_name
def func2(filename):
path = "/home/"
return "".join([path, filename])
x = func1()
y = func2(x)
# test_code.py
from code_to_be_tested import func1, func2
import pytest
def test_func1():
try:
e = func1("one")
assert isinstance(e, str) is True
except AssertionError as asErr:
raise asErr
def test_func2():
try:
f = func2()
assert isinstance(f, str) is True
except AssertionError as asErr:
raise asErr
I understand that I can use fixture as shown below but it creates duplicate files and outputs.
# test_code.py
from code_to_be_tested import func1, func2
import pytest
def test_func1():
try:
e = func1("one")
assert isinstance(e, str) is True
except AssertionError as asErr:
raise asErr
#pytest.fixture
def fix1():
out = func1()
return out
def test_func2(fix1):
try:
f = func2(fix1)
assert isinstance(f, str) is True
except AssertionError as asErr:
raise asErr
I'd like to be able to return "e" local variable from test_func1() and "f" from test_func2() and use them in other test functions. Please let me know if there's anyway I can achieve this.
Lets say I have the following test class in a tests.py:
class MyTest(unittest.TestCase):
#classmethod
def setUpClass(cls, ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"):
cls.driver = get_remote_webdriver(ip, port, browserType)
cls.driver.implicitly_wait(30)
cls.h5_client_url = h5_client_url
cls.h5_username = h5_username
cls.h5_password = h5_password
#classmethod
def tearDownClass(cls):
cls.driver.quit()
def test_01(self):
# test code
def test_02(self):
# test code
...
def test_N(self):
# test code
All my tests (test_01 to test_N) use the parameters, provided in the setUpClass. Those parameters have default values:
ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"
So I wonder if I can inject new values for those parameters. And I want to do it from another python script so there will be no changes or just minor changes to code of the tests.
Note: I want to run my tests by a batch/shell command and save the output of the test to a log file (to redirect the standard output to that log file)
One think I did was to create a function decorator, that passes a dictionary with key=parameter_name and value=parameter_new_value, but I had to write to much additional code in the tests.py:
I defined the function_decorator logic
I put that #function_decorator annotation above every function I want to decorate
That function decorator needs that dictionary as a parameter, so I made a main, that looks something like that:
if __name__ == '__main__':
# terminal command to run tests should look like this /it is executed by the run-test PARROT command/
# python [this_module_name] [dictionary_containing_parameters] [log_file.log] *[tests]
parser = argparse.ArgumentParser()
# add testbeds_folder as scripts' first parameter, test_log_file as second and tests as the rest
parser.add_argument('dictionary_containing_parameters')
parser.add_argument('test_log_file')
parser.add_argument('unittest_args', nargs='*')
args = parser.parse_args()
dictionary_containing_parameters = sys.argv[1]
test_log_file = sys.argv[2]
# removes the "dictionary_containing_parameters" and "test_log_file" from sys.args - otherwise an error occurs unittest TestRunner
sys.argv[1:] = args.unittest_args
# executes the test/tests and save the output to the test_log_file
with open(test_log_file, "w") as f:
runner = unittest.TextTestRunner(f)
unittest.main(defaultTest=sys.argv[1:], exit=False, testRunner=runner)
Here is one possible solution:
You run your test from a different module this way:
if __name__ == '__main__':
testbed_dict = {"ip": "11.111.111.112",
"browserType": "Chrome",
"port": "4444",
"h5_client_url": "https://new_somelink.com/",
"h5_username": "new_username",
"h5_password": "new_pass"}
sys.argv.append(testbed_dict)
from your_tests_module import *
with open("test.log", "w") as f:
runner = unittest.TextTestRunner(f)
unittest.main(argv=[sys.argv[0]], defaultTest='test_class.test_name', exit=False, testRunner=runner)
You can nottice that argv=[sys.argv[0]] in unittest.main(argv=[sys.argv[0]], defaultTest='test_class.test_name', exit=False, testRunner=runner). Doing that you change the unittests argument to one (no error occurs) to a list with your real arguments. Note that at the end of this list is the dictionary with the new values of test parameters.
Ok, now you write an function decorator, that should look like this:
def load_params(system_arguments_list):
def decorator(func_to_decorate):
#wraps(func_to_decorate)
def wrapper(self, *args, **kwargs):
kwargs = system_arguments_list[-1]
return func_to_decorate(self, **kwargs)
return wrapper
return decorator
And use this decorator this way:
#classmethod
#load_params(sys.argv)
def setUpClass(cls, ip="11.111.111.111",
browserType="Chrome",
port="4444",
h5_client_url="https://somelink.com/",
h5_username="username",
h5_password="pass"):
cls.driver = get_remote_webdriver(ip, port, browserType)
cls.driver.implicitly_wait(30)
cls.h5_client_url = h5_client_url
cls.h5_username = h5_username
I can't seem to get my head around mocking in Python. I have a global function:
a.py:
def has_permission(args):
ret_val = ...get-true-or-false...
return ret_val
b.py:
class MySerializer(HyperlinkedModelSerializer):
def get_fields():
fields = super().get_fields()
for f in :
if has_permission(...):
ret_val[f.name] = fields[f]
return ret_val
c.py:
class CountrySerializer(MySerializer):
class Meta:
model = Country
Question: Now i want to test c.py, but i want to mock the has_permission function that is defined in a.py, but is called in the get_fields-method of the class MySerializer that is defined in b.py ... How do i do that?
I've tried things like:
#patch('b.MySerializer.has_permission')
and
#patch('b.MySerializer.get_fields.has_permission')
and
#patch('a.has_permission')
But everything i try either just doesn't work and has_permission is still executed, or python complains about that it can't find the attribute 'has_permission'
with the patching done in:
test.py
class TestSerializerFields(TestCase):
#patch(... the above examples....)
def test_my_country_serializer():
s = CountrySerializer()
self..assertTrue(issubclass(my_serializer_fields.MyCharField, type(s.get_fields()['field1'])))
You need to patch the global in the b module:
#patch('b.has_permission')
because that's where your code looks for it.
Also see the Where to patch section of the mock documentation.
You need to patch the method where it exists at the time your test runs. If you try and patch the method where it is defined after the test code has already imported it, then the patch will have no effect. At the point where the #patch(...) executes, the test code under test has already grabbed the global method into its own module.
Here is an example:
app/util/config.py:
# This is the global method we want to mock
def is_search_enabled():
return True
app/service/searcher.py:
# Here is where that global method will be imported
# when this file is first imported
from app.util.config import is_search_enabled
class Searcher:
def __init__(self, api_service):
self._api_service = api_service
def search(self):
if not is_search_enabled():
return None
return self._api_service.perform_request('/search')
test/service/test_searcher.py:
from unittest.mock import patch, Mock
# The next line will cause the imports of `searcher.py` to execute...
from app.service.searcher import Searcher
# At this point, searcher.py has imported is_search_enabled into its module.
# If you later try and patch the method at its definition
# (app.util.config.is_search_enabled), it will have no effect because
# searcher.py won't look there again.
class MockApiService:
pass
class TestSearcher:
# By the time this executes, `is_search_enabled` has already been
# imported into `app.service.searcher`. So that is where we must
# patch it.
#patch('app.service.searcher.is_search_enabled')
def test_no_search_when_disabled(self, mock_is_search_enabled):
mock_is_search_enabled.return_value = False
mock_api_service = MockApiService()
mock_api_service.perform_request = Mock()
searcher = Searcher(mock_api_service)
results = searcher.search()
assert results is None
mock_api_service.perform_request.assert_not_called()
# (For completeness' sake, make sure the code actually works when search is enabled...)
def test_search(self):
mock_api_service = MockApiService()
mock_api_service.perform_request = mock_perform_request = Mock()
searcher = Searcher(mock_api_service)
expected_results = [1, 2, 3]
mock_perform_request.return_value = expected_results
actual_results = searcher.search()
assert actual_results == expected_results
mock_api_service.perform_request.assert_called_once_with('/search')
py.test documentations says that I should add capsys parameter to my test methods but in my case this doesn't seem to be possible.
class testAll(unittest.TestCase):
def setUp(self):
self.cwd = os.path.abspath(os.path.split(inspect.getfile(inspect.currentframe()))[0])
os.chdir(self.cwd)
def execute(self, cmd, result=0):
"""
Helper method used by many other tests, that would prevent replicating too much code.
"""
# cmd = "%s > /dev/null 2>&1" % cmd
ret = os.system(cmd) >> 8
self.assertEqual(ret, result, "`%s` returned %s instead of %s (cws=%s)\n\t%s" % (cmd, ret, result, os.getcwd(), OUTPUT)) ### << how to access the output from here
def test_1(self):
self.execute("do someting", 0)
You could define a helper function in the class that inherits the capsys fixture:
#pytest.fixture(autouse=True)
def capsys(self, capsys):
self.capsys = capsys
Then call this function inside the test:
out,err = self.capsys.readouterr()
assert out == 'foobar'
Kudos to Michał Krassowski for his workaround which helped me work through a similar problem.
https://github.com/pytest-dev/pytest/issues/2504#issuecomment-309475790
Thomas Wright's answer is perfect. I'm just sticking this code block here for my own reference as my search led me here and I'll likely forget this in future! [doing a few things in this so useful reference for me]. If anyone is looking and sees where it can be improved - suggest away!
import os
import pytest
from _pytest.monkeypatch import MonkeyPatch
from unittest import TestCase
# -----------------------------------------------------------------------------
def foo_under_test(inp1):
"""Example of a Method under test"""
do_some_calcs_here = inp1*2
get_a_return = ClassCalled.foo_called(do_some_calcs_here)
return get_a_return
# -----------------------------------------------------------------------------
class ClassUnderTest():
"""Example of a Class contained Method under test"""
def __init__(self):
"""Instantiate the class"""
self.var1 = "TestVar"
def foo_under_test2(self, inp11):
"""The class method under test"""
return self.var1 + self.foo_called2(inp11)
def foo_called2(self, inp12):
"""Nominal sub-foo to foo_under_test2"""
return str(inp12*5)
# -----------------------------------------------------------------------------
class ClassCalled:
"""Example of a class that could be called by foo_under_test"""
def foo_called(inp2):
"""Sub-foo to foo_under_test"""
return inp2 * 2
# -----------------------------------------------------------------------------
class MockResponses:
"""Class for holding the mock responses"""
def foo_called(inp2):
"""**Mock of foo_called**"""
return inp2*3
def foo_called2(inp12):
"""**Mock of foo_called**"""
return str(inp12*4)
# -----------------------------------------------------------------------------
class Test_foo_under_test(TestCase):
"""Test class - means of grouping up tests for a target function
This one is addressing the individual function (not within a class)
"""
# ---------------------------------------------------------------
#pytest.fixture(autouse=True)
def capsys(self, capsys):
"""Capsys hook into this class"""
self.capsys = capsys
def print_to_console(self, strOut):
"""Print strOut to console (even within a pyTest execution)"""
with self.capsys.disabled():
print(strOut)
def setUp(self):
"""Ran by pyTest before running any test_*() functions"""
self.monkeypatch = MonkeyPatch()
# ---------------------------------------------------------------
def test_1(self):
"""**Test case**"""
def mock_foo_called(inp2):
return MockResponses.foo_called(inp2)
mockedFoo = ClassCalled.foo_called # Need to get this handle here
self.monkeypatch.setattr(ClassCalled, "foo_called", mock_foo_called)
x = foo_under_test(1)
self.print_to_console("\n")
strOut = "Rtn from foo: " + str(x)
self.print_to_console(strOut)
assert x == 6
# Manually clear the monkey patch
self.monkeypatch.setattr(
ClassCalled, "foo_called", mockedFoo)
"""I've noticed with me having monkeypatch inside the
class, the damn thing persists across functions.
This is the only workaround I've found so far"""
# -----------------------------------------------------------------------------
class Test_ClassUnderTest_foo_under_test(TestCase):
"""Test class - means of grouping up tests for a target function
This one is addressing the function within a class
"""
# ---------------------------------------------------------------
#pytest.fixture(autouse=True)
def capsys(self, capsys):
"""Capsys hook into this class"""
self.capsys = capsys
def print_to_console(self, strOut):
"""Print strOut to console (even within a pyTest execution)"""
with self.capsys.disabled():
print(strOut)
def setUp(self):
"""Ran by pyTest before running any test_*() functions"""
self.monkeypatch = MonkeyPatch()
# ---------------------------------------------------------------
def test_1(self):
"""**Test case**"""
def mock_foo_called2(self, inp2):
"""
Mock function
Defining a mock function, note this can be dealt with directly
here, or if its more comprehensible, put it in a separate class
(i.e. MockResponses)
"""
# return MockResponses.foo_called2(inp2) # Delegated approach
return str(inp2*4) # Direct approach
"""Note that the existence of self within this test class forces
a wrapper around calling a MockClass - so we have to go through
both the line below and the function mock_foo_called2() above to
properly invoke MockResponses.foo_called2()
"""
mockedFoo = ClassUnderTest.foo_called2
self.monkeypatch.setattr(
ClassUnderTest, "foo_called2", mock_foo_called2)
x = ClassUnderTest().foo_under_test2(1)
strOut = "Rtn from foo: " + str(x)
self.print_to_console("\n")
self.print_to_console(strOut)
assert x == "TestVar" + str(4)
self.monkeypatch.setattr(
ClassUnderTest, "foo_called2", mockedFoo)
# -----------------------------------------------------------------------------
# ---- Main
if __name__ == "__main__":
#
# Setup for pytest
outFileName = os.path.basename(__file__)[:-3] # Remove the .py from end
currScript = os.path.basename(__file__)
# -------------------------------------------------------------------------
# PyTest execution
pytest.main([currScript, "--html", outFileName + "_report.html"])
rtnA = foo_under_test(1)
print(rtnA == 4)
# This should output 4, demonstrating effect of stub (which produced 6)
rtnB = ClassUnderTest().foo_under_test2(1)
print(rtnB == "TestVar"+str(5))
# This should output "TestVar5", demonstrating effect of stub
# conftest.py
class TTY:
def communicate(self):
with self.trace():
print('wow!')
#pytest.fixture(autouse=True)
def set_capsys(capsys):
TTY.trace = capsys.disabled
#pytest.fixture
def get_tty():
_get_tty():
return TTY()
return _get_tty
# test_wow.py
def test_wow(get_tty):
get_tty().communicate()