I'm trying to mock subprocess.Popen. When I run the following code however, the mock is completely ignored and I'm not sure why
Test Code:
def test_bring_connection_up(self):
# All settings should either overload the update or the run method
mock_popen = MagicMock()
mock_popen.return_value = {'communicate': (lambda: 'hello','world')}
with patch('subprocess.Popen', mock_popen):
self.assertEqual(network_manager.bring_connection_up("test"), "Error: Unknown connection: test.\n")
Module Code:
from subprocess import Popen, PIPE
# ........
def list_connections():
process = Popen(["nmcli", "-t", "-fields", "NAME,TYPE", "con", "list"], stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate() # <--- Here's the failure
return stdout
You're not patching in the right place. You patch where Popen is defined:
with patch('subprocess.Popen', mock_popen):
You need to patch where Popen is imported, i.e. in the "Module Code" where you have written this line:
from subprocess import Popen, PIPE
I.e., it should look something like:
with patch('myapp.mymodule.Popen', mock_popen):
For a quick guide, read the section in the docs: Where to patch.
I'm trying to mock subprocess.Popen. When I run the following code however, the mock is completely ignored and I'm not sure why
Test Code:
def test_bring_connection_up(self):
# All settings should either overload the update or the run method
mock_popen = MagicMock()
mock_popen.return_value = {'communicate': (lambda: 'hello','world')}
with patch('subprocess.Popen', mock_popen):
self.assertEqual(network_manager.bring_connection_up("test"), "Error: Unknown connection: test.\n")
Module Code:
from subprocess import Popen, PIPE
# ........
def list_connections():
process = Popen(["nmcli", "-t", "-fields", "NAME,TYPE", "con", "list"], stdout=PIPE, stderr=PIPE)
stdout, stderr = process.communicate() # <--- Here's the failure
return stdout
You're not patching in the right place. You patch where Popen is defined:
with patch('subprocess.Popen', mock_popen):
You need to patch where Popen is imported, i.e. in the "Module Code" where you have written this line:
from subprocess import Popen, PIPE
I.e., it should look something like:
with patch('myapp.mymodule.Popen', mock_popen):
For a quick guide, read the section in the docs: Where to patch.
I have a method - run_script() - I would like to test. Specifically, I want to test that a call to subprocess.Popen occurs. It would be even better to test that subprocess.Popen is called with certain parameters. When I run the test however I get TypeError: 'tuple' object is not callable.
How can I test my method to ensure that subprocess is actually being called using mocks?
#mock.patch("subprocess.Popen")
def run_script(file_path):
process = subprocess.Popen(["myscript", -M, file_path], stdout=subprocess.PIPE)
output, err = process.communicate()
return process.returncode
def test_run_script(self, mock_subproc_popen):
mock_subproc_popen.return_value = mock.Mock(
communicate=("ouput", "error"), returncode=0
)
am.account_manager("path")
self.assertTrue(mock_subproc_popen.called)
It seems unusual to me that you use the patch decorator over the run_script function, since you don't pass a mock argument there.
How about this:
from unittest import mock
import subprocess
def run_script(file_path):
process = subprocess.Popen(["myscript", -M, file_path], stdout=subprocess.PIPE)
output, err = process.communicate()
return process.returncode
#mock.patch("subprocess.Popen")
def test_run_script(self, mock_subproc_popen):
process_mock = mock.Mock()
attrs = {"communicate.return_value": ("output", "error")}
process_mock.configure_mock(**attrs)
mock_subproc_popen.return_value = process_mock
am.account_manager("path") # this calls run_script somewhere, is that right?
self.assertTrue(mock_subproc_popen.called)
Right now, your mocked subprocess.Popen seems to return a tuple, causeing process.communicate() to raise TypeError: 'tuple' object is not callable.. Therefore it's most important to get the return_value on mock_subproc_popen just right.
The testfixtures library (docs, github) can mock the subprocess package.
Here's an example on using the mock subprocess.Popen:
from unittest import TestCase
from testfixtures.mock import call
from testfixtures import Replacer, ShouldRaise, compare
from testfixtures.popen import MockPopen, PopenBehaviour
class TestMyFunc(TestCase):
def setUp(self):
self.Popen = MockPopen()
self.r = Replacer()
self.r.replace(dotted_path, self.Popen)
self.addCleanup(self.r.restore)
def test_example(self):
# set up
self.Popen.set_command("svn ls -R foo", stdout=b"o", stderr=b"e")
# testing of results
compare(my_func(), b"o")
# testing calls were in the right order and with the correct parameters:
process = call.Popen(["svn", "ls", "-R", "foo"], stderr=PIPE, stdout=PIPE)
compare(Popen.all_calls, expected=[process, process.communicate()])
def test_example_bad_returncode(self):
# set up
Popen.set_command("svn ls -R foo", stdout=b"o", stderr=b"e", returncode=1)
# testing of error
with ShouldRaise(RuntimeError("something bad happened")):
my_func()
If you want to check that the mocked object was called with a certain parameter, you can add the side_effect argument to the mock.patch decorator.
The return value of the side_effect function determines the return value of subprocess.Popen. If the side_effect_func returns DEFAULT, subprocess.Popen will be called in a normal way.
from unittest import mock, TestCase
from unittest.mock import DEFAULT
import subprocess
def run_script(script_path, my_arg):
process = subprocess.Popen([script_path, my_arg])
return process
def side_effect_func(*args, **kwargs):
# Print the arguments
print(args)
# If 'bar' is contained within the arguments, return 'foo'
if any(['bar' in arg for arg in args]):
return 'foo'
# If 'bar' is not contained within the arguments, run subprocess.Popen
else:
return DEFAULT
class TestRunScriptClass(TestCase):
#mock.patch("subprocess.Popen", side_effect=side_effect_func)
def test_run_script(self, mock):
# Run the function
process = run_script(script_path='my_script.py', my_arg='bar')
# Assert if the mock object was called
self.assertTrue(mock.called)
# Assert if the mock object returned 'foo' when providing 'bar'
self.assertEqual(process, 'foo')
There is no need for anything complex. You can simply move the call to subprocess to a function and pass a mock function in your test.
import subprocess
import unittest
import logging
from unittest.mock import Mock
def _exec(cmd):
p = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout, stderr = p.communicate()
return p, stdout, stderr
def uses_exec(execute=_exec):
cmd = ['ls']
p, stdout, stderr = execute(cmd)
if p.returncode != 0:
logging.error(
'%s returned Error Code: %s',
cmd,
p.returncode
)
logging.error(stderr)
else:
logging.info(stdout)
class TestUsesExecute(unittest.TestCase):
def test_get_aws_creds_from_namespace(self):
p = lambda:None
p.returncode = 0
stdout = 'file1 file2'
mock_execute = Mock(return_value=(p, stdout, ''))
uses_exec(execute=mock_execute)
mock_execute.assert_called_with(['ls'])
if __name__ == '__main__':
unittest.main()
This pattern works in all programming languages that allow functions to be passed as parameters. It can be used with other python system calls for file and network reads just as easily.
I used this for a test suite where there were many subprocess.run calls to check.
import contextlib
import re
import subprocess
from typing import NamedTuple
from unittest.mock import MagicMock
class CmdMatch(NamedTuple):
cmd: str
match: str = ".*"
result: str = ""
side_effect: Callable = None
#contextlib.contextmanager
def mock_run(*cmd_match: Union[str, CmdMatch], **kws):
sub_run = subprocess.run
mock = MagicMock()
if isinstance(cmd_match[0], str):
cmd_match = [CmdMatch(*cmd_match, **kws)]
def new_run(cmd, **_kws):
check_cmd = " ".join(cmd[1:])
mock(*cmd[1:])
for m in cmd_match:
if m.cmd in cmd[0].lower() and re.match(m.match, check_cmd):
if m.side_effect:
m.side_effect()
return subprocess.CompletedProcess(cmd, 0, m.result, "")
assert False, "No matching call for %s" % check_cmd
subprocess.run = new_run
yield mock
subprocess.run = sub_run
So now you can write a test like this:
def test_call_git():
with mock_run("git", "describe.*", "v2.5") as cmd:
do_something()
cmd.assert_called_once()
def test_other_calls():
with mock_run(
CmdMatch("git", "describe.*", "v2.5"),
CmdMatch("aws", "s3.*links.*", side_effect=subprocess.CalledProcessError),
) as cmd:
do_something()
Couple funky things you might want to change, but I liked them:
the resulting mock calls ignore the first argument (so which/full-paths aren't part of the tests)
if nothing matches any calls, it asserts (you have to have a match)
the first positional is considered the "name of the command"
kwargs to run are ignored (_call assertions are easy but loose)
I'm trying to use LocalPath.sysexec() from the py library, but the documentation is not clear enough for me to understand how to invoke it with the right syntax.
Here is some fake code that mimics my syntax:
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Testing pylib implementation of subprocess.Popen
from py.path import local
import sys
path = local(sys.argv[1])
path.sysexec("/usr/bin/foo", ["arg1", "arg2"])
You can look the source code:
def sysexec(self, *argv, **popen_opts):
""" return stdout text from executing a system child process,
where the 'self' path points to executable.
The process is directly invoked and not through a system shell.
"""
from subprocess import Popen, PIPE
argv = map_as_list(str, argv)
popen_opts['stdout'] = popen_opts['stderr'] = PIPE
proc = Popen([str(self)] + argv, **popen_opts)
stdout, stderr = proc.communicate()
ret = proc.wait()
if py.builtin._isbytes(stdout):
stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
if ret != 0:
if py.builtin._isbytes(stderr):
stderr = py.builtin._totext(stderr, sys.getdefaultencoding())
raise py.process.cmdexec.Error(ret, ret, str(self),
stdout, stderr,)
return stdout
clearly, It use the python subprocess module. If you have not been used subprocess, you can click the above link to raed the docs.
In this function, It use subprocess construct a Popen object, and use the argv you pass.
Then call the Popen.wait(), block until the command execute finish. And return the stdout of the command.
Example:
local_path = py._path.local.LocalPath('/usr/bin/ls')
print(local_path.sysexec())
# out: file1\nfile2\nfile3...
print(local_path.sysexec('-l'))
# out likes "ls -l" out
I have a faulty third party python module that is outputing to stdout or stderr while it is imported and this is breaking the output of my unittests.
How can I temporary redirect the stdout in order to hide its output.
Limit to Python 2.5 syntax :)
Update, I forgot to mention that sys.stdout and sys.__stderr__ methods do not work in this case. As far as I know this faulty module is using native code.
You can also use mock to let you patch sys.stdout and sys.stderr for you when the module is imported. An example of a testing module that using this strategy would be:
import os
devnull = open(os.devnull, 'w')
from mock import patch
with patch('sys.stdout', devnull):
with patch('sys.stderr', devnull):
import bad_module
# Test cases writen here
where bad_module is the third party module that is printing to sys.stdout and sys.stderr when is being imported.
You can do it something like this:
>>> import sys, os
>>> _stderr = sys.stderr
>>> _stdout = sys.stdout
>>> null = open(os.devnull,'wb')
>>> sys.stdout = sys.stderr = null
>>> print("Bleh")
>>> sys.stderr = _stderr
>>> sys.stdout = _stdout
>>> print("Bleh")
Bleh