Develop Reference

Python mocking not working for shutil.rmtree

I have a class with a method which uses shutil.rmtree to remove some files if a param is passed as true, How to mock this behavior so as other tests don't break which needs these files.
My class looks like this -
class FileConverter(object):
def __init__(self, path_to_files):
self._path_to_files = path_to_files
def convert_files(self, rmv_src=False):
doStuff()
if rmv_src:
shutil.rmtree(self.__path_to_files)
def doStuff():
# does some stuff
Now my tests look like -
class TestFileConverter(unittest.TestCase):
def test_convert_success(self):
input_dir = 'resources/files'
file_converter = FileConverter(input_dir)
file_converter.convert_files()
# assert the things from doStuff
#mock.patch('shutil.rmtree')
def test_convert_with_rmv(self, rm_mock):
input_dir = 'resources/files'
file_converter = FileConverter(input_dir)
file_converter.convert_files(True)
self.assertEquals(rm_mock, [call(input_dir)])
Now when I run this testsuite the test with rmv gives me assertionError
<MagicMock name='rmtree' id='139676276822984'> != [call('resources/images')]
and the first test gives me file not found error since the mock did not work and the rmv source test removed the file
FileNotFoundError: [Errno 2] No such file or directory: 'resources/images'
If I comment out the second test with rmv_source true then my first test works fine.
What am I doing wrong here?

Your module has already imported shutil.rmtree so mocking it later in the test suite won't do anything.
You need to mock the module when you import FileConverter, not afterwards.
import sys
from mock import MagicMock
sys.modules['shutil'] = MagicMock()
# and/or
sys.modules['shutil.rmtree'] = MagicMock()
import FileConverter
If you still need to use shutil in your test code, then first import it using an alias,and use that when you need the 'real' module:
import sys
from mock import MagicMock
import shutil as shutil_orig
sys.modules['shutil'] = MagicMock()
import shutil
print(type(shutil_orig.rmtree))
# <class 'function'>
print(type(shutil.rmtree))
# <class 'mock.mock.MagicMock'>

The original post should work except the call(input_dir) did not work for me
#mock.patch('shutil.rmtree')
def test_convert_with_rmv(self, rm_mock):
input_dir = 'resources/files'
rm_mock.return_value = 'REMOVED'
file_converter = FileConverter(input_dir)
file_converter.convert_files(True)
rm_mock.assert_called_with(input_dir)
self.assertEqual(rm_mock.return_value, 'REMOVED')
The test_convert_with_rmv has no way removed the input_dir, it probably never created in the first place. You could assert this statement in each test before and after convert_files called:
self.asserTrue(os.path.isdir(input_dir))

Mock nested import in Python with MagicMock

My file (ensure_path.py):
import os
def ensure_path(path):
if not os.path.exists(path):
os.makedirs(path)
return path
My test:
import unittest
from unittest.mock import patch, MagicMock
from src.util.fs.ensure_path import ensure_path
FAKE_PATH = '/foo/bar'
class EnsurePathSpec(unittest.TestCase):
#patch('os.path.exists', side_effect=MagicMock(return_value=False))
#patch('os.makedirs', side_effect=MagicMock(return_value=True))
def test_path_exists_false(self, _mock_os_path_exists_false, _mock_os_makedirs):
ensure_path(FAKE_PATH)
_mock_os_path_exists_false.assert_called_with(FAKE_PATH)
_mock_os_makedirs.assert_called_with(FAKE_PATH)
#patch('os.path.exists', side_effect=MagicMock(return_value=True))
#patch('os.makedirs', side_effect=MagicMock(return_value=True))
def test_path_exists_true(self, _mock_os_path_exists_true, _mock_os_makedirs):
ensure_path(FAKE_PATH)
_mock_os_path_exists_true.assert_called_with(FAKE_PATH)
_mock_os_makedirs.assert_not_called()
This is giving the failed assertion Expected call: makedirs('/foo/bar') which I think makes sense because I think I'm mocking os.makedirs at the wrong level.
I've tried replacing #patch('os.makedirs', with #patch('src.util.fs.ensure_path.os.makedirs', and a couple variations of that but I get
ImportError: No module named 'src.util.fs.ensure_path.os'; 'src.util.fs.ensure_path' is not a package
Here is my __init__.py flow :
Is there an obvious fix I'm missing?

Your patch arguments need to be in the reverse order of the #patch decorators.

Mocked patch of os.listdir not working for unittesting

I have a class method that I am trying to test that requires two patched methods, ConfigB.__init__ and listdir:
from os import listdir
from config.ConfigB import ConfigB
class FileRunner(object):
def runProcess(self, cfgA)
cfgB = ConfigB(cfgA)
print(listdir())
I have the following test set up:
import unittest
import unittest.mock imort MagicMock
import mock
from FileRunner import FileRunner
class TestFileRunner(unittest.TestCase):
#mock.patch('ConfigB.ConfigB.__init__')
#mock.patch('os.listdir')
def test_methodscalled(self, osListDir, cfgB):
cfgA = MagicMock()
fileRunner = FileRunner()
cfgB.return_value = None
osListDir.return_value = None
fileRunner.runProcess(cfgA)
Now the patched mock and return value works for ConfigB.ConfigB, but it does not work for os.listdir. When the print(listdir()) method runs I get a list of file in the current directory, not a value of None as I specified in the patched return value. Any idea what is going wrong?

You need to patch your relative path to your code. patch('os.listdir') doesn't works because you need to patch this:
#mock.patch("path.to.your.pythonfile.listdir")
Try with that.

Mocking two functions with patch for a unit test

I have a function I want to unit test contains calls two other functions. I am unsure how can I mock both functions at the same time properly using patch. I have provided an example of what I mean below. When I run nosetests, the tests pass but I feel that there must be a cleaner way to do this and I do not really Understand the piece regarding f.close()...
The directory structure looks like this:
program/
program/
data.py
tests/
data_test.py
data.py:
import cPickle
def write_out(file_path, data):
f = open(file_path, 'wb')
cPickle.dump(data, f)
f.close()
data_test.py:
from mock import MagicMock, patch
def test_write_out():
path = '~/collection'
mock_open = MagicMock()
mock_pickle = MagicMock()
f_mock = MagicMock()
with patch('__builtin__.open', mock_open):
f = mock_open.return_value
f.method.return_value = path
with patch('cPickle.dump', mock_pickle):
write_out(path, 'data')
mock_open.assert_called_once_with('~/collection', 'wb')
f.close.assert_any_call()
mock_pickle.assert_called_once_with('data', f)
Results:
$ nosetests
.
----------------------------------------------------------------------
Ran 1 test in 0.008s
OK

You can simplify your test by using the patch decorator and nesting them like so (they are MagicMock objects by default):
from unittest.mock import patch
#patch('cPickle.dump')
#patch('__builtin__.open')
def test_write_out(mock_open, mock_pickle):
path = '~/collection'
f = mock_open.return_value
f.method.return_value = path
write_out(path, 'data')
mock_open.assert_called_once_with('~/collection', 'wb')
mock_pickle.assert_called_once_with('data', f)
f.close.assert_any_call()
Calls to a MagicMock instance return a new MagicMock instance, so you can check that the returned value was called just like any other mocked object. In this case f is a MagicMock named 'open()' (try printing f).

In addition to the response #Matti John you can also use patch inside function test_write_out:
from mock import MagicMock, patch
def test_write_out():
path = '~/collection'
with patch('__builtin__.open') as mock_open, \
patch('cPickle.dump') as mock_pickle:
f = mock_open.return_value
...

As of Python 3.10 you can do use Parenthesized Context Managers like this
from unittest.mock import patch
def test_write_out():
with (
patch('cPickle.dump'),
patch('__builtin__.open') as open_mock, # example of using `as`
):
yield

Here's a simple example on how to test raising ConflictError in create_collection function using mock:
import os
from unittest import TestCase
from mock import patch
from ..program.data import ConflictError, create_collection
class TestCreateCollection(TestCase):
def test_path_exists(self):
with patch.object(os.path, 'exists') as mock_method:
mock_method.return_value = True
self.assertRaises(ConflictError, create_collection, 'test')
Please, also see mock docs and Michael Foord's awesome introduction to mock.

How do I change the name of an imported library?

I am using jython with a third party application. The third party application has some builtin libraries foo. To do some (unit) testing we want to run some code outside of the application. Since foo is bound to the application we decided to write our own mock implementation.
However there is one issue, we implemented our mock class in python while their class is in java. Thus to use their code one would do import foo and foo is the mock class afterwards. However if we import the python module like this we get the module attached to the name, thus one has to write foo.foo to get to the class.
For convenience reason we would love to be able to write from ourlib.thirdparty import foo to bind foo to the foo-class. However we would like to avoid to import all the classes in ourlib.thirdparty directly, since the loading time for each file takes quite a while.
Is there any way to this in python? ( I did not get far with Import hooks I tried simply returning the class from load_module or overwriting what I write to sys.modules (I think both approaches are ugly, particularly the later))
edit:
ok: here is what the files in ourlib.thirdparty look like simplified(without magic):
foo.py:
try:
import foo
except ImportError:
class foo
....
Actually they look like this:
foo.py:
class foo
....
__init__.py in ourlib.thirdparty
import sys
import os.path
import imp
#TODO: 3.0 importlib.util abstract base classes could greatly simplify this code or make it prettier.
class Importer(object):
def __init__(self, path_entry):
if not path_entry.startswith(os.path.join(os.path.dirname(__file__), 'thirdparty')):
raise ImportError('Custom importer only for thirdparty objects')
self._importTuples = {}
def find_module(self, fullname):
module = fullname.rpartition('.')[2]
try:
if fullname not in self._importTuples:
fileObj, self._importTuples[fullname] = imp.find_module(module)
if isinstance(fileObj, file):
fileObj.close()
except:
print 'backup'
path = os.path.join(os.path.join(os.path.dirname(__file__), 'thirdparty'), module+'.py')
if not os.path.isfile(path):
return None
raise ImportError("Could not find dummy class for %s (%s)\n(searched:%s)" % (module, fullname, path))
self._importTuples[fullname] = path, ('.py', 'r', imp.PY_SOURCE)
return self
def load_module(self, fullname):
fp = None
python = False
print fullname
if self._importTuples[fullname][1][2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.PY_FROZEN):
fp = open( self._importTuples[fullname][0], self._importTuples[fullname][1][1])
python = True
try:
imp.load_module(fullname, fp, *self._importTuples[fullname])
finally:
if python:
module = fullname.rpartition('.')[2]
#setattr(sys.modules[fullname], module, getattr(sys.modules[fullname], module))
#sys.modules[fullname] = getattr(sys.modules[fullname], module)
if isinstance(fp, file):
fp.close()
return getattr(sys.modules[fullname], module)
sys.path_hooks.append(Importer)

As others have remarked, it is such a plain thing in Python that the import statement iself has a syntax for that:
from foo import foo as original_foo, for example -
or even import foo as module_foo
Interesting to note is that the import statemente binds a name to the imported module or object ont he local context - however, the dictionary sys.modules (on the moduels sys of course), is a live reference to all imported modules, using their names as a key. This mechanism plays a key role in avoding that Python re-reads and re-executes and already imported module , when running (that is, if various of yoru modules or sub-modules import the samefoo` module, it is just read once -- the subsequent imports use the reference stored in sys.modules).
And -- besides the "import...as" syntax, modules in Python are just another object: you can assign any other name to them in run time.
So, the following code would also work perfectly for you:
import foo
original_foo = foo
class foo(Mock):
...