I am trying to mockup the call to open a file in a S3 bucket. The code that I have is:
# mymodule.py
import s3fs
#...
def __init__(self):
self.s3_filesystem = s3fs.S3FileSystem(anon=False, key=s3_key,
secret=s3_secret)
#...
def mymethod(self):
with self.s3_filesystem.open(filename, 'r') as csv_file:
file_dataframe = pd.read_csv(csv_file)
The mock that I have in pytest is:
import pytest
def test(mocker)
mocker.patch('mypackage.mymodule.s3fs.S3FileSystem.open',
return_value=open)
But at run the test I get the error:
> with self.s3_filesystem.open(filename, 'r') as csv_file:
E AttributeError: __enter__
Any idea why?
Disclaimer: All credits to my answer are due to the OP Juan Fernando Jaramillo Botero and his comment to his own question, above. I am posting an answer here, because it appears that the OP does not have at least 15 reputation points at the moment - and thus, can't answer his own question, as indicated in this help page. Please, if this is not the case, I encourage the OP to add his answer - and accept it - because it helped me a lot!
I was having the same problem, where I had to mock two methods from s3fs.S3FileSystem: open (as OP), and also ls, in a pytest suite of tests.
By using the decorator patch, and its argument side_effect, all calls for s3fs.S3FileSystem.open and s3fs.S3FileSystem.ls during tests are properly replaced by open and os.listdir, respectively, which in my case was the desired effect.
Simplified situation:
my_module.py
import s3fs
class Operation():
def compile_files(self, fs, input_bucket):
files = fs.ls(input_bucket)
return files
def copy_files(self, fs, files, input_bucket, output_bucket):
for file in files:
with fs.open(f'{input_bucket}/{file}', 'rb') as source:
with fs.open(f'{output_bucket}/{file}', 'wb') as destination:
destination.write(source.read())
def transform(self, input_bucket, output_bucket):
fs = s3fs.S3FileSystem()
files = self.compile_files(fs, input_bucket)
self.copy_files(fs, files, input_bucket, output_bucket)
my_module_test.py
import os
from unittest.mock import patch
from my_module import Operation
#patch('s3fs.S3FileSystem.open', side_effect=open)
#patch('s3fs.S3FileSystem.ls', side_effect=os.listdir)
def test_my_module(mock_s3fs_ls, mock_s3fs_open):
input_bucket = 'test/fixtures/files'
output_bucket = 'test/fixtures/tmp_output'
os.mkdir(output_bucket)
Operation().transform(input_bucket, output_bucket)
# Proceed to assert everything went as expected...
Note that the test file does not import pytest, but it is fully compatible and can be run within a pytest test suite.
Related
I'm writing unit tests for a simple function that writes bytes into s3:
import s3fs
def write_bytes_as_csv_to_s3(payload, bucket, key):
fs = s3fs.S3FileSystem()
fname = f"{bucket}/{key}"
print(f"writing {len(payload)} bytes to {fname}")
with fs.open(fname, "wb") as f:
f.write(payload)
return fname
def test_write_bytes_as_csv_to_s3(mocker):
s3fs_mock = mocker.patch('s3fs.S3FileSystem')
open_mock = mocker.MagicMock()
# write_mock = mocker.MagicMock()
# open_mock.write.return_value = write_mock
s3fs_mock.open.invoke.return_value = open_mock
result = write_bytes_as_csv_to_s3('awesome'.encode(), 'random', 'key')
assert result == 'random/key'
s3fs_mock.assert_called_once()
open_mock.assert_called_once()
# write_mock.assert_called_once()
How can I check if method open and write has been called once? Not sure how to set mocker to cover my case.
The unit-test you written above is perfect and mostly covered all the functionality of the methods which you want to test.
In pytest, there is a functionality to get the unittest coverage report which will show the lines covered by unittest.
Kindly install the pytest plugin html-report(if not installed) and execute the following document:-
py.test --cov=<filename to cover: unittest> --cov-report=html <testfile>
After that, you would likely found a html file in the current location o/r in the htmlconv/ directory. And from that, you could easily figure it out about the line covered and also the percentage of the unittest test coverage.
The issue is understanding how each mock is created and what exactly it mocks. For example mocker.patch('s3fs.S3FileSystem') returns a mock of s3fs.S3FileSystem, not the instance returned by calling s3fs.S3FileSystem(). Then to mock with fs.open(fname, "wb") as f you need to mock what the __enter__ dunder method returns. Hopefully the following code makes the relations clear:
def test_write_bytes_as_csv_to_s3(mocker):
# Mock of the open's context manager
open_cm_mock = mocker.MagicMock()
# Mock of the object returned by fs.open()
open_mock = mocker.MagicMock()
open_mock.__enter__.return_value = open_cm_mock
# Mock of the new instance returned by s3fs.S3FileSystem()
fs_mock = mocker.MagicMock()
fs_mock.open.return_value = open_mock
# Patching of s3fs.S3FileSystem
mocker.patch('s3fs.S3FileSystem').return_value = fs_mock
# Running the tested code and making assertions
result = write_bytes_as_csv_to_s3('awesome'.encode(), 'random', 'key')
assert result == 'random/key'
assert open_cm_mock.write.call_count == 1
I readily admit to going a bit overboard with unit testing.
While I have passing tests, I find my solution to be inelegant, and I'm curious if anyone has a cleaner solution.
The class being tested:
class Config():
def __init__(self):
config_parser = ConfigParser()
try:
self._read_config_file(config_parser)
except FileNotFoundError as e:
pass
self.token = config_parser.get('Tokens', 'Token', )
#staticmethod
def _read_config_file(config):
if not config.read(os.path.abspath(os.path.join(BASE_DIR, ROOT_DIR, CONFIG_FILE))):
raise FileNotFoundError(f'File {CONFIG_FILE} not found at path {BASE_DIR}{ROOT_DIR}')
The ugly test:
class TestConfiguration(unittest.TestCase):
#mock.patch('config.os.path.abspath')
def test_config_init_sets_token(self, mockFilePath: mock.MagicMock):
with open('mock_file.ini', 'w') as file: #here's where it gets ugly
file.write('[Tokens]\nToken: token')
mockFilePath.return_value = 'mock_file.ini'
config = Config()
self.assertEqual(config.token, 'token')
os.remove('mock_file.ini') #quite ugly
EDIT: What I mean is I'm creating a file instead of mocking one.
Does anyone know how to mock a file object, while having its data set so that it reads ascii text? The class is deeply buried.
Other than that, the way ConfigParser sets data with .read() is throwing me off. Granted, the test "works", it doesn't do it nicely.
For those asking about other testing behaviors, here's an example of another test in this class:
#mock.patch('config.os.path.abspath')
def test_warning_when_file_not_found(self, mockFilePath: mock.MagicMock):
mockFilePath.return_value = 'mock_no_file.ini'
with self.assertRaises(FileNotFoundError):
config.Config._read_config_file(ConfigParser())
Thank you for your time.
I've found it!
I had to start off with a few imports:from io import TextIOWrapper, BytesIO
This allows a file object to be created: TextIOWrapper(BytesIO(b'<StringContentHere>'))
The next part involved digging into the configparser module to see that it calls open(), in order to mock.patch the behavior, and, here we have it, an isolated unittest!
#mock.patch('configparser.open')
def test_bot_init_sets_token(self, mockFileOpen: mock.MagicMock):
mockFileOpen.return_value = TextIOWrapper(BytesIO(b'[Tokens]\nToken: token'))
config = Config()
self.assertEqual(config.token, 'token')
I have read many article over the last 6 hours and i still don't understand mocking and unit-testing. I want to unit test a open function, how can i do this correctly?
i am also concerned as the bulk of my code is using external files for data import and manipulation. I understand that i need to mock them for testing, but I am struggling to understand how to move forward.
Some advice please. Thank you in advance
prototype5.py
import os
import sys
import io
import pandas
pandas.set_option('display.width', None)
def openSetupConfig (a):
"""
SUMMARY
Read setup file
setup file will ONLY hold the file path of the working directory
:param a: str
:return: contents of the file stored as str
"""
try:
setupConfig = open(a, "r")
return setupConfig.read()
except Exception as ve:
ve = (str(ve) + "\n\nPlease ensure setup file " + str(a) + " is available")
sys.exit(ve)
dirPath = openSetupConfig("Setup.dat")
test_prototype5.py
import prototype5
import unittest
class TEST_openSetupConfig (unittest.TestCase):
"""
Test the openSetupConfig function from the prototype 5 library
"""
def test_open_correct_file(self):
result = prototype5.openSetupConfig("Setup.dat")
self.assertTrue(result)
if __name__ == '__main__':
unittest.main()
So the rule of thumb is to mock, stub or fake all external dependencies to the method/function under test. The point is to test the logic in isolation. So in your case you want to test that it can open a file or log an error message if it can't be opened.
import unittest
from mock import patch
from prototype5 import openSetupConfig # you don't want to run the whole file
import __builtin__ # needed to mock open
def test_openSetupConfig_with_valid_file(self):
"""
It should return file contents when passed a valid file.
"""
expect = 'fake_contents'
with patch('__builtin__.open', return_value=expect) as mock_open:
actual = openSetupConfig("Setup.dat")
self.assertEqual(expect, actual)
mock_open.assert_called()
#patch('prototype5.sys.exit')
def test_openSetupConfig_with_invalid_file(self, mock_exit):
"""
It should log an error and exit when passed an invalid file.
"""
with patch('__builtin__.open', side_effect=FileNotFoundError) as mock_open:
openSetupConfig('foo')
mock_exit.assert_called()
I am trying to create a unit test for the following function:
def my_function(path):
#Search files at the given path
for file in os.listdir(path):
if file.endswith(".json"):
#Search for file i'm looking for
if file == "file_im_looking_for.json":
#Open file
os.chdir(path)
json_file=json.load(open(file))
print json_file["name"]
However I am having trouble successfully creating a fake directory with files in order for the function to work correctly and not through errors.
Below is what I have so far but it is not working for me, and I'm not sure how to incorporate "file_im_looking_for" as the file in the fake directory.
tmpfilepath = os.path.join(tempfile.gettempdir(), "tmp-testfile")
#mock.patch('my_module.os')
def test_my_function(self):
# make the file 'exist'
mock_path.endswith.return_value = True
file_im_looking_for=[{
"name": "test_json_file",
"type": "General"
}]
my_module.my_function("tmpfilepath")
Any advice where I'm going wrong or other ideas to approach this problem are appreciated!
First of all, you forgot to pass the mocked object to test function. The right way to use mock in your test should be like this.
#mock.patch('my_module.os')
def test_my_function(self, mock_path):
Anyway, you shouldn't mock the endswith, but the listdir. The snippet below is an example and may help you.
app.py
def check_files(path):
files = []
for _file in os.listdir(path):
if _file.endswith('.json'):
files.append(_file)
return files
test_app.py
import unittest
import mock
from app import check_files
class TestCheckFile(unittest.TestCase):
#mock.patch('app.os.listdir')
def test_check_file_should_succeed(self, mock_listdir):
mock_listdir.return_value = ['a.json', 'b.json', 'c.json', 'd.txt']
files = check_files('.')
self.assertEqual(3, len(files))
#mock.patch('app.os.listdir')
def test_check_file_should_fail(self, mock_listdir):
mock_listdir.return_value = ['a.json', 'b.json', 'c.json', 'd.txt']
files = check_files('.')
self.assertNotEqual(2, len(files))
if __name__ == '__main__':
unittest.main()
Edit: Answering your question in comment, you need to mock the json.loads and the open from your app.
#mock.patch('converter.open')
#mock.patch('converter.json.loads')
#mock.patch('converter.os.listdir')
def test_check_file_load_json_should_succeed(self, mock_listdir, mock_json_loads, mock_open):
mock_listdir.return_value = ['a.json', 'file_im_looking_for.json', 'd.txt']
mock_json_loads.return_value = [{"name": "test_json_file", "type": "General"}]
files = check_files('.')
self.assertEqual(1, len(files))
But remember! If your is too broad or hard to maintain, perhaps refactoring your API should be a good idea.
I would suggest to use Python's tempfile library, specifically TemporaryDirectory.
The issue with your and Mauro Baraldi's solution is that you have to patch multiple functions. This is a very error prone way, since with mock.patch you have to know exactly what you are doing! Otherwise, this may cause unexpected errors and eventually frustration.
Personally, I prefer pytest, since it has IMO nicer syntax and better fixtures, but since the creator used unittest I will stick with it.
I would rewrite your test code like this:
import json
import pathlib
import tempfile
import unittest
wrong_data = {
"name": "wrong_json_file",
"type": "Fake"
}
correct_data = {
"name": "test_json_file",
"type": "General"
}
class TestMyFunction(unittest.TestCase):
def setUp(self):
""" Called before every test. """
self._temp_dir = tempfile.TemporaryDirectory()
temp_path = pathlib.Path(self._temp_dir.name)
self._create_temporary_file_with_json_data(temp_path / 'wrong_json_file.json', wrong_data)
self._create_temporary_file_with_json_data(temp_path / 'file_im_looking_for.json', correct_data)
def tearDown(self):
""" Called after every test. """
self._temp_dir.cleanup()
def _create_temporary_file_with_json_data(self, file_path, json_data):
with open(file_path, 'w') as ifile:
ifile.write(json.dumps(content))
def test_my_function(self):
my_module.my_function(str(self._temp_dir))
You see that your actual test is compressed down to a single line! Admittedly, there is no assert, but if your function would return something, the result would behave as expected.
No mocking, because everything exists and will be cleaned up afterwards. And the best thing is that you now can add more tests with a lower barrier of entry.
I don't know why I'm just not getting this, but I want to use mock in Python to test that my functions are calling functions in ftplib.FTP correctly. I've simplified everything down and still am not wrapping my head around how it works. Here is a simple example:
import unittest
import ftplib
from unittest.mock import patch
def download_file(hostname, file_path, file_name):
ftp = ftplib.FTP(hostname)
ftp.login()
ftp.cwd(file_path)
class TestDownloader(unittest.TestCase):
#patch('ftplib.FTP')
def test_download_file(self, mock_ftp):
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
mock_ftp.cwd.assert_called_with('pub/files')
When I run this, I get:
AssertionError: Expected call: cwd('pub/files')
Not called
I know it must be using the mock object since that is a fake server name, and when run without patching, it throws a "socket.gaierror" exception.
How do I get the actual object the fuction is running? The long term goal is not having the "download_file" function in the same file, but calling it from a separate module file.
When you do patch(ftplib.FTP) you are patching FTP constructor. dowload_file() use it to build ftp object so your ftp object on which you call login() and cmd() will be mock_ftp.return_value instead of mock_ftp.
Your test code should be follow:
class TestDownloader(unittest.TestCase):
#patch('ftplib.FTP', autospec=True)
def test_download_file(self, mock_ftp_constructor):
mock_ftp = mock_ftp_constructor.return_value
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
mock_ftp_constructor.assert_called_with('ftp.server.local')
self.assertTrue(mock_ftp.login.called)
mock_ftp.cwd.assert_called_with('pub/files')
I added all checks and autospec=True just because is a good practice
Like Ibrohim's answer, I prefer pytest with mocker.
I have went a bit further and have actually wrote a library which helps me to mock easily. Here is how to use it for your case.
You start by having your code and a basic pytest function, with the addition of my helper library to generate mocks to modules and the matching asserts generation:
import ftplib
from mock_autogen.pytest_mocker import PytestMocker
def download_file(hostname, file_path, file_name):
ftp = ftplib.FTP(hostname)
ftp.login()
ftp.cwd(file_path)
def test_download_file(mocker):
import sys
print(PytestMocker(mocked=sys.modules[__name__],
name=__name__).mock_modules().prepare_asserts_calls().generate())
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
When you run the test for the first time, it would fail due to unknown DNS, but the print statement which wraps my library would give us this valuable input:
...
mock_ftplib = mocker.MagicMock(name='ftplib')
mocker.patch('test_29817963.ftplib', new=mock_ftplib)
...
import mock_autogen
...
print(mock_autogen.generator.generate_asserts(mock_ftplib, name='mock_ftplib'))
I'm placing this in the test and would run it again:
def test_download_file(mocker):
mock_ftplib = mocker.MagicMock(name='ftplib')
mocker.patch('test_29817963.ftplib', new=mock_ftplib)
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
import mock_autogen
print(mock_autogen.generator.generate_asserts(mock_ftplib, name='mock_ftplib'))
This time the test succeeds and I only need to collect the result of the second print to get the proper asserts:
def test_download_file(mocker):
mock_ftplib = mocker.MagicMock(name='ftplib')
mocker.patch(__name__ + '.ftplib', new=mock_ftplib)
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
mock_ftplib.FTP.assert_called_once_with('ftp.server.local')
mock_ftplib.FTP.return_value.login.assert_called_once_with()
mock_ftplib.FTP.return_value.cwd.assert_called_once_with('pub/files')
If you would like to keep using unittest while using my library, I'm accepting pull requests.
I suggest using pytest and pytest-mock.
from pytest_mock import mocker
def test_download_file(mocker):
ftp_constructor_mock = mocker.patch('ftplib.FTP')
ftp_mock = ftp_constructor_mock.return_value
download_file('ftp.server.local', 'pub/files', 'wanted_file.txt')
ftp_constructor_mock.assert_called_with('ftp.server.local')
assert ftp_mock.login.called
ftp_mock.cwd.assert_called_with('pub/files')