I have a Python class whose __init__ method raises a custom exception called WrongFileSpecified.
However, when I write a unit test, I want to assign the attributes of the instance object from a test fixture. So normally what I would be doing is reading data off a file and then working with the instance object.
But with the test, I cannot use any test files, so I basically need to hard code the data in the instance object in the setUp method of the unit test. Is there any way to get a instance created without __init__ complaining about the exception?
Sample code:
class A(object):
def __init__(self, folderPath):
#check folder path using os.isdir() otherwise raise exception
#...
self.folderPath = folderPath
#Call load record
self._load_records() #uses self.folderPath and raises exceptions as well
#Note i cannot avoid raising these exceptions, its required
class TestA(unittest.TestCase):
.......
obj = None
def setUp(self):
obj = A('fake folder path')
obj.val1 = "testparam1"
obj.param2 = "testparam2"
def test_1(self):
.....
You can create an empty object, bypassing __init__ by using __new__.
obj = obj_type.__new__(obj_type)
Note that obj_type is the appropriate type object. This is a little hacky but it works. You are reponsible for setting the object's members.
Edit: here is an example.
class Foo():
def __init__(self):
self.x = 1
self.y = 2
def say_hello(self):
print('Hello!')
r = Foo.__new__(Foo)
r.say_hello()
print(r.x)
Console output:
Hello!
Traceback (most recent call last):
File "C:\WinPython-64bit-3.3.5.7\python-
3.3.5.amd64\Scripts\projects\luc_utils\dev\test\
unit_test_serialization.py", line 29, in <module>
print(r.x)
AttributeError: 'Foo' object has no attribute 'x'
Here are two options:
Refactor the file loading out to a class method, which is the Pythonic method of providing an alternate constructor (see below); or
Provide an additional parameter to __init__ to suppress the exceptions when necessary (e.g. def __init__(self, folderPath, suppress=False), or validate=True, whichever makes more sense for your usage).
The latter is a bit awkward, in my opinion, but would mean that you don't have to refactor existing code creating A instances. The former would look like:
class A(object):
def __init__(self, ...):
"""Pass whatever is loaded from the file to __init__."""
...
#classmethod
def from_file(cls, folderPath):
"""Load the data from the file, or raise an exception."""
...
and you would replace e.g. a = A(whatever) with a = A.from_file(whatever).
There is a very useful module called mock, you can check it out later, I feel that in this case it will be too much. Instead, you should consider redesigning your class, like this, for example:
class A(object):
def __init__(self, folderPath):
self.folderPath = folderPath
def _load_records(self)
#check folder path using os.isdir() otherwise raise exception
...
#uses self.folderPath and raises exceptions as well
...
#classmethod
def load_records(cls, folderpath):
obj = cls(folderpath)
obj._load_records()
return obj
# Usage
records = A.load_records('/path/to/records')
Then you can do:
class TestA(unittest.TestCase):
.......
obj = None
def setUp(self):
self.obj = A('fake folder path')
self.obj.val1 = "testparam1"
self.obj.param2 = "testparam2"
def test_1(self):
self.assertRaises(self.obj._load_records, HorribleFailureError)
Also i highly recommend to check out pytest, it has wonderful facilities for test fixtures, including fixtures for files and folders.
Related
I am using a 3rd party Python library (wxPython), which has a buggy class in one of its modules.
The problematic code section looks like this:
def OnText(self, event):
value = self.GetValue()
if value != self.__oldvalue:
pass # Here some more code follows ...
self.__oldvalue = value
The problem is the if statement, because at the first call to this method self.__oldvalue has not been initialized yet. So for a workaround until this bug has been fixed by the library devs I thought I could fix this with a little workaround. I simply wanted to derive a child class from that faulty class and initialize self.__oldvalue in this constructor:
class MyIntCtrl(wx.lib.intctrl.IntCtrl):
def __init__(self, *args, **kw):
self.__oldvalue = None
super().__init__(*args, **kw)
However, now when I use this new class MyIntCtrl instead of the original IntCtrl class, I do get exactly the same error as before:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/wx/lib/intctrl.py", line 509, in OnText
if value != self.__oldvalue:
AttributeError: 'MyIntCtrl' object has no attribute '_IntCtrl__oldvalue'
Now I am wondering: What am I doing wrong, how else can I fix this issue in a child class?
Any member of class which starts with __ (double underscore) is private, you can use single underscore _ or not use underscores in naming for access them in derived classes.
class Parent:
def __init__(self):
self.__private_field = "private field"
self._protected_field = "protected field"
self.public_field = "public field"
class Child(Parent):
def __init__(self):
pass
def do(self):
print(self.__private_field) # It will throw exception
print(self._protected_field) # It will not throw exception
print(self.public_field) # It will not throw exception
Or you can bypass private/protected members by calling them like:
print(_Parent__private_field)
I've been looking into using unittest.mock to replace an object with a fake (as a toy project, as it seems interesting). Part way down the documentation, there's an example of replacing an object with an io.StringIO instance, so I can see that in theory this should work, but I can't get a particular toy example to work without having to jump through some additional hoops. Consider this code, saved as main.py:
from unittest.mock import patch
class FileSystem:
def __init__(self, filename="file.txt"):
self.filename = filename
def write(self, content):
with open(self.filename, 'w') as f:
f.write(content)
def read(self):
with open(self.filename) as f:
return f.read()
class FileSystemFake:
def __init__(self, filename="file.txt"):
self.filename = filename
self.content = None
self.file_exists = False
def write(self, content):
self.content = content
self.file_exists = True
def read(self):
if self.file_exists:
return self.content
raise FileNotFoundError
class FileSystemFakeHoldingInstance(FileSystemFake):
instance = None
#classmethod
def __call__(cls, *args, **kwargs):
if cls.instance is not None:
return cls.instance
cls.instance = FileSystemFakeHoldingInstance(*args, **kwargs)
return cls.instance
def say_hello():
filesystem = FileSystem("hello.txt")
filesystem.write("hello")
def test_self_no_patch():
say_hello()
with open("hello.txt") as f:
assert f.read() == "hello"
#patch('main.FileSystem', new_callable=FileSystem)
def test_self(fake):
say_hello()
assert fake.read() == "hello"
#patch('main.FileSystem', new_callable=FileSystemFake)
def test_fake(fake):
say_hello()
assert fake.read() == "hello"
#patch('main.FileSystem', new_callable=FileSystemFakeHoldingInstance)
def test_fake_holder(fake):
say_hello()
assert fake.read() == "hello"
#patch('main.FileSystem', new_callable=FileSystemFakeHoldingInstance)
def test_fake_holder_reference(fake):
say_hello()
assert FileSystemFakeHoldingInstance.instance.read() == "hello"
The toy example features a class FileSystem that saves a file on the file system and reads it back. This class is the one I'm faking out to avoid the file system access. I've made a FileSystemFake class, which pretends to save a file and read it back, and a FileSystemFakeHoldingInstance class, which holds an instance of FileSystemFake and implements a __call__ method that returns it.
I have a simple test (test_self_no_patch) which interacts using the file system and works fine. My first point of confusion happens with test_self, which attempts to replace a call to FileSystem with a call to FileSystem (the same class), but throws "TypeError: 'FileSystem' object is not callable". I put this one in after writing test_fake, which replaces FileSystem with FileSystemFake fails with a similar error: "TypeError: 'FileSystemFake' object is not callable".
The second issue is visible in test_fake_holder, where we get a FileNotFoundError: by the documentation, the object passed into the function is the created fake. The "write" function runs through fine, but calling "read" on the fake doesn't return the written text.
Finally, in test_fake_holder_reference, I read the text from a static instance on the holder, which works.
Why am I seeing this particular behaviour (classes not being callable, patch not returning the "right" object)? Is there a series of parameters I can pass to patch to make it work with FileSystemFake directly?
I have a Python class that issues a warning inside __init__(). It also provides a factory class method for opening and reading a file:
from warnings import warn
class MyWarning(Warning):
"""Warning issued when an invalid name is found."""
pass
class MyClass:
def __init__(self, names):
# Simplified; actual code is longer
if is_invalid(names):
names = fix_names(names)
warn(f'{names!r} contains invalid element(s)',
MyWarning, stacklevel=2)
self._names = names
#classmethod
def from_file(cls, filename):
with open(filename) as file:
names = extract_names(file)
return cls(names)
stacklevel=2 makes the warning refer to the call to MyClass() rather than the warn() statement itself. This works when user code directly instantiates MyClass. However, when MyClass.from_file() issues the warning, MyWarning refers to return cls(names), not the user code calling from_file().
How do I ensure that the factory method also issues a warning that points to the caller? Some options I've considered:
Add a "hidden" _stacklevel parameter to __init__(), and instantiate MyClass with _stacklevel=2 inside from_file().
This is super ugly, and exposes internal behavior to the API.
Add a "hidden" _stacklevel class attribute, and access it inside __init__(). Then temporarily modify this attribute in from_file()
Also super ugly.
Add a _set_names() method that checks/fixes the names and issues a warning when needed. Then call this method inside the constructor. For from_file(), first instantiate MyClass with empty args, then directly call _set_names() to ensure that MyWarning points to the caller.
Still hacky, and effectively calls _set_names() twice when from_file() is called.
Catch and re-throw the warning, similar to exception chaining.
Sounds good, but I have no idea how to do this.
I read the warning module docs but it offers little help on safely catching and re-throwing warnings. Converting the warning to an exception using warnings.simplefilter() would interrupt MyClass() and force me to call it again.
You can catch warnings similar to the way you catch exceptions using warnings.catch_warnings():
import warnings
class MyWarning(Warning):
"""Warning issued when an invalid name is found."""
pass
class MyClass:
def __init__(self, names):
# Simplified; actual code is longer
if is_invalid(names):
names = fix_names(names)
warn(f'{names!r} contains invalid element(s)',
MyWarning, stacklevel=2)
self._names = names
#classmethod
def from_file(cls, filename):
with open(filename) as file:
names = extract_names(file)
with warnings.catch_warnings(record=True) as cx_manager:
inst = cls(names)
#re-report warnings with the stack-level we want
for warning in cx_manager:
warnings.warn(warning.message, warning.category, stacklevel=2)
return inst
Just keep in mind the following note from the documentation of warnings.catch_warnings():
Note The catch_warnings manager works by replacing and then later restoring the module’s showwarning() function and internal list of filter specifications. This means the context manager is modifying global state and therefore is not thread-safe.
David is right, warnings.catch_warnings(record=True) is probably what you want. Though I would write it as a function decorator instead:
def reissue_warnings(func):
def inner(*args, **kwargs):
with warnings.catch_warnings(record = True) as warning_list:
result = func(*args, **kwargs)
for warning in warning_list:
warnings.warn(warning.message, warning.category, stacklevel = 2)
return result
return inner
And then in your example:
class MyClass:
def __init__(self, names):
# ...
#classmethod
#reissue_warnings
def from_file(cls, filename):
with open(filename) as file:
names = extract_names(file)
return cls(names)
inst = MyClass(['some', 'names']) # 58: MyWarning: ['some', 'names'] contains invalid element(s)
inst = MyClass.from_file('example') # 59: MyWarning: ['example'] contains invalid element(s)
This way also allows you to cleanly collect and reissue warnings across multiple functions as well:
class Test:
def a(self):
warnings.warn("This is a warning issued from a()")
#reissue_warnings
def b(self):
self.a()
#reissue_warnings
def c(self):
warnings.warn("This is a warning issued from c()")
self.b()
#reissue_warnings
def d(self):
self.c()
test = Test()
test.d() # Line 59
# 59: UserWarning: This is a warning issued from c()
# 59: UserWarning: This is a warning issued from a()
Suppose I want to extend the built-in file abstraction with extra operations at open and close time. In Python 2.7 this works:
class ExtFile(file):
def __init__(self, *args):
file.__init__(self, *args)
# extra stuff here
def close(self):
file.close(self)
# extra stuff here
Now I'm looking at updating the program to Python 3, in which open is a factory function that might return an instance of any of several different classes from the io module depending on how it's called. I could in principle subclass all of them, but that's tedious, and I'd have to reimplement the dispatching that open does. (In Python 3 the distinction between binary and text files matters rather more than it does in 2.x, and I need both.) These objects are going to be passed to library code that might do just about anything with them, so the idiom of making a "file-like" duck-typed class that wraps the return value of open and forwards necessary methods will be most verbose.
Can anyone suggest a 3.x approach that involves as little additional boilerplate as possible beyond the 2.x code shown?
You could just use a context manager instead. For example this one:
class SpecialFileOpener:
def __init__ (self, fileName, someOtherParameter):
self.f = open(fileName)
# do more stuff
print(someOtherParameter)
def __enter__ (self):
return self.f
def __exit__ (self, exc_type, exc_value, traceback):
self.f.close()
# do more stuff
print('Everything is over.')
Then you can use it like this:
>>> with SpecialFileOpener('C:\\test.txt', 'Hello world!') as f:
print(f.read())
Hello world!
foo bar
Everything is over.
Using a context block with with is preferred for file objects (and other resources) anyway.
tl;dr Use a context manager. See the bottom of this answer for important cautions about them.
Files got more complicated in Python 3. While there are some methods that can be used on normal user classes, those methods don't work with built-in classes. One way is to mix-in a desired class before instanciating it, but this requires knowing what the mix-in class should be first:
class MyFileType(???):
def __init__(...)
# stuff here
def close(self):
# more stuff here
Because there are so many types, and more could possibly be added in the future (unlikely, but possible), and we don't know for sure which will be returned until after the call to open, this method doesn't work.
Another method is to change both our custom type to have the returned file's ___bases__, and modifying the returned instance's __class__ attribute to our custom type:
class MyFileType:
def close(self):
# stuff here
some_file = open(path_to_file, '...') # ... = desired options
MyFileType.__bases__ = (some_file.__class__,) + MyFile.__bases__
but this yields
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: __bases__ assignment: '_io.TextIOWrapper' deallocator differs from 'object'
Yet another method that could work with pure user classes is to create the custom file type on the fly, directly from the returned instance's class, and then update the returned instance's class:
some_file = open(path_to_file, '...') # ... = desired options
class MyFile(some_file.__class__):
def close(self):
super().close()
print("that's all, folks!")
some_file.__class__ = MyFile
but again:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: __class__ assignment: only for heap types
So, it looks like the best method that will work at all in Python 3, and luckily will also work in Python 2 (useful if you want the same code base to work on both versions) is to have a custom context manager:
class Open(object):
def __init__(self, *args, **kwds):
# do custom stuff here
self.args = args
self.kwds = kwds
def __enter__(self):
# or do custom stuff here :)
self.file_obj = open(*self.args, **self.kwds)
# return actual file object so we don't have to worry
# about proxying
return self.file_obj
def __exit__(self, *args):
# and still more custom stuff here
self.file_obj.close()
# or here
and to use it:
with Open('some_file') as data:
# custom stuff just happened
for line in data:
print(line)
# data is now closed, and more custom stuff
# just happened
An important point to keep in mind: any unhandled exception in __init__ or __enter__ will prevent __exit__ from running, so in those two locations you still need to use the try/except and/or try/finally idioms to make sure you don't leak resources.
I had a similar problem, and a requirement of supporting both Python 2.x and 3.x. What I did was similar to the following (current full version):
class _file_obj(object):
"""Check if `f` is a file name and open the file in `mode`.
A context manager."""
def __init__(self, f, mode):
if isinstance(f, str):
self.file = open(f, mode)
else:
self.file = f
self.close_file = (self.file is not f)
def __enter__(self):
return self
def __exit__(self, *args, **kwargs):
if (not self.close_file):
return # do nothing
# clean up
exit = getattr(self.file, '__exit__', None)
if exit is not None:
return exit(*args, **kwargs)
else:
exit = getattr(self.file, 'close', None)
if exit is not None:
exit()
def __getattr__(self, attr):
return getattr(self.file, attr)
def __iter__(self):
return iter(self.file)
It passes all calls to the underlying file objects and can be initialized from an open file or from a filename. Also works as a context manager. Inspired by this answer.
I am finding that I am using plenty of context managers in Python. However, I have been testing a number of things using them, and I am often needing the following:
class MyTestCase(unittest.TestCase):
def testFirstThing(self):
with GetResource() as resource:
u = UnderTest(resource)
u.doStuff()
self.assertEqual(u.getSomething(), 'a value')
def testSecondThing(self):
with GetResource() as resource:
u = UnderTest(resource)
u.doOtherStuff()
self.assertEqual(u.getSomething(), 'a value')
When this gets to many tests, this is clearly going to get boring, so in the spirit of SPOT/DRY (single point of truth/dont repeat yourself), I'd want to refactor those bits into the test setUp() and tearDown() methods.
However, trying to do that has lead to this ugliness:
def setUp(self):
self._resource = GetSlot()
self._resource.__enter__()
def tearDown(self):
self._resource.__exit__(None, None, None)
There must be a better way to do this. Ideally, in the setUp()/tearDown() without repetitive bits for each test method (I can see how repeating a decorator on each method could do it).
Edit: Consider the undertest object to be internal, and the GetResource object to be a third party thing (which we aren't changing).
I've renamed GetSlot to GetResource here—this is more general than specific case—where context managers are the way which the object is intended to go into a locked state and out.
How about overriding unittest.TestCase.run() as illustrated below? This approach doesn't require calling any private methods or doing something to every method, which is what the questioner wanted.
from contextlib import contextmanager
import unittest
#contextmanager
def resource_manager():
yield 'foo'
class MyTest(unittest.TestCase):
def run(self, result=None):
with resource_manager() as resource:
self.resource = resource
super(MyTest, self).run(result)
def test(self):
self.assertEqual('foo', self.resource)
unittest.main()
This approach also allows passing the TestCase instance to the context manager, if you want to modify the TestCase instance there.
Manipulating context managers in situations where you don't want a with statement to clean things up if all your resource acquisitions succeed is one of the use cases that contextlib.ExitStack() is designed to handle.
For example (using addCleanup() rather than a custom tearDown() implementation):
def setUp(self):
with contextlib.ExitStack() as stack:
self._resource = stack.enter_context(GetResource())
self.addCleanup(stack.pop_all().close)
That's the most robust approach, since it correctly handles acquisition of multiple resources:
def setUp(self):
with contextlib.ExitStack() as stack:
self._resource1 = stack.enter_context(GetResource())
self._resource2 = stack.enter_context(GetOtherResource())
self.addCleanup(stack.pop_all().close)
Here, if GetOtherResource() fails, the first resource will be cleaned up immediately by the with statement, while if it succeeds, the pop_all() call will postpone the cleanup until the registered cleanup function runs.
If you know you're only ever going to have one resource to manage, you can skip the with statement:
def setUp(self):
stack = contextlib.ExitStack()
self._resource = stack.enter_context(GetResource())
self.addCleanup(stack.close)
However, that's a bit more error prone, since if you add more resources to the stack without first switching to the with statement based version, successfully allocated resources may not get cleaned up promptly if later resource acquisitions fail.
You can also write something comparable using a custom tearDown() implementation by saving a reference to the resource stack on the test case:
def setUp(self):
with contextlib.ExitStack() as stack:
self._resource1 = stack.enter_context(GetResource())
self._resource2 = stack.enter_context(GetOtherResource())
self._resource_stack = stack.pop_all()
def tearDown(self):
self._resource_stack.close()
Alternatively, you can also define a custom cleanup function that accesses the resource via a closure reference, avoiding the need to store any extra state on the test case purely for cleanup purposes:
def setUp(self):
with contextlib.ExitStack() as stack:
resource = stack.enter_context(GetResource())
def cleanup():
if necessary:
one_last_chance_to_use(resource)
stack.pop_all().close()
self.addCleanup(cleanup)
pytest fixtures are very close to your idea/style, and allow for exactly what you want:
import pytest
from code.to.test import foo
#pytest.fixture(...)
def resource():
with your_context_manager as r:
yield r
def test_foo(resource):
assert foo(resource).bar() == 42
The problem with calling __enter__ and __exit__ as you did, is not that you have done so: they can be called outside of a with statement. The problem is that your code has no provision to call the object's __exit__ method properly if an exception occurs.
So, the way to do it is to have a decorator that will wrap the call to your original method in a withstatement. A short metaclass can apply the decorator transparently to all methods named test* in the class -
# -*- coding: utf-8 -*-
from functools import wraps
import unittest
def setup_context(method):
# the 'wraps' decorator preserves the original function name
# otherwise unittest would not call it, as its name
# would not start with 'test'
#wraps(method)
def test_wrapper(self, *args, **kw):
with GetSlot() as slot:
self._slot = slot
result = method(self, *args, **kw)
delattr(self, "_slot")
return result
return test_wrapper
class MetaContext(type):
def __new__(mcs, name, bases, dct):
for key, value in dct.items():
if key.startswith("test"):
dct[key] = setup_context(value)
return type.__new__(mcs, name, bases, dct)
class GetSlot(object):
def __enter__(self):
return self
def __exit__(self, *args, **kw):
print "exiting object"
def doStuff(self):
print "doing stuff"
def doOtherStuff(self):
raise ValueError
def getSomething(self):
return "a value"
def UnderTest(*args):
return args[0]
class MyTestCase(unittest.TestCase):
__metaclass__ = MetaContext
def testFirstThing(self):
u = UnderTest(self._slot)
u.doStuff()
self.assertEqual(u.getSomething(), 'a value')
def testSecondThing(self):
u = UnderTest(self._slot)
u.doOtherStuff()
self.assertEqual(u.getSomething(), 'a value')
unittest.main()
(I also included mock implementations of "GetSlot" and the methods and functions in your example so that I myself could test the decorator and metaclass I am suggesting on this answer)
I'd argue you should separate your test of the context manager from your test of the Slot class. You could even use a mock object simulating the initialize/finalize interface of slot to test the context manager object, and then test your slot object separately.
from unittest import TestCase, main
class MockSlot(object):
initialized = False
ok_called = False
error_called = False
def initialize(self):
self.initialized = True
def finalize_ok(self):
self.ok_called = True
def finalize_error(self):
self.error_called = True
class GetSlot(object):
def __init__(self, slot_factory=MockSlot):
self.slot_factory = slot_factory
def __enter__(self):
s = self.s = self.slot_factory()
s.initialize()
return s
def __exit__(self, type, value, traceback):
if type is None:
self.s.finalize_ok()
else:
self.s.finalize_error()
class TestContextManager(TestCase):
def test_getslot_calls_initialize(self):
g = GetSlot()
with g as slot:
pass
self.assertTrue(g.s.initialized)
def test_getslot_calls_finalize_ok_if_operation_successful(self):
g = GetSlot()
with g as slot:
pass
self.assertTrue(g.s.ok_called)
def test_getslot_calls_finalize_error_if_operation_unsuccessful(self):
g = GetSlot()
try:
with g as slot:
raise ValueError
except:
pass
self.assertTrue(g.s.error_called)
if __name__ == "__main__":
main()
This makes code simpler, prevents concern mixing and allows you to reuse the context manager without having to code it in many places.