I'm facing some difficulties unittest my project, mainly due to the fact that the controllers reference a singleton produced by a factory.
A simple demonstration of this problem would be:
databasefactory.py
class DataBaseFactory(object):
# Lets imagine we support a number of databases. The client implementation all gives us a similar interfaces to use
# This is a singleton through the whole application
_database_client = None
#classmethod
def get_database_client(cls):
# type: () -> DataBaseClientInterFace
if not cls._database_client:
cls._database_client = DataBaseClient()
return cls._database_client
class DataBaseClientInterFace(object):
def get(self, key):
# type: (any) -> any
raise NotImplementedError()
def set(self, key, value):
# type: (any, any) -> any
raise NotImplementedError()
class DataBaseClient(DataBaseClientInterFace):
# Mock some real world database - The unittest mocking should be providing another client
_real_world_data = {}
def get(self, key):
return self._real_world_data[key]
def set(self, key, value):
self._real_world_data[key] = value
return value
model.py
from .databasefactory import DataBaseFactory
class DataModel(object):
# The DataBase type never changes so its a constant
DATA_BASE_CLIENT = DataBaseFactory.get_database_client()
def __init__(self, model_name):
self.model_name = model_name
def save(self):
# type: () -> None
"""
Save the current model into the database
"""
key = self.get_model_key()
data = vars(self)
self.DATA_BASE_CLIENT.set(key, data)
#classmethod
def load(cls):
# type: () -> DataModel
"""
Load the model
"""
key = cls.get_model_key()
data = cls.DATA_BASE_CLIENT.get(key)
return cls(**data)
#staticmethod
def get_model_key():
return 'model_test'
datacontroller.py
from .databasefactory import DataBaseFactory
from .model import DataModel
class DataBaseController(object):
"""
Does some stuff with the databaase
"""
# Also needs the database client. This is the same instance as on DataModel
DATA_BASE_CLIENT = DataBaseFactory.get_database_client()
_special_key = 'not_model_key'
#staticmethod
def save_a_model():
a_model = DataModel('test')
a_model.save()
#staticmethod
def load_a_model():
a_model = DataModel.load()
return a_model
#classmethod
def get_some_special_key(cls):
return cls.DATA_BASE_CLIENT.get(cls._special_key)
#classmethod
def set_some_special_key(cls):
return cls.DATA_BASE_CLIENT.set(cls._special_key, 1)
And finally the unittest itself:
test_simple.py
import unittest
from .databasefactory import DataBaseClientInterFace
from .datacontroller import DataBaseController
from .model import DataModel
class MockedDataBaseClient(DataBaseClientInterFace):
_mocked_data = {DataBaseController._special_key: 2,
DataModel.get_model_key(): {'model_name': 'mocked_test'}}
def get(self, key):
return self._mocked_data[key]
def set(self, key, value):
self._mocked_data[key] = value
return value
class SimpleOne(unittest.TestCase):
def test_controller(self):
"""
I want to mock the singleton instance referenced in both DataBaseController and DataModel
As DataBaseController imports DataModel, both classes have the DATA_BASE_CLIENT attributed instantiated with the factory result
"""
# Initially it'll throw a keyerror
with self.assertRaises(KeyError):
DataBaseController.get_some_special_key()
# Its impossible to just change the DATA_BASE_CLIENT in the DataBaseController as DataModel still points towards the real implementation
# Should not be done as it won't change anything to data model
DataBaseController.DATA_BASE_CLIENT = MockedDataBaseClient()
self.assertEqual(DataBaseController.get_some_special_key(), 2)
# Will fail as the DataModel still uses the real implementation
# I'd like to mock DATA_BASE_CLIENT for both classes without explicitely giving inserting a new class
# The project I'm working on has a number of these constants that make it a real hassle to inject it a new one
# There has to be a better way to tackle this issue
model = DataBaseController.load_a_model()
The moment the unittest imports the DataBaseController, DataModel is imported through the DataBaseController module.
This means that both DATA_BASE_CLIENT class variables are instantiated.
If my factory were to catch it running inside a unittest, it still would not matter as the import happens outside the unittest.
My question is: is there a way to mock this singleton and replace across the whole application at once?
Replacing the cached instance on the factory is not an option as the references in the classes point to the old object.
It might be a design flaw to put these singleton instances as class variables in the first place. But I'd rather retrieve a class variable than calling the factory each time for the singleton.
In your use case, a single module is in charge of providing the singleton to the whole application. So I would try to inject the mock in that module before it is used by anything else. The problem is that the mock cannot be fully constructed before the other classes are declared. A possible way is to construct the singleton in 2 passes: first pass does not depend on anything, then that minimal object is used to construct the classes and then its internal dictionnary is populated. Code could be:
import unittest
from .databasefactory import DataBaseClientInterFace
class MockedDataBaseClient(DataBaseClientInterFace):
_mocked_data = {} # no dependance outside databasefactory
def get(self, key):
return self._mocked_data[key]
def set(self, key, value):
self._mocked_data[key] = value
return value
# inject the mock into DataBaseFactory
from .databasefactory import DataBaseFactory
DataBaseFactory._database_client = MockedDataBaseClient()
# use the empty mock to construct other classes
from .datacontroller import DataBaseController
from .model import DataModel
# and populate the mock
DataBaseFactory._database_client._mocked_data.update(
{DataBaseController._special_key: 2,
DataModel.get_model_key(): {'model_name': 'mocked_test'}})
class SimpleOne(unittest.TestCase):
def test_controller(self):
"""
I want to mock the singleton instance referenced in both DataBaseController and DataModel
As DataBaseController imports DataModel, both classes have the DATA_BASE_CLIENT attributed instantiated with the factory result
"""
self.assertEqual(DataBaseController.get_some_special_key(), 2)
model = DataBaseController.load_a_model()
self.assertEqual('mocked_test', model.model_name)
But beware: this assumes that the test procedure does not load model.py or datacontroller.py before test_simple.py
Related
I have two classes, Manager and DataManager, simplified in the example below:
import numpy as np
class Manager:
def __init__(self, value, delay_init=True):
self.value = value
self.is_init = False
self.data = None
if not delay_init:
self._initialize()
#staticmethod
def delayed_init(fn):
def wrapped_delayed_init(obj, *args, **kwargs):
if not obj.is_init:
obj.data = np.random.randn(obj.value, obj.value)
obj.is_init = True
return fn(obj, *args, **kwargs)
return wrapped_delayed_init
#delayed_init.__get__(object)
def _initialize(self):
pass
class DataManager(Manager):
def __init__(self, value):
super().__init__(value)
#Manager.delayed_init
def calculate_mean(self):
return np.mean(self.data)
data_manager = DataManager(100)
assert data_manager.data is None
mean = data_manager.calculate_mean()
What my code needs to do is pass the method calculate as an argument to some other function as part of a test suite. In order to do this I need to create an instance of DataManager. What I must avoid is the time incurred by the full instance creation (since it involved downloading data), and so I delegate this task to some function in the parent class called delayed_init. There are a subset of methods belonging to DataManager that require this delayed_init to have been run, and so I choose to decorate them with delayed_init to ensure it is run whenever 1) another method requires it and 2) it has not already been run.
Now my problem: Currently it appears I need to explicitly define the decorator as #Manager.delayed_init, but this can be re-written as #<parent>.delayed_init. I would like to write it this way if possible given that in my opinion it is cleaner to not have to explicitly write out a given type if the type is always the parent. However, I cannot find a way to properly reference the parent class before an instance/object is created. Is it possible to access the parent class without the creation of any instances?
Thank you for the assistance.
I have defined a python context class and a Test class in a file:
class Test(object):
pass
class MyContext(object):
def __init(self):
self._vars = []
def __enter__(self):
pass
def __exit(self, ....):
pass
In another file using that context:
from somewhere import Test, MyContext
with MyContext() as ctx:
mytest = Test()
So what I want to achieve is that when I exit the context, I want to be aware of the mytest instance created and add it in the ctx._vars = [<instance of Test >].
I don't want to have a ctx.add_var(mytest) method, I want those Test instances to be added automatically to the ctx instance.
That is possible of being done, using Python's introspection capabilities, but you have to be aware this is not what the with context block was created for.
I agree it is a useful syntax construction that can be "deviated" to do things like what you want: annotate the objects created inside a code block in a "registry".
Before showing how to do that with a context manager consider if a class body would not suffice you. Using a class body this way also deviates from its primary purpose, but you have your "registry" for free:
from somewhere import Test, MyContext
class ctx:
mytest = Test()
vars = ctx.__dict__.values()
In order to do that with a context manager, you have to inspect the local variables at the start and at the end of the with block. While that is not hard to do, it wuld not cover all instances of Test created - because if the code is like this:
mytests = []
with Mycontext as ctx:
mytests.append(Test())
No new variable is created - so code tracking the local variables would not find anything. Code could be written to look recursively into variables with containers, such as dictionaries and lists - but then mytest() instances could be added to a container referenced as a global variable, or a variable in other module.
It turns out that a reliable way to track Test instances would be to instrument the Test class itself to annotate new instances ina registry. That is far easier and less depentend on "local variable introspection" tricks.
The code for that is somewhat like:
class Test(object):
pass
class MyContext(object):
def __init(self, *args):
self.vars = []
self.track = args
self.original_new = {}
def patch(self, cls_to_patch):
cls_new = getattr(cls_to_patch, "__new__")
if "__new__" in cls.__dict__:
self.original_new[cls_to_patch] = cls_new
def patched_new(cls, *args, **kwargs):
instance = cls_new(*args, **kwags)
self.vars.append(instance)
return instance
cls_to_patch.__new__ = patched_new
def restore(self, cls):
if cls in self.original_new:
# class had a very own __new_ prior to patching
cls.__new__ = self.original_new[cls]
else:
# just remove the wrapped new method, restores access to superclass `__new__`
del cls.__new__
def __enter__(self):
for cls in self.track:
self.patch(cls)
return self
def __exit(self, ....):
for cls in self.track:
self.restore(cls)
...
from somewhere import Test, MyContext
with MyContext(Test) as ctx:
mytest = Test()
The gist of the question: if inheriting multiple classes how can I guarantee that if one class is inherited, a compliment Abstract Base Class (abc) is also used by the child object.
I've been messing around with pythons inheritance trying to see what kind of cool stuff I can do and I came up with this pattern, which is kind of interesting.
I've been trying to use this make implementing and testing objects that interface with my cache easier. I've got three modules:
ICachable.py
Cacheable.py
SomeClass.py
ICacheable.py
import abc
class ICacheable(abc.ABC):
#property
#abc.abstractmethod
def CacheItemIns(self):
return self.__CacheItemIns
#CacheItemIns.setter
#abc.abstractmethod
def CacheItemIns(self, value):
self.__CacheItemIns = value
return
#abc.abstractmethod
def Load(self):
"""docstring"""
return
#abc.abstractmethod
def _deserializeCacheItem(self):
"""docstring"""
return
#abc.abstractmethod
def _deserializeNonCacheItem(self):
"""docstring"""
return
Cacheable.py
class Cacheable:
def _getFromCache(self, itemName, cacheType,
cachePath=None):
"""docstring"""
kwargs = {"itemName" : itemName,
"cacheType" : cacheType,
"cachePath" : cachePath}
lstSearchResult = CacheManager.SearchCache(**kwargs)
if lstSearchResult[0]:
self.CacheItemIns = lstSearchResult[1]
self._deserializeCacheItem()
else:
cacheItem = CacheManager.NewItem(**kwargs)
self.CacheItemIns = cacheItem
self._deserializeNonCacheItem()
return
SomeClass.py
import ICacheable
import Cacheable
class SomeClass(Cacheable, ICacheable):
__valueFromCache1:str = ""
__valueFromCache2:str = ""
__CacheItemIns:dict = {}
#property
def CacheItemIns(self):
return self.__CacheItemIns
#CacheItemIns.setter
def CacheItemIns(self, value):
self.__CacheItemIns = value
return
def __init__(self, itemName, cacheType):
#Call Method from Cacheable
self.__valueFromCache1
self.__valueFromCache2
self.__getItemFromCache(itemName, cacheType)
return
def _deserializeCacheItem(self):
"""docstring"""
self.__valueFromCache1 = self.CacheItemIns["val1"]
self.__valueFromCache2 = self.CacheItemIns["val2"]
return
def _deserializeNonCacheItem(self):
"""docstring"""
self.__valueFromCache1 = #some external function
self.__valueFromCache2 = #some external function
return
So this example works, but the scary thing is that there is no gurantee that a class inherriting Cacheable also inherits ICacheable. Which seems like a design flaw, as Cacheable is useless on its own. However the ability to abstract things from my subclass/child class with this is powerful. Is there a way to guarantee Cacheable's dependency on ICacheable?
If you explicitly do not want inheritance, you can register classes as virtual subclasses of an ABC.
#ICacheable.register
class Cacheable:
...
That means every subclass of Cacheable is automatically treated as subclass of ICacheable as well. This is mostly useful if you have an efficient implementation that would be slowed down by having non-functional Abstract Base Classes to traverse, e.g. for super calls.
However, ABCs are not just Interfaces and it is fine to inherit from them. In fact, part of the benefit of ABC is that it enforces subclasses to implement all abstract methods. An intermediate helper class, such as Cacheable, is fine not to implement all methods when it is never instantiated. However, any non-virtual subclass that is instantiated must be concrete.
>>> class FailClass(Cacheable, ICacheable):
... ...
...
>>> FailClass()
TypeError: Can't instantiate abstract class FailClass with abstract methods CacheItemIns, Load, _deserializeCacheItem, _deserializeNonCacheItem
Note that if you
always subclass as class AnyClass(Cacheable, ICacheable):
never instantiate Cacheable
that is functionally equivalent to Cacheable inheriting from ICacheable. The Method Resolution Order (i.e. the inheritance diamond) is the same.
>>> AnyClass.__mro__
(__main__. AnyClass, __main__.Cacheable, __main__.ICacheable, abc.ABC, object)
In the Flask-RESTful example application posted here, the TODOS collection is a global variable.
After the Todo Resource is registered:
api.add_resource(Todo, '/todos/<string:todo_id>')
The Todo methods access the global TODOS variable when web requests are processed.
Instead, I want to instantiate the API within a class and pass a TODOS collection that is a class variable rather than a global variable.
When using Flask-RESTful, what is the proper way to allow methods in a Resource class to gain access to a variable provided by the calling class without using global variables?
Looks like I didn't understand you the first time, You can just use a classmethod to construct your API. Then add it as a resource
from flask import Flask
from flask.ext.restful import Api
class SomeApi(Resource):
def get(self):
return self.response
#classmethod
def make_api(cls, response):
cls.response = response
return cls
class KillerApp(object):
def __init__(self):
self.app = Flask()
app_api = Api(self.app)
MyApi = SomeAPI.make_api({"key": "value"})
app_api.add_resource(MyApi, "/api/path")
def run(self)
self.app.run()
KillerApp().run()
add_resource accepts two arguments, resource_class_args and resource_class_kwargs, used to pass arguments to the constructor. (source)
So you could have a Resource:
from flask_restful import Resource
class TodoNext(Resource):
def __init__(self, **kwargs):
# smart_engine is a black box dependency
self.smart_engine = kwargs['smart_engine']
def get(self):
return self.smart_engine.next_todo()
You can inject the required dependency into TodoNext like so:
smart_engine = SmartEngine()
api.add_resource(TodoNext, '/next',
resource_class_kwargs={ 'smart_engine': smart_engine })
based on #Greg answer I've added an initialization check in the init method:
creating and calling Todo Resource class for flask-restful api:
todo = Todo.create(InMemoryTodoRepository())
api.add_resource(todo, '/api/todos/<todo_id>')
The Todo Resource class:
from flask_restful import reqparse, abort, Resource
from server.ApiResources.DTOs.TodoDTO import TodoDTO
from server.Repositories.ITodoRepository import ITodoRepository
from server.Utils.Exceptions import InvalidInstantiationError
from server.Utils.GeneralUtils import member_exists
class Todo(Resource):
"""shows a single todo item and lets you delete a todo item
use the 'create' class method to instantiate the class
"""
def __init__(self):
if not member_exists(self, "todo_repository", of_type=ITodoRepository):
raise InvalidInstantiationError("Todo", "todo_repository", "ITodoRepository", "create")
self._parser = reqparse.RequestParser()
self._parser.add_argument('task', type=str)
#classmethod
def create(cls, todo_repository):
"""
:param todo_repository: an instance of ITodoRepository
:return: class object of Todo Resource
"""
cls.todo_repository = todo_repository
return cls
the member_exists helper methods:
def member_exists(obj, member, of_type):
member_value = getattr(obj, member, None)
if member_value is None:
return False
if not isinstance(member_value, of_type):
return False
return True
and the custom exception class:
class InvalidInstantiationError(Exception):
def __init__(self, origin_class_name, missing_argument_name, missing_argument_type, instantiation_method_to_use):
message = """Invalid instantiation for class '{class_name}':
missing instantiation argument '{arg}' of type '{arg_type}'.
Please use the '{method_name}' factory class method""" \
.format(class_name=origin_class_name,
arg=missing_argument_name,
arg_type=missing_argument_type,
method_name=instantiation_method_to_use)
# Call the base class constructor with the parameters it needs
super(InvalidInstantiationError, self).__init__(message)
Thus, trying to use the default constructor will end up in getting this exception:
server.Utils.Exceptions.InvalidInstantiationError: Invalid instantiation for class 'Todo':
missing instantiation argument 'todo_repository' of type 'ITodoRepository'.
Please use the 'create' factory class method
edit: this can be useful for using dependency injection with flask-restful api Resource classes (with or without IoC)
edit 2:
we can even go cleaner and add another help function (ready to import):
def must_have(obj, member, of_type, use_method):
if not member_exists(obj, member, of_type=of_type):
raise InvalidInstantiationError(obj.__class__.__name__,
member,
of_type.__name__,
use_method)
and then use it in the constructor like that:
from server.Utils.GeneralUtils import must_have
class Todo(Resource):
def __init__(self):
must_have(self,
member="todo_repository",
of_type=ITodoRepository,
use_method=Todo.create.__name__)
I have one django model which needs to do some processing referring the custom user model.
I can't work with the class of this model at class loading time because the loading order of the classes is unknown.
So I need to add some class attributes at runtime, at the moment I'm adding them in the __init__ or __new__ like:
def __new__(cls, *args, **kwargs):
# hack to avoid INSTALLED_APPS initialization conflicts.
# get_user_model() can't be called from this module at class loading time,
# so some class attributes must be added later.
# Metaclasses could me more appropiate but I don't want to override
# dango's metaclasses.
if not hasattr(cls, '_reverse_field_name_to_user'):
cls._find_reverse_field_name_to_user()
return Group.__new__(cls, *args, **kwargs)
It works but looks horrible so I've thought about using something like #lazyclassproperty for these attributes.
I've found several #classproperty and #lazyproperty decorators but not one for both and I have no idea how to write one myself.
Question: How could I code such decorator? or suggest another cleaner alternative to my current silly implementation.
Pyramid framework has a very nice decorator called reify, but it only works at instance level, and you want class level, so let's modify it a bit
class class_reify(object):
def __init__(self, wrapped):
self.wrapped = wrapped
try:
self.__doc__ = wrapped.__doc__
except: # pragma: no cover
pass
# original sets the attributes on the instance
# def __get__(self, inst, objtype=None):
# if inst is None:
# return self
# val = self.wrapped(inst)
# setattr(inst, self.wrapped.__name__, val)
# return val
# ignore the instance, and just set them on the class
# if called on a class, inst is None and objtype is the class
# if called on an instance, inst is the instance, and objtype
# the class
def __get__(self, inst, objtype=None):
# ask the value from the wrapped object, giving it
# our class
val = self.wrapped(objtype)
# and set the attribute directly to the class, thereby
# avoiding the descriptor to be called multiple times
setattr(objtype, self.wrapped.__name__, val)
# and return the calculated value
return val
class Test(object):
#class_reify
def foo(cls):
print("foo called for class", cls)
return 42
print(Test.foo)
print(Test.foo)
Run the program and it prints
foo called for class <class '__main__.Test'>
42
42