I have a working solution for what I am trying to achieve, but I am looking for simpler way to do it.
I have a class that encapsulates a function and a user can pass a function (lambda expression) to it. Those functions always take one input data argument and an arbitrary amount of user defined custom-arguments:
c.set_func(lambda x, offset, mul: mul*(x**2 + offset), offset=3, mul=1)
The user can then call a class method that will run the function with a predefined input and the currently set custom-arguments. The user also has the option to change the custom-arguments by just changing attributes of the class.
Here is my code:
from functools import partial
class C:
def __init__(self):
self.data = 10 # just an example
self.func = None
self.arg_keys = []
def set_func(self, func, **kwargs):
self.func = func
for key, value in kwargs.iteritems():
# add arguments to __dict__
self.__dict__[key] = value
self.arg_keys.append(key)
# store a list of the argument names
self.arg_keys = list(set(self.arg_keys))
def run_function(self):
# get all arguments from __dict__ that are in the stored argument-list
argdict = {key: self.__dict__[key] for key in self.arg_keys}
f = partial(self.func, **argdict)
return f(self.data)
if __name__ == '__main__':
# Here is a testrun:
c = C()
c.set_func(lambda x, offset, mul: mul*(x**2 + offset), offset=3, mul=1)
print c.run_function()
# -> 103
c.offset = 5
print c.run_function()
# -> 105
c.mul = -1
print c.run_function()
# -> -105
The important part are:
that the user can initially set the function with any number of arguments
The values of those arguments are stored until changed
Is there any builtin or otherwise simpler solution to this?
Related
Given a class with class methods that contain only self input:
class ABC():
def __init__(self, input_dict)
self.variable_0 = input_dict['variable_0']
self.variable_1 = input_dict['variable_1']
self.variable_2 = input_dict['variable_2']
self.variable_3 = input_dict['variable_3']
def some_operation_0(self):
return self.variable_0 + self.variable_1
def some_operation_1(self):
return self.variable_2 + self.variable_3
First question: Is this very bad practice? Should I just refactor some_operation_0(self) to explicitly take the necessary inputs, some_operation_0(self, variable_0, variable_1)? If so, the testing is very straightforward.
Second question: What is the correct way to setup my unit test on the method some_operation_0(self)?
Should I setup a fixture in which I initialize input_dict, and then instantiate the class with a mock object?
#pytest.fixture
def generator_inputs():
f = open('inputs.txt', 'r')
input_dict = eval(f.read())
f.close()
mock_obj = ABC(input_dict)
def test_some_operation_0():
assert mock_obj.some_operation_0() == some_value
(I am new to both python and general unit testing...)
Those methods do take an argument: self. There is no need to mock anything. Instead, you can simply create an instance, and verify that the methods return the expected value when invoked.
For your example:
def test_abc():
a = ABC({'variable_0':0, 'variable_1':1, 'variable_2':2, 'variable_3':3))
assert a.some_operation_0() == 1
assert a.some_operation_1() == 5
If constructing an instance is very difficult, you might want to change your code so that the class can be instantiated from standard in-memory data structures (e.g. a dictionary). In that case, you could create a separate function that reads/parses data from a file and uses the "data-structure-based" __init__ method, e.g. make_abc() or a class method.
If this approach does not generalize to your real problem, you could imagine providing programmatic access to the key names or other metadata that ABC recognizes or cares about. Then, you could programmatically construct a "defaulted" instance, e.g. an instance where every value in the input dict is a default-constructed value (such as 0 for int):
class ABC():
PROPERTY_NAMES = ['variable_0', 'variable_1', 'variable_2', 'variable_3']
def __init__(self, input_dict):
# implementation omitted for brevity
pass
def some_operation_0(self):
return self.variable_0 + self.variable_1
def some_operation_1(self):
return self.variable_2 + self.variable_3
def test_abc():
a = ABC({name: 0 for name in ABC.PROPERTY_NAMES})
assert a.some_operation_0() == 0
assert a.some_operation_1() == 0
I have the following code, in which I simply have a decorator for caching a function's results, and as a concrete implementation, I used the Fibonacci function.
After playing around with the code, I wanted to print the cache variable, that's initiated in the cache wrapper.
(It's not because I suspect the cache might be faulty, I simply want to know how to access it without going into debug mode and put a breakpoint inside the decorator)
I tried to explore the fib_w_cache function in debug mode, which is supposed to actually be the wrapped fib_w_cache, but with no success.
import timeit
def cache(f, cache = dict()):
def args_to_str(*args, **kwargs):
return str(args) + str(kwargs)
def wrapper(*args, **kwargs):
args_str = args_to_str(*args, **kwargs)
if args_str in cache:
#print("cache used for: %s" % args_str)
return cache[args_str]
else:
val = f(*args, **kwargs)
cache[args_str] = val
return val
return wrapper
#cache
def fib_w_cache(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_w_cache(n-2) + fib_w_cache(n-1)
def fib_wo_cache(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_wo_cache(n-1) + fib_wo_cache(n-2)
print(timeit.timeit('[fib_wo_cache(i) for i in range(0,30)]', globals=globals(), number=1))
print(timeit.timeit('[fib_w_cache(i) for i in range(0,30)]', globals=globals(), number=1))
I admit this is not an "elegant" solution in a sense, but keep in mind that python functions are also objects. So with some slight modification to your code, I managed to inject the cache as an attribute of a decorated function:
import timeit
def cache(f):
def args_to_str(*args, **kwargs):
return str(args) + str(kwargs)
def wrapper(*args, **kwargs):
args_str = args_to_str(*args, **kwargs)
if args_str in wrapper._cache:
#print("cache used for: %s" % args_str)
return wrapper._cache[args_str]
else:
val = f(*args, **kwargs)
wrapper._cache[args_str] = val
return val
wrapper._cache = {}
return wrapper
#cache
def fib_w_cache(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_w_cache(n-2) + fib_w_cache(n-1)
#cache
def fib_w_cache_1(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_w_cache(n-2) + fib_w_cache(n-1)
def fib_wo_cache(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_wo_cache(n-1) + fib_wo_cache(n-2)
print(timeit.timeit('[fib_wo_cache(i) for i in range(0,30)]', globals=globals(), number=1))
print(timeit.timeit('[fib_w_cache(i) for i in range(0,30)]', globals=globals(), number=1))
print(fib_w_cache._cache)
print(fib_w_cache_1._cache) # to prove that caches are different instances for different functions
cache is of course a perfectly normal local variable in scope within the cache function, and a perfectly normal nonlocal cellvar in scope within the wrapper function, so if you want to access the value from there, you just do it—as you already are.
But what if you wanted to access it from somewhere else? Then there are two options.
First, cache happens to be defined at the global level, meaning any code anywhere (that hasn't hidden it with a local variable named cache) can access the function object.
And if you're trying to access the values of a function's default parameters from outside the function, they're available in the attributes of the function object. The inspect module docs explain the inspection-oriented attributes of each builtin type:
__defaults__ is a sequence of the values for all positional-or-keyword parameters, in order.
__kwdefaults__ is a mapping from keywords to values for all keyword-only parameters.
So:
>>> def f(a, b=0, c=1, *, d=2, e=3): pass
>>> f.__defaults__
(0, 1)
>>> f.__kwdefaults__
{'e': 3, 'd': 2}
So, for a simple case where you know there's exactly one default value and know which argument it belongs to, all you need is:
>>> cache.__defaults__[0]
{}
If you need to do something more complicated or dynamic, like get the default value for c in the f function above, you need to dig into other information—the only way to know that c's default value will be the second one in __defaults__ is to look at the attributes of the function's code object, like f.__code__.co_varnames, and figure it out from there. But usually, it's better to just use the inspect module's helpers. For example:
>>> inspect.signature(f).parameters['c'].default
1
>>> inspect.signature(cache).parameters['cache'].default
{}
Alternatively, if you're trying to access the cache from inside fib_w_cache, while there's no variable in lexical scope in that function body you can look at, you do know that the function body is only called by the decorator wrapper, and it is available there.
So, you can get your stack frame
frame = inspect.currentframe()
… follow it back to your caller:
back = frame.f_back
… and grab it from that frame's locals:
back.f_locals['cache']
It's worth noting that f_locals works like the locals function: it's actually a copy of the internal locals storage, so modifying it may have no effect, and that copy flattens nonlocal cell variables to regular local variables. If you wanted to access the actual cell variable, you'd have to grub around in things like back.f_code.co_freevars to get the index and then dig it out of the function object's __closure__. But usually, you don't care about that.
Just for a sake of completeness, python has caching decorator built-in in functools.lru_cache with some inspecting mechanisms:
from functools import lru_cache
#lru_cache(maxsize=None)
def fib_w_cache(n):
if n == 0: return 0
elif n == 1: return 1
else:
return fib_w_cache(n-2) + fib_w_cache(n-1)
print('fib_w_cache(10) = ', fib_w_cache(10))
print(fib_w_cache.cache_info())
Prints:
fib_w_cache(10) = 55
CacheInfo(hits=8, misses=11, maxsize=None, currsize=11)
I managed to find a solution (in some sense by #Patrick Haugh's advice).
I simply accessed cache.__defaults__[0] which holds the cache's dict.
The insights about the shared cache and how to avoid it we're also quite useful.
Just as a note, the cache dictionary can only be accessed through the cache function object. It cannot be accessed through the decorated functions (at least as far as I understand). It logically aligns well with the fact that the cache is shared in my implementation, where on the other hand, in the alternative implementation that was proposed, it is local per decorated function.
You can make a class into a wrapper.
def args_to_str(*args, **kwargs):
return str(args) + str(kwargs)
class Cache(object):
def __init__(self, func):
self.func = func
self.cache = {}
def __call__(self, *args, **kwargs):
args_str = args_to_str(*args, **kwargs)
if args_str in self.cache:
return self.cache[args_str]
else:
val = self.func(*args, **kwargs)
self.cache[args_str] = val
return val
Each function has its own cache. you can access it by calling function.cache. This also allows for any methods you wish to attach to your function.
If you wanted all decorated functions to share the same cache, you could use a class variable instead of an instance variable:
class SharedCache(object):
cache = {}
def __init__(self, func):
self.func = func
#rest of the the code is the same
#SharedCache
def function_1(stuff):
things
I am creating a class to make some calculations. The class would have 3 arguments to get started. I have done like this in a simplified representation:
class TheCalcs:
def __init__(self, pk_from_db, cat_score_list, final_score):
self.pk_from_db = pk_from_db
self.cat_score_list = cat_score_list
self.final_score = final_score
def calculate_cat_score(self):
#Do some calcs with the data of the pk_from_db and return that!
a_list_of_scores = [] # create a list of scores
return a_list_of_scores
def final_score(self): # The argument for this function would be the return of the calculate_cat_score function!
# Again do some calcs and return the final score
the_final_score = int()
return the_final_score
def score_grade(self): # the argument this this function again the return but now from the final_score function
# Do some cals and return the grade
the_grade = ("a string", "an integer")
return the_grade
When I call the class I would have to present the arguments --> However as you can see I just do now the value of the first argument. The second and the third being calculated throughout the class. When I call the class just with one argument I will of course have an error of failing arguments. Anyone has an idea on that?
If those values are calculated, simply don't make them arguments. You could instead call those calculation methods to compute the values:
class TheCalcs:
def __init__(self, pk_from_db):
self.pk_from_db = pk_from_db
self.cat_score_list = self.calculate_cat_score()
self.final_score = self.calculate_final_score()
# ...
or postpone calculations until you need them.
I am trying to build some variable dependencies in Python. For example, if a = x, b = y and c = a + b, then if a or b changes the value of c should be automatically updated. I am aware the Python variables and values work on the basis of tags and have been trying to work around this using __setattr__. I seem to be having some trouble doing this, due to the cyclic dependency in __setattr__.
Consider this small code snippet:
class DelayComponents(object):
'''
Delay Components Class
'''
def __init__(self, **delays):
'''
Constructor
'''
self.prop_delay = round(float(delays['prop_delay']), 2)
self.trans_delay = round(float(delays['trans_delay']), 2)
self.proc_delay = round(float(delays['proc_delay']), 2)
self.queue_delay = round(float(delays['queue_delay']), 2)
self.delay = (self.prop_delay + self.proc_delay +
self.trans_delay + self.queue_delay)
def __setattr__(self, key, value):
self.__dict__[key] = value
if (key in ("prop_delay", "trans_delay",
"proc_delay", "queue_delay")):
self.delay = (self.prop_delay + self.proc_delay +
self.trans_delay + self.queue_delay)
This seems to serve the purpose well, but when I create an object of DelayComponents for the first time, since __setattr__ has been overridden and is called for each of the values being created, the if check inside __setattr__ throws an error saying the remaining three variables have not been found (which is true, since they have not yet been created).
How do I resolve this dependency?
Also, is there some way to accomplish the same with a dict? More specifically, if the three variables were actually key-value pairs in a dict, where the third key's value was the sum of the values of the first two keys, would it be possible to update the third value automatically when either of the first two changes?
Assuming that you want zero default values for the unset _delays (in both __init__ and __setattr__) you could do something like:
class DelayComponents(object):
'''
Delay Components Class
'''
ATTRS = ['prop_delay', 'trans_delay', 'proc_delay', 'queue_delay']
def __init__(self, **delays):
'''
Constructor
'''
for attr in self.ATTRS:
setattr(self, attr, round(float(delays.get(attr, 0)), 2))
# No point in setting delay here - it's already done!
def __setattr__(self, key, value):
super(DelayComponents, self).__setattr__(key, value)
# This avoids directly interacting with the __dict__
if key in self.ATTRS:
self.delay = sum(getattr(self, attr, 0) for attr in self.ATTRS)
In use:
>>> d = DelayComponents(prop_delay=1, trans_delay=2, proc_delay=3, queue_delay=4)
>>> d.delay
10.0
Should you want different defaults for different attributes, DelayComponents.ATTRS could be a dictionary {'attribute_name': default_value, ...}.
A much simpler alternative is to make delay a #property, that is calculated only as required:
class DelayComponents(object):
'''
Delay Components Class
'''
ATTRS = ['prop_delay', 'trans_delay', 'proc_delay', 'queue_delay']
def __init__(self, **delays):
'''
Constructor
'''
for attr in self.ATTRS:
setattr(self, attr, round(float(delays.get(attr, 0)), 2))
#property
def delay(self):
return sum(getattr(self, attr, 0) for attr in self.ATTRS)
To answer your sub-question: no, there's no way to do this with a vanilla dict; the values for keys aren't reevaluated based on changes to the values from which they're calculated.
Also, in all seriousness, there is no point to your current docstrings; you might as well leave them out entirely. They provide no information, and aren't compliant with PEP-257 either.
I've been working on a way to get tests produced from a generator in nose to have descriptions that are customized for the specific iteration being tested. I have something that works, as long as my generator target method never tries to access self from my generator class. I'm seeing that all my generator target instances have a common test class instance while nose is generating a one-offed instance of the test class for each test run from the generator. This is resulting in setUp being run on each test instance nose creates, but never running on the instance the generator target is bound to (of course, the real problem is that I can't see how to bind the nose-created instance to the generator target). Here's the code I'm using to try to figure this all out (yes, I know the decorator would probably be better as a callable class, but nose, at least version 1.2.1 that I have, explicitly checks that tests are either functions or methods, so a callable class won't run at all):
import inspect
def labelable_yielded_case(case):
argspec = inspect.getargspec(case)
if argspec.defaults is not None:
defaults_list = [''] * (len(argspec.args) - len(argspec.defaults)) + argspec.defaults
else:
defaults_list = [''] * len(argspec.args)
argument_defaults_list = zip(argspec.args, defaults_list)
case_wrappers = []
def add_description(wrapper_id, argument_dict):
case_wrappers[wrapper_id].description = case.__doc__.format(**argument_dict)
def case_factory(*factory_args, **factory_kwargs):
def case_wrapper_wrapper():
wrapper_id = len(case_wrappers)
def case_wrapper(*args, **kwargs):
args = factory_args + args
argument_list = []
for argument in argument_defaults_list:
argument_list.append(list(argument))
for index, value in enumerate(args):
argument_list[index][1] = value
argument_dict = dict(argument_list)
argument_dict.update(factory_kwargs)
argument_dict.update(kwargs)
add_description(wrapper_id, argument_dict)
return case(*args, **kwargs)
case_wrappers.append(case_wrapper)
case_wrapper.__name__ = case.__name__
return case_wrapper
return case_wrapper_wrapper()
return case_factory
class TestTest(object):
def __init__(self):
self.data = None
def setUp(self):
print 'setup', self
self.data = (1,2,3)
def test_all(self):
for index, value in enumerate((1,2,3)):
yield self.validate_equality(), index, value
def test_all_again(self):
for index, value in enumerate((1,2,3)):
yield self.validate_equality_again, index, value
#labelable_yielded_case
def validate_equality(self, index, value):
'''element {index} equals {value}'''
print 'test', self
assert self.data[index] == value, 'expected %d got %d' % (value, self.data[index])
def validate_equality_again(self, index, value):
print 'test', self
assert self.data[index] == value, 'expected %d got %d' % (value, self.data[index])
validate_equality_again.description = 'again'
When run through nose, the again tests work just fine, but the set of tests using the decorated generator target all fail because self.data is None (because setUp is never run because the instance of TestTest stored in the closures is not the instances run by nose). I tried making the decorator an instance member of a base class for TestTest, but then nose threw errors about having too few arguments (no self) passed to the unbound labelable_yielded_case. Is there any way I can make this work (short of hacking nose), or am I stuck choosing between either not being able to have the yield target be an instance member or not having per-test labeling for each yielded test?
Fixed it (at least for the case here, though I think I got it for all cases). I had to fiddle with case_wrapper_wrapper and case_wrapper to get the factory to return the wrapped cases attached to the correct class, but not bound to any given instance in any way. I also had another code issue because I was building the argument dict in wrapper wrapper, but then not passing it to the case. Working code:
import inspect
def labelable_yielded_case(case):
argspec = inspect.getargspec(case)
if argspec.defaults is not None:
defaults_list = [''] * (len(argspec.args) - len(argspec.defaults)) + argspec.defaults
else:
defaults_list = [''] * len(argspec.args)
argument_defaults_list = zip(argspec.args, defaults_list)
case_wrappers = []
def add_description(wrapper_id, argument_dict):
case_wrappers[wrapper_id].description = case.__doc__.format(**argument_dict)
def case_factory(*factory_args, **factory_kwargs):
def case_wrapper_wrapper():
wrapper_id = len(case_wrappers)
def case_wrapper(*args, **kwargs):
argument_list = []
for argument in argument_defaults_list:
argument_list.append(list(argument))
for index, value in enumerate(args):
argument_list[index][1] = value
argument_dict = dict(argument_list)
argument_dict.update(kwargs)
add_description(wrapper_id, argument_dict)
return case(**argument_dict)
case_wrappers.append(case_wrapper)
case_name = case.__name__ + str(wrapper_id)
case_wrapper.__name__ = case_name
if factory_args:
setattr(factory_args[0].__class__, case_name, case_wrapper)
return getattr(factory_args[0].__class__, case_name)
else:
return case_wrapper
return case_wrapper_wrapper()
return case_factory
class TestTest(object):
def __init__(self):
self.data = None
def setUp(self):
self.data = (1,2,3)
def test_all(self):
for index, value in enumerate((1,2,3)):
yield self.validate_equality(), index, value
#labelable_yielded_case
def validate_equality(self, index, value):
'''element {index} equals {value}'''
assert self.data[index] == value, 'expected %d got %d' % (value, self.data[index])