I try to use the # in the class method. like this
class Dataset:
#parse_func
def get_next_batch(self):
return self.generator.__next__()
and the parse function like this:
def parse_func(load_batch):
def wrapper(**para):
batch_files_path, batch_masks_path, batch_label = load_batch(**para)
batch_images = []
batch_masks = []
for (file_path, mask_path) in zip(batch_files_path, batch_masks_path):
image = cv2.imread(file_path)
mask = cv2.imread(mask_path)
batch_images.append(image)
batch_masks.append(mask)
return np.asarray(batch_images, np.float32), np.asarray(batch_masks, np.uint8), batch_label
return wrapper
However, when I call dataset.get_next_batch(), it will raise a exception as followed.
Traceback (most recent call last):
TypeError: wrapper() takes exactly 0 arguments (1 given)
Do you know why raise this error and any solution? Thank you very much!
The function wrapper(**kwargs) accepts named arguments only. However, in instance methods, the self is automatically passed as the first positional argument. Since your method does not accept positional arguments, it fails.
You could edit to wrapper(self, **kwargs) or, more general wrapper(*args, **kwargs). However, the way you are using it, it is not clear what those arguments are.
Just simply change
def parse_func(load_batch):
def wrapper(*para):
batch_files_path, batch_masks_path, batch_label = load_batch(*para)
batch_images = []
batch_masks = []
for (file_path, mask_path) in zip(batch_files_path, batch_masks_path):
image = cv2.imread(file_path)
mask = cv2.imread(mask_path)
batch_images.append(image)
batch_masks.append(mask)
return np.asarray(batch_images, np.float32), np.asarray(batch_masks, np.uint8), batch_label
return wrapper()
# symbol mean a decorator function. Here, it means parse_func(get_next_batch). So if the wrapper using the keyword params (**para), you just want to pass some params to the wrapper but you don't actually except for the self args. So here I replace the params to positional params *para.
Related
I've been tinkering with decorators lately and (as an academic exercise) tried to implement a decorator that allows for partial application and/or currying of the decorated function. Furthermore this decorator should be optionally parameterizable and take a kwarg asap which determines if the decorated function should return as soon as all mandatory args/kwargs are aquired (default: asap=True) or if the decoratored function should keep caching args/kwargs until the function is called without arguments (asap=False).
Here is the decorator I came up with:
def partialcurry(_f=None, *, asap: bool=True):
""" Decorator; optionally parameterizable; Allows partial application /and/or/ currying of the decorated function F. Decorated F fires as soon as all mandatory args and kwargs are supplied, or, if ASAP=False, collects args and kwargs and fires only if F is called without args/kwargs. """
def _decor(f, *args, **kwargs):
_all_args, _all_kwargs = list(args), kwargs
#functools.wraps(f)
def _wrapper(*more_args, **more_kwargs):
nonlocal _all_args, _all_kwargs # needed for resetting, not mutating
_all_args.extend(more_args)
_all_kwargs.update(more_kwargs)
if asap:
try:
result = f(*_all_args, **_all_kwargs)
# reset closured args/kwargs caches
_all_args, _all_kwargs = list(), dict()
except TypeError:
result = _wrapper
return result
elif not asap:
if more_args or more_kwargs:
return _wrapper
else:
result = f(*_all_args, **_all_kwargs)
# again, reset closured args/kwargs caches
_all_args, _all_kwargs = list(), dict()
return result
return _wrapper
if _f is None:
return _decor
return _decor(_f)
### examples
#partialcurry
def fun(x, y, z=3):
return x, y, z
print(fun(1)) # preloaded function object
print(fun(1, 2)) # all mandatory args supplied; (1,1,2); reset
print(fun(1)(2)) # all mandatory args supplied; (1,2,3); reset
print()
#partialcurry(asap=False)
def fun2(x, y, z=3):
return x, y, z
print(fun2(1)(2, 3)) # all mandatory args supplied; preloaded function object
print(fun2()) # fire + reset
print(fun2(1)(2)) # all mandatory args supplied; preloaded function object
print(fun2(4)()) # load one more and fire + reset
I am sure that this can be generally improved (implementing this as a class would be a good idea for example) and any suggestions are much appreciated, my main question however is how to determine if all mandatory args/kwargs are supplied, because I feel like to check for a TypeError is too generic and could catch all kinds of TypeErrors. One idea would be to define a helper function that calculates the number of mandatory arguments, maybe something like this:
def _required_args_cnt(f):
""" Auxiliary function: Calculate the number of /required/ args of a function F. """
all_args_cnt = f.__code__.co_argcount + f.__code__.co_kwonlyargcount
def_args_cnt = len(f.__defaults__) if f.__defaults__ else 0
return all_args_cnt - def_args_cnt
Obviously unsatisfactory..
Any suggestions are much appreciated!
In the following example I'm trying to pass arguments to a function that itself has been passed as a kwarg. I have not been successful in passing arguments to the function 'func' from within the class 'TestClass' in the following example:
import sys, threading; from threading import Thread
def func(kwargs):
print('IN:', sys._getframe(0).f_code.co_name)
for key, value in kwargs.items() :
print ('KEY:', key, ', VAL:', value, sep='')
class TestClass(Thread):
def __init__(self, name = sys._getframe(0).f_code.co_name, kwargs = None):
Thread.__init__(self)
self.name = name
self.kwargs = kwargs
print('IN:', self.name)
def run(self):
func = self.kwargs['func']
func_kwargs_inner = {'arg_1': 'INNER-1', 'arg_2': 'INNER-2'}
func() # how to pass func_kwargs_inner to func?
def main():
func_kwargs = {'arg_1': 'OUTER-1', 'arg_2': 'OUTER-2'} # these get passed
# func_kwargs = {} # func_kwargs never gets populated
kwargs = {'func': (lambda: func(func_kwargs))}
test = TestClass(name='my-test', kwargs=kwargs)
test.start()
print('PROGRAM END')
if __name__ == '__main__':
main()
If I try to pass 'func_kwargs_inner' to 'func()', I get syntax errors; if I leave the argument list empty - as in the example - the result is:
IN: my-test
IN: func
KEY:arg_1, VAL:OUTER-1
KEY:arg_2, VAL:OUTER-2
PROGRAM END
whereas the required output once I find a way to pass the arguments correctly is:
IN: my-test
IN: func
KEY:arg_1, VAL:INNER-1
KEY:arg_2, VAL:INNER-2
PROGRAM END
How do I pass 'func_kwargs_inner' to 'func()'?
It seems that if you do the obvious thing, then it will work, and that your code at present explicitly avoids passing the arguments that you want. Specifically, in your TestClass.run you are not passing any arguments to func but instead relies on function arguments that are hard-coded into the lambda expression. So change your line:
func() # how to pass func_kwargs_inner to func?
to pass the arguments:
func(func_kwargs_inner)
Then in main, instead of that lambda expression:
kwargs = {'func': (lambda: func(func_kwargs))}
simply pass the function object itself:
kwargs = {'func': func}
Then you get the expected output:
IN: my-test
IN: func
PROGRAM END
KEY:arg_1, VAL:INNER-1
KEY:arg_2, VAL:INNER-2
I have a functions module that has some functions all with some common inputs, and others that are particular to them. e.g.
def func_a(time_series, window ='1D'):
def func_b(time_series, window ='1D', availability_history ):
def func_c(time_series, window ='1D', max_lag=25, use_probability ='T'):
I am trying to run these functions in a loop as follows:
func_list = [func_a, func_b, func_c]
windows = ['1D', '5D']
params = ['', hist, (25, 'T')]
for i_func, func in enumerate(func_list):
class_obj = class_X(A,B,func)
for window in windows:
args = (window, params[i_func]) # params is a list or tuple of other params for funcs e.g.
class_obj.run_func(args)
And in another module
class class_X(object):
def __init__(self, a, b, func_to_run):
self.a = a
self.ts = b
self.method = func_to_run
def generate_output(self, *args):
return self.method(self.ts, args) # time series is common and fixed for all, other params differ or change
The above code wouldn't work because I think the functions that I am calling need to be changed to make use of *argsrather than having fixed defined params.
I think *args is meant for functions where number of input params are not known, but I am trying to use it in a case where the number of input params is known, but varies across different functions in a loop.
Is there any fix for this where I don't have to modify the functions module and can still pass all the required params as a single object (e.g. list or tuple)?
EDIT-
macromoonshine's answer states I can use kwargs like this:
def generate_output(self, **kwargs):
return self.method(self.ts, kwargs)
With this modification you can call generate_outputs() as follows:
x.generate_outputs( window ='1D', max_lag=25, use_probability ='T')
where xis an instance of your class X
Can this be enhanced so I can pass args other than time_series and window as a lookup value in a loop e.g.
x.generate_outputs( window ='1D', params[iloop])
where
params[iloop] = max_lag=25, use_probability ='T'
I tried doing this:
params = (30, "F")
x.generate_outputs( window, *params)
but get an error
TypeError: generate_output() takes 1 positional argument but 4 were given
You can use the **kwargs instead which allows arbitrary keyword parameters. This should be easier than chinging each function. You have just to modify your generate_outputs() method in your code:
def generate_output(self, **kwargs):
return self.method(self.ts, kwargs)
With this modification you can call generate_outputs() as follows:
x.generate_outputs(time_series, window ='1D', max_lag=25, use_probability ='T')
where xis an instance of your class X.
If you want to pass the kwargs from a dict instead named parameter, you have to prefix the dictionary variable with **. The adapted code should look like this:
params = [{max_lag: 35, use_probability: 'F'}, ... ]
TS= [1,2,3,4]
for i_func, func in enumerate(func_list):
class_obj = class_X(TS, func)
for window in windows:
req_args = dict(params[i_func])
req_args['window'] = 0
class_obj.generate_output(**req_args)
Here is definition of my class full of static functions. I want to use all of them in "sendLog" function which call himself with time interval (10 sec here). When I run this interpreter tells me "TypeError: sendLog() takes at least 5 arguments (0 given)"
But it if I enter the same params I will need to define sendLog again and again because it calls himself repeatly.. I know its not the way But cant figure it out.
class AccessLog:
#staticmethod
def backupAccessLog(target, source):
newfile = os.path.splitext(source)[0] + "_" + time.strftime("%Y%m%d-%H%M%S") + os.path.splitext(source)[1]
copyfile(source,newfile)
shutil.move(newfile,target)
#staticmethod
def emptyAccessLog(filename):
open(filename, 'w').close()
#staticmethod
def postLogstoElastic():
fileLogs = open("example.log", "rw+")
fileBackups = open("logs_of_accesslog.log","rw+")
lines = fileLogs.read().splitlines()
logging.basicConfig(format='%(asctime)s>>>%(message)s',filename='logs_exceptions.log',level=logging.DEBUG)
es = Elasticsearch(['http://localhost:9200/'], verify_certs=True)
#es.create(index="index_log23June", doc_type="type_log23June")
es.indices.create(index='index_log23June', ignore=400)
i=0
for item in lines:
try:
i+=1
if bool(item):
es.index(index="index_log23June",doc_type="type_log23June", body={"Log":item})
else:
print "a speace line ignored. at line number:", i
raise ValueError('Error occurred on this line: ', i)
print "lines[",i,"]:",item,"\n"
except ValueError as err:
logging.error(err.args)
#staticmethod
def sendLog(interval, worker_functions, iterations=1):
def call_worker_functions():
for f in worker_functions:
f() #ERROR: Msg: 'NoneType' object is not callable
for i in range(iterations):
threading.Timer(interval * i, call_worker_functions).start()
and I want to call this method with this line:
try:
AccessLog.AccessLog.sendLog(
interval=10,
worker_functions=(
AccessLog.AccessLog.backupAccessLog("logbackups","example.log"),
AccessLog.AccessLog.emptyAccessLog("example.log"),
AccessLog.AccessLog.postLogstoElastic()
),
iterations=999
)
except ValueError as err:
logging.error(err.args)
"TypeError: sendLog() takes at least 5 arguments (0 given)" It looks normal but How can I handle this ?
Have you tried to set the #staticmethod on the same level as the function?
Apparently you want sendLog() to call the worker functions every 10 seconds or so.
Here's an easy way to do that:
class AccessLog:
#staticmethod
def sendLog(interval, worker_functions, iterations=1):
def call_worker_functions():
for f in worker_functions:
f(*worker_functions[f])
for i in range(iterations):
threading.Timer(interval * i, call_worker_functions).start()
And now use it like this:
AccessLog.AccessLog.sendLog(
interval=10,
worker_functions={
AccessLog.AccessLog.backupAccessLog: ("logbackups", "example.log"),
AccessLog.AccessLog.emptyAccessLog: ("example.log",),
AccessLog.AccessLog.postLogstoElastic: ()
),
iterations=999
)
And this is just one of many many ways, but there's no need to pass the function as its own argument like you did.
Sorry if I did not explain myself clearly.
I would like to create a wrapper to call pre-defined functions with different number of inputs. Of course, I can create an individual wrapper for each function, but I am wondering if there is a way to create a generic wrapper for all cases.
The functions that should be called are named 'fun1' and 'fun2' with different number of inputs. I need to create a wrapper 'fun_wrap(func_name, uncertain amount of inputs)', which only needs the function name to be called and its associated amount of inputs.
One more thing, I need to change the input names by adding '_in' and make them global variables first. Below is my broken code. Thanks for any suggestions!
def fun1(a,b):
return a+b
def fun2(a,b,c):
return a*b/c
def set_globals(**kwargs):
for argname in kwargs:
globals()['%s_in' % argname] = kwargs[argname]
def fun_wrap(func_name, uncertain amount of inputs):
ffunc_name(set_globals(uncertain amount of inputs))
In this way, if I can call final_fun with arguments like:
fun_wrap(fun1,a,b)
fun_wrap(fun2,a,b)
UPDATE
I tried to use *arg, but failed...
def fun1(a,b):
return a+b
def fun2(a,b,c):
return a*b/c
def set_globals(**kwargs):
for argname in kwargs:
globals()['%s_in' % argname] = kwargs[argname]
def fun_wrap(func_name, *arg):
func_name(set_globals(*arg))
fun_wrap(fun2,a=1,b=2,c=3)
got error:
Traceback (most recent call last):
File "D:\Dropbox\AppPest\rice\try.py", line 19, in <module>
fun_wrap(fun2,a=1,b=2,c=3)
TypeError: fun_wrap() got an unexpected keyword argument 'a'
def fun1(a,b):
return a + b
def fun2(a,b,c):
return a * b / c
def set_globals(**kwargs):
for argname in kwargs:
globals()['%s_in' % argname] = kwargs[argname]
def fun_wrap(func, **kwargs):
set_globals(**kwargs) # made the call to set_globals before calling your function
return func(**kwargs) # return the value returned by the functions called