In order to use multiprocessing in interactive python on windows (miniconda), I've found a very useful code that works very well. The code, however, can not pass the self argument inside a class to the function to be pooled. Here is my code that works on google colab but never finishes on windows iPython:
import multiprocessing
from multiprocessing import Pool
from poolable import make_applicable, make_mappable
def worker(d):
"""worker function"""
for i in range(10000):
j = i **(1/3) + d.bias
return j
class dummy():
def __init__(self):
self.bias = 1000
def calc(self):
pool = Pool(processes=12)
results = {}
for i in range(5):
results[i] = (pool.apply_async(*make_applicable(worker,self)))
pool.close()
pool.join()
print([results[i].get() for i in range(5)])
d=dummy()
d.calc()
The code works well on windows if I pass other types of variables, for example:
results[i] = (pool.apply_async(*make_applicable(worker,self.bias)))
But when I pass self to the function the process never finishes. I have no idea what to do.
poolable.py from here:
from types import FunctionType
import marshal
def _applicable(*args, **kwargs):
name = kwargs['__pw_name']
code = marshal.loads(kwargs['__pw_code'])
gbls = globals() #gbls = marshal.loads(kwargs['__pw_gbls'])
defs = marshal.loads(kwargs['__pw_defs'])
clsr = marshal.loads(kwargs['__pw_clsr'])
fdct = marshal.loads(kwargs['__pw_fdct'])
func = FunctionType(code, gbls, name, defs, clsr)
func.fdct = fdct
del kwargs['__pw_name']
del kwargs['__pw_code']
del kwargs['__pw_defs']
del kwargs['__pw_clsr']
del kwargs['__pw_fdct']
return func(*args, **kwargs)
def make_applicable(f, *args, **kwargs):
if not isinstance(f, FunctionType): raise ValueError('argument must be a function')
kwargs['__pw_name'] = f.__name__ # edited
kwargs['__pw_code'] = marshal.dumps(f.__code__) # edited
kwargs['__pw_defs'] = marshal.dumps(f.__defaults__) # edited
kwargs['__pw_clsr'] = marshal.dumps(f.__closure__) # edited
kwargs['__pw_fdct'] = marshal.dumps(f.__dict__) # edited
return _applicable, args, kwargs
def _mappable(x):
x,name,code,defs,clsr,fdct = x
code = marshal.loads(code)
gbls = globals() #gbls = marshal.loads(gbls)
defs = marshal.loads(defs)
clsr = marshal.loads(clsr)
fdct = marshal.loads(fdct)
func = FunctionType(code, gbls, name, defs, clsr)
func.fdct = fdct
return func(x)
def make_mappable(f, iterable):
if not isinstance(f, FunctionType): raise ValueError('argument must be a function')
name = f.__name__ # edited
code = marshal.dumps(f.__code__) # edited
defs = marshal.dumps(f.__defaults__) # edited
clsr = marshal.dumps(f.__closure__) # edited
fdct = marshal.dumps(f.__dict__) # edited
return _mappable, ((i,name,code,defs,clsr,fdct) for i in iterable)
Edit:
It seems that the problem exist not only for self but also for any other classes that pass to the make_applicable function. The following code also don't finish:
class dummy2():
def __init__(self):
self.bias = 1000
class dummy():
def __init__(self):
self.bias = 1000
def copy(self):
return copy.deepcopy(self)
def calc(self):
pool = Pool(processes=12)
results = {}
for i in range(5):
d = dummy2()
results[i] = pool.apply_async(*make_applicable(worker,d))
pool.close()
pool.join()
print([results[i].get() for i in range(5)])
Using IPython console:
Put your code in a module (mp.py) ensuring the the class instantiation and method call are executed in a if __name__ == '__main__': conditional
import multiprocessing
from multiprocessing import Pool
from poolable import make_applicable, make_mappable
def worker(d):
"""worker function"""
for i in range(10000):
j = i **(1/3) + d.bias
return j
class Dummy():
def __init__(self):
# self.bias = 1000
self.bias = 10
def calc(self):
pool = Pool(processes=12)
results = {}
for i in range(5):
results[i] = (pool.apply_async(*make_applicable(worker,self)))
pool.close()
pool.join()
return [results[i].get() for i in range(5)]
if __name__ == '__main__':
d=Dummy()
print(d.calc())
Then in the console run the module:
In [1]: runfile('P:/pyProjects3/mp.py', wdir='P:/pyProjects3')
[31.543628731482663, 31.543628731482663, 31.543628731482663, 31.543628731482663, 31.543628731482663]
I have Jupyter Notebook (Anaconda) but don't know how to use it, if I figure it out I'll update this answer.
try to use this code to test
import pickle
def worker(d):
"""worker function"""
d = pickle.loads(d)
for i in range(10000):
j = i **(1/3) + d.bias
return j
class dummy():
def __init__(self):
self.bias = 1000
def calc(self):
pool = Pool(processes=12)
results = {}
for i in range(5):
x = pickle.dumps(self)
results[i] = (pool.apply_async(*make_applicable(worker,x)))
pool.close()
pool.join()
print([results[i].get() for i in range(5)])
Related
I was trying to make a loop function that uses class, and to make it easier i think i will make it as decorator, but there are a few problem when i tried my code,
When i tried this code, it says that I'm missing 1 required argument and it's "self"
I tried doing this to my code, but realized that it used __call__, which i don't want to because it will executed at the time the function is called. (I dont know if this true)
And this is how i used it, and https://hastebin.com/onozarogac.py (for the full code):
#looper(seconds=3) # this code is from other class
def print(self):
print(len(self.cache.get_available_guilds_view()))
self.print.start() # and by using this to trigger it
def looper(seconds):
def decorator(func):
return Looper(func, seconds * 1000)
return decorator
class Looper:
def __init__(self, fn, seconds):
self.fn = fn
self.interval_ms = seconds / 1000
self.time_last = time()
self.started = False
def loop(self):
time_now = time()
if time_now >= self.time_last + self.interval_ms:
self.fn()
self.time_last = time_now
return True
else:
return False
def set_interval_ms(self, interval_ms):
self.interval_ms = interval_ms / 1000
def set_interval_s(self, set_interval):
self.interval_ms = set_interval
def get_interval_ms(self):
return self.interval_ms
def get_interval_s(self):
return self.interval_ms * 1000
def start(self):
self.started = True
while True:
if self.started:
self.loop()
else:
break
def stop(self):
self.started = False
Its giving me the TypeError which:
TypeError: print() missing 1 required positional argument: 'self'
convert looper function to this
from functools import wraps
def looper(seconds):
def inner(func):
#wraps(func)
def wrapper(*args, **kwargs):
return Looper(func, seconds * 1000)
return wrapper
return inner
I keep running into this error that I never had before every time I try running python run_p2pool.py, I even tried doing -n and typing in the argument and that doesn't fix it (as the internet suggested). I also have the configurations already there and even tried reinstalling python twisted with no avail. There is no error in the code I am trying to run and I am yet to figure out how to get this fixed.
# Twisted doesn't have abortConnection! Upgrade to a newer version of Twisted to avoid memory leaks!
> Pausing for 3 seconds...
> 2017-11-02 01:07:47.958817 > Traceback (most recent call last):
> 2017-11-02 01:07:47.958986 > File "run_p2pool.py", line 5, in <module>
> 2017-11-02 01:07:47.959116 > main.run()
> 2017-11-02 01:07:47.959191 > File "/var/www/html/DashPool/DPI-MASTER/Dash-Pool-Integration/p2pool/main.py", line 687, in run
> 2017-11-02 01:07:47.959422 > deferral.RobustLoopingCall(logfile.reopen).start(5)
> 2017-11-02 01:07:47.959490 > File "/var/www/html/DashPool/DPI-MASTER/Dash-Pool-Integration/p2pool/util/deferral.py", line 277, in start
> 2017-11-02 01:07:47.959605 > self._df = self._worker(period).addErrback(lambda fail: fail.trap(defer.CancelledError))
> 2017-11-02 01:07:47.959686 > File "/var/www/html/DashPool/DPI-MASTER/Dash-Pool-Integration/p2pool/util/deferral.py", line 225, in _
> 2017-11-02 01:07:47.960104 > df = defer.Deferred(cancelled)
> 2017-11-02 01:07:47.960195 > TypeError: __init__() takes exactly 1 argument (2 given)
I never had this problem before and was able to run the same program before, but just encase, here is the defferal.py file
from __future__ import division
import itertools
import random
import sys
from twisted.internet import defer, reactor
from twisted.python import failure, log
def sleep(t):
d = defer.Deferred(canceller=lambda d_: dc.cancel())
dc = reactor.callLater(t, d.callback, None)
return d
def run_repeatedly(f, *args, **kwargs):
current_dc = [None]
def step():
delay = f(*args, **kwargs)
current_dc[0] = reactor.callLater(delay, step)
step()
def stop():
current_dc[0].cancel()
return stop
class RetrySilentlyException(Exception):
pass
def retry(message='Error:', delay=3, max_retries=None, traceback=True):
'''
#retry('Error getting block:', 1)
#defer.inlineCallbacks
def get_block(hash):
...
'''
def retry2(func):
#defer.inlineCallbacks
def f(*args, **kwargs):
for i in itertools.count():
try:
result = yield func(*args, **kwargs)
except Exception as e:
if i == max_retries:
raise
if not isinstance(e, RetrySilentlyException):
if traceback:
log.err(None, message)
else:
print >>sys.stderr, message, e
yield sleep(delay)
else:
defer.returnValue(result)
return f
return retry2
class ReplyMatcher(object):
'''
Converts request/got response interface to deferred interface
'''
def __init__(self, func, timeout=5):
self.func = func
self.timeout = timeout
self.map = {}
def __call__(self, id):
if id not in self.map:
self.func(id)
df = defer.Deferred()
def timeout():
self.map[id].remove((df, timer))
if not self.map[id]:
del self.map[id]
df.errback(failure.Failure(defer.TimeoutError('in ReplyMatcher')))
timer = reactor.callLater(self.timeout, timeout)
self.map.setdefault(id, set()).add((df, timer))
return df
def got_response(self, id, resp):
if id not in self.map:
return
for df, timer in self.map.pop(id):
df.callback(resp)
timer.cancel()
class GenericDeferrer(object):
'''
Converts query with identifier/got response interface to deferred interface
'''
def __init__(self, max_id, func, timeout=5, on_timeout=lambda: None):
self.max_id = max_id
self.func = func
self.timeout = timeout
self.on_timeout = on_timeout
self.map = {}
def __call__(self, *args, **kwargs):
while True:
id = random.randrange(self.max_id)
if id not in self.map:
break
def cancel(df):
df, timer = self.map.pop(id)
timer.cancel()
try:
df = defer.Deferred(cancel)
except TypeError:
df = defer.Deferred() # handle older versions of Twisted
def timeout():
self.map.pop(id)
df.errback(failure.Failure(defer.TimeoutError('in GenericDeferrer')))
self.on_timeout()
timer = reactor.callLater(self.timeout, timeout)
self.map[id] = df, timer
self.func(id, *args, **kwargs)
return df
def got_response(self, id, resp):
if id not in self.map:
return
df, timer = self.map.pop(id)
timer.cancel()
df.callback(resp)
def respond_all(self, resp):
while self.map:
id, (df, timer) = self.map.popitem()
timer.cancel()
df.errback(resp)
class NotNowError(Exception):
pass
class DeferredCacher(object):
'''
like memoize, but for functions that return Deferreds
#DeferredCacher
def f(x):
...
return df
#DeferredCacher.with_backing(bsddb.hashopen(...))
def f(x):
...
return df
'''
#classmethod
def with_backing(cls, backing):
return lambda func: cls(func, backing)
def __init__(self, func, backing=None):
if backing is None:
backing = {}
self.func = func
self.backing = backing
self.waiting = {}
#defer.inlineCallbacks
def __call__(self, key):
if key in self.waiting:
yield self.waiting[key]
if key in self.backing:
defer.returnValue(self.backing[key])
else:
self.waiting[key] = defer.Deferred()
try:
value = yield self.func(key)
finally:
self.waiting.pop(key).callback(None)
self.backing[key] = value
defer.returnValue(value)
_nothing = object()
def call_now(self, key, default=_nothing):
if key in self.backing:
return self.backing[key]
if key not in self.waiting:
self.waiting[key] = defer.Deferred()
def cb(value):
self.backing[key] = value
self.waiting.pop(key).callback(None)
def eb(fail):
self.waiting.pop(key).callback(None)
if fail.check(RetrySilentlyException):
return
print
print 'Error when requesting noncached value:'
fail.printTraceback()
print
self.func(key).addCallback(cb).addErrback(eb)
if default is not self._nothing:
return default
raise NotNowError(key)
def deferred_has_been_called(df):
still_running = True
res2 = []
def cb(res):
if still_running:
res2[:] = [res]
else:
return res
df.addBoth(cb)
still_running = False
if res2:
return True, res2[0]
return False, None
def inlineCallbacks(f):
from functools import wraps
#wraps(f)
def _(*args, **kwargs):
gen = f(*args, **kwargs)
stop_running = [False]
def cancelled(df_):
assert df_ is df
stop_running[0] = True
if currently_waiting_on:
currently_waiting_on[0].cancel()
df = defer.Deferred(cancelled)
currently_waiting_on = []
def it(cur):
while True:
try:
if isinstance(cur, failure.Failure):
res = cur.throwExceptionIntoGenerator(gen) # external code is run here
else:
res = gen.send(cur) # external code is run here
if stop_running[0]:
return
except StopIteration:
df.callback(None)
except defer._DefGen_Return as e:
# XXX should make sure direct child threw
df.callback(e.value)
except:
df.errback()
else:
if isinstance(res, defer.Deferred):
called, res2 = deferred_has_been_called(res)
if called:
cur = res2
continue
else:
currently_waiting_on[:] = [res]
def gotResult(res2):
assert currently_waiting_on[0] is res
currently_waiting_on[:] = []
if stop_running[0]:
return
it(res2)
res.addBoth(gotResult) # external code is run between this and gotResult
else:
cur = res
continue
break
it(None)
return df
return _
class RobustLoopingCall(object):
def __init__(self, func, *args, **kwargs):
self.func, self.args, self.kwargs = func, args, kwargs
self.running = False
def start(self, period):
assert not self.running
self.running = True
self._df = self._worker(period).addErrback(lambda fail: fail.trap(defer.CancelledError))
#inlineCallbacks
def _worker(self, period):
assert self.running
while self.running:
try:
self.func(*self.args, **self.kwargs)
except:
log.err()
yield sleep(period)
def stop(self):
assert self.running
self.running = False
self._df.cancel()
return self._df
How can I provide a list of methods in a class for use outside of it?
What I'm looking for
When I create an object from a class extending a higher class, I want to get a list of specific methods, some "meta data" of them, and to able to call them outside the class.
Example:
def params(params):
def params_fn(f):
def params_dec(*args, **kwargs):
for i in params:
f.__setattr__(i, params[i])
return f(*args, **kwargs)
return params_dec
return params_fn
class Channel:
queue = None
# some code & docs omitted
def __init__(self, queue):
self.queue = queue
def start(self):
while True:
if not self.check():
break
class channelA(Channel):
extra_methods = ['get_users', 'get_groups']
def start(self):
# omitted
pass
def __internal_method(self, d):
# omitted
pass
#params({"name": "Get users", "desc": "Get a list of users", "icon": "user-plus"})
def get_users(self):
# omitted
return [i for i in self.queue if i.type = 1]
#params({"name": "Get groups", "desc": "Get a list of groups", "icon": "group-plus"})
def get_groups(self):
# omitted
return [i for i in self.queue if i.type = 2]
q = []
ch = channelA(q)
# some code goes here
# it may be something like
# fns = ch.get_extra_fns()
# or anything similar
for fnName in fns:
print("Callable name:", fnName)
print("Name:", fns[fnName].name)
print("Description:", fns[fnName].desc)
print("Icon ID:", fns[fnName].icon)
print("Result:")
print(ch.call(fns[fnName])())
Is it possible to achieve something like this?
You don't need a wrapper for your decorated methods, since you only set attributes of the method-object. To identify your methods, I would suggest to set a special attribute for these methods, iterate over all class methods and pick those, that have this special method set:
def special(**kw):
def mark(function):
function.is_special = True
function.keywords = kw
return function
return mark
class Channel:
def __init__(self, queue):
self.queue = queue
def start(self):
while True:
if not self.check():
break
class ChannelA(Channel):
def start(self):
# omitted
pass
#special(icon="users.png")
def get_users(self):
# omitted
return [i for i in self.queue if i.type == 1]
#special(hint="don't feed the trolls")
def get_groups(self):
# omitted
return [i for i in self.queue if i.type == 2]
ch = ChannelA(q)
for methodname in dir(type(ch)):
method = getattr(ch, methodname)
if getattr(method, 'is_special', False):
print("Callable name:", method.__name__)
print("Keywords:", method.keywords)
print("Result:", method())
def get_extra_fns(self):
fns = {}
for i in self.extra_methods:
fns['i'] = self.__getattribute__(i)
return fns
Python 3:
import types
import inspect
class A:
def a(self):
pass
class B(A):
def b(self, bool_param, int_param):
pass
b = B()
for member_name in dir(B):
member = getattr(B, member_name)
if isinstance(member, types.FunctionType):
member(b, True) # as illustrated by you... this is hardcoded
argspec = inspect.getargspec(member)
args_for_call = {}
for argname in argspec.args:
if argname == 'bool_param':
args_for_call['bool_param'] = True
if argname == 'int_param':
args_for_call['int_param'] = 3
member(b, **args_for_call) # This is less hardcoded
I'm writing a very simple decorator to give me some basic debug information about a function.
from functools import wraps
from time import perf_counter
class debug(object):
def __init__(self, Time=False, Parameters=False, Doc=False):
self.t = Time
self.p = Parameters
self.d = Doc
def __call__(self, func):
#wraps(func)
def run(*args, **kwargs):
params = ""
if self.p:
params = ", ".join(["{}".format(arg) for arg in args] + ["{}={}".format(k, v) for k, v in kwargs.items()])
print("\n\tDebug output for '{}({})'".format(func.__name__, params))
if self.d:
print('\tDocstring: "{}"'.format(func.__doc__))
if self.t:
t1 = perf_counter()
val = func(*args, **kwargs)
if self.t:
t2 = perf_counter()
print("\tTime Taken: {:.3e} seconds".format(t2 - t1))
print("\tReturn Type: '{}'\n".format(type(val).__name__))
return val
return run
This is all well and good for normal functions.
#debug(Parameters=True, Time=True, Doc=True)
def foo(i, j=5):
"""Raises i to 2j"""
for _ in range(j):
i **= 2
return i
i = foo(5, j=3)
# Output:
"""
Debug output for 'foo(5, j=3)'
Docstring: "Raises i to 2j"
Time Taken: 1.067e-05 seconds
Return Type: 'int'
"""
However, generators are a different story.
#debug(Parameters=True, Time=True, Doc=True)
def bar(i, j=2):
"""Infinite iterator of increment j"""
while True:
yield i
i += j
b = bar() # Output occurs here
next(b) # No output
Now, from what I have coded, that is completely expected, but I'm wondering how I can hook the .__next__() method or what the best way of going about this is.
You can simply change your __call__ method and return a generator if a generator is given as an input (add import types at the top of your file):
def __call__(self, f):
if isinstance(f, types.GeneratorType):
def run_gen(*args, **kwargs):
# do pre stuff...
for _ in f(*argw, **kwargs):
yield _
# do post stuff...
return run_gen
else:
def run(*args, **kwargs):
# do pre stuff...
r = f(*argw, **kwargs)
# do post stuff...
return r
return run
You can't replace function.next as it is a read only value. But you can do something like this (see debug_generator function):
from functools import wraps
import inspect
class debug(object):
def __init__(self, Time=False, Parameters=False, Doc=False):
self.t = Time
self.p = Parameters
self.d = Doc
def __call__(self, func):
#wraps(func)
def debug_generator(func):
for i, x in enumerate(list(func)):
# here you add your debug statements
print "What you want: step %s" % i
yield x
#wraps(func)
def run(*args, **kwargs):
params = ""
if self.p:
params = ", ".join(["{}".format(arg) for arg in args] + ["{}={}".format(k, v) for k, v in kwargs.items()])
print("\n\tDebug output for '{}({})'".format(func.__name__, params))
if self.d:
print('\tDocstring: "{}"'.format(func.__doc__))
val = func(*args, **kwargs)
print("\tReturn Type: '{}'\n".format(type(val).__name__))
if inspect.isgenerator(val):
return debug_generator(val)
return val
return run
Basically you just get all the value from the generator you want to debug, and then you yield them again, adding debug statement in the loop.
I want to call some functions to a single value and return the collective result.
class Foo:
def __init__(self, i):
self.i = i
def get(self):
return self.fn1(self.fn2(self.i)) #200
def fn1(self, i):
return i + i #10+10 = 20
def fn2(self, i):
return i * i #20*20 = 200
#...
foo = Foo(10)
print(foo.get())
Is there a more elegant way or pattern?
Here is my try to improve this a little bit.
def fn1(i):
return i + i #10+10 = 20
def fn2(i):
return i * i #20*20 = 200
def get(i):
funcs = [fn2, fn1]
for f in funcs:
i = f(i)
return i
print(get(10))
In general, nesting functions as you do above is the most straightforward and readable way to compose functions in Python.
If you're composing many functions, it might be worth writing a compose function.
def compose(*funcs):
if len(funcs) == 1:
return funcs[0]
else:
def composition(*args, **kwargs):
return funcs[0](compose(*funcs[1:])(*args, **kwargs))
return composition
Or, if you prefer an iterative over a recursive solution:
def compose_pair(f1, f2):
def composition(*args, **kwargs):
return f1(f2(*args, **kwargs))
return composition
def compose_iterative(*funcs):
iterfuncs = iter(funcs)
comp = next(iterfuncs)
for f in iterfuncs:
comp = compose_pair(comp, f)
return comp
Personally, two of my favorite python functions are map and reduce.
def get(i):
return reduce(lambda acc, f: f(acc), [i,fn2,fn1] )
def fn1(i):
return i + i #10+10 = 20
def fn2(i):
return i * i #20*20 = 200
print( get(10) ) # 200
You could use a decorator-style solution:
class Base()
def __init__(self, decorated):
self.decorates = decorated
def foo(self, arg):
if self.decorates:
arg = self.decorates.foo( arg )
return self._do_foo( arg )
def _do_foo(self, arg):
return arg
Your implementations will inherit from Base and implement _do_foo().
You set it up like this:
a = Subclass(None)
b = AnotherSublcass( a )
c = YetAnotherSubclass( b )
all of the Sublcasses inherit from Base. when you call c.foo( arg ), you'll get the result passed through all three _do_foo() methods.