Develop Reference

Python, tracking using tqdm across parallel sub-tasks

In order to control the code I am working on, I have tried to create a single tracking across many tasks that occur in different threads.
I know at the beginning of the run the amount of tasks (and workers).
For demonstration (does not work, toy example):
from multiprocessing import Pool
from tqdm import tqdm
def work(i, t):
for _ in range(10**6):
t.update()
return i
def wrapped_work(params):
work(*params)
def main(n=1):
# another loop:
with Pool(processes=8) as p:
with tqdm(total=n * 10**6) as t:
return sum(p.map(work, ((i, t) for i in range(1, n+1))))
if __name__ == "__main__":
main(5)
I tried to implies this topic with pool, but without success.
I would greatly appreciate your help.

based on this post:
from multiprocessing import Pool, Process, Value
from ctypes import c_bool, c_long
from tqdm.auto import tqdm
class TqdmMultiprocessing:
max_processes = 64
def __init__(self, static_func, processes=64):
self.counter = Value(c_long, lock=False)
self.pool = Pool(
processes=min(processes, self.max_processes),
initializer=self.worker_init,
initargs=(static_func, self.counter)
)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.pool.close()
def tqdm(self, static_func, iterable, **kwargs):
done_value = Value(c_bool)
proc = Process(target=self.listener, args=(self.counter, done_value, kwargs,))
proc.start()
result = self.pool.map(static_func, iterable)
done_value.value = True
proc.join()
self.counter.value = 0
return result
#staticmethod
def listener(counter: Value, is_done: Value, kwargs):
with tqdm(**kwargs) as tqdm_bar:
old_counter = 0
while not is_done.value:
new_counter = counter.value
tqdm_bar.update(new_counter - old_counter)
old_counter = new_counter
tqdm_bar.update(tqdm_bar.total - old_counter)
#staticmethod
def worker_init(static_func, counter: Value):
static_func.counter = counter
def work(i):
for _ in range(10**6):
work.counter.value += 1
return i
def main(n=1):
with TqdmMultiprocessing(work, processes=3) as p:
p.tqdm(work, range(n), total=n * 10 ** 6)
p.tqdm(work, range(n), total=n * 10 ** 6)
if __name__ == "__main__":
main(5)

Define decorator parameter outside of module scope

I have the following decorator running fine using a parameter
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt):
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
cnt = 3
#sumproduct(cnt)
def myfunc(num):
print(num)
if __name__ == "__main__":
myfunc(10)
The output is 30 which is 0*10 + 1*10+ 2*10
However, I would like to import this module somewhere else, for example into a test module. I would like to do something like the following so that cnt is not defined in global scope:
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt):
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
#sumproduct(cnt)
def myfunc(num):
print(num)
if __name__ == "__main__":
cnt = 3
myfunc(10)
How can I define cnt so that
cnt is always 3 when code is executed?
and cnt is not imported when module is imported?
Note: This is just a sample representation of the code. Suppose that cnt is database connection which connects to production database. I would like to use a different database connection for tests, hence I don't want to import production database connection into test module.

You could use #sumproduct(lambda: cnt). That way the execution is delayed.
For example:
from functools import wraps
def sumproduct(cnt):
def dec(f):
#wraps(f)
def wrap(*args):
print('inside wrapper')
_sum = 0
for i in range(cnt()): # <---- Note the ()
_sum = _sum + sum([i * a for a in args])
f(_sum)
return wrap
return dec
#sumproduct(lambda: cnt) # <--- put lambda: here
def myfunc(num):
print(num)
if __name__ == "__main__":
cnt = 3
myfunc(10)
Prints:
inside wrapper
30

concurrent.futures: to run multiple task at the same time

Below, I'm try to use the Python futures module to run 2 functions at the same time. However,
I'm getting error future._condition.acquire() AttributeError: 'function' object has no attribute '_condition'
Did I used the futures module correctly?
from concurrent.futures import ThreadPoolExecutor, as_completed
def f1():
print ("1")
return ('a')
def f2():
return 'b'
executors_list = []
with ThreadPoolExecutor(max_workers=2) as executor:
executors_list.append(f1)
executors_list.append(f2)
for future in as_completed(executors_list):
r = (future.result())
print("r: ".format(r))

Need to use executor.submit(f1)
from concurrent.futures import ThreadPoolExecutor, as_completed
def f1():
return ('a')
def f2():
return 'b'
executors_list = []
with ThreadPoolExecutor(max_workers=2) as executor:
to_do = [executor.submit(f1),
executor.submit(f2)]
for future in as_completed(to_do):
print("future: {}, result {} ".format(future, future.result()))

How to use Python Concurrent Futures with decorators

I'm using a decorator for the thread pool executor:
from functools import wraps
from .bounded_pool_executor import BoundedThreadPoolExecutor
_DEFAULT_POOL = BoundedThreadPoolExecutor(max_workers=5)
def threadpool(f, executor=None):
#wraps(f)
def wrap(*args, **kwargs):
return (executor or _DEFAULT_POOL).submit(f, *args, **kwargs)
where the BoundedThreadPoolExecutor is defined here
When I try to use the concurrent futures in a function decorated with #threadpool and then waiting all the futures withas_completed like
def get_results_as_completed(futures):
# finished, pending = wait(futures, return_when=ALL_COMPLETED)
futures_results = as_completed(futures)
for f in futures_results:
try:
yield f.result()
except:
pass
for some worker defined like
from thread_support import threadpool
from time import sleep
from random import randint
#threadpool
def my_worker:
res = {}
# do something
sleep(randint(1, 5))
return res
if __name__ == "__main__":
futures_results = get_results_as_completed(futures)
for r in futures_results:
results.append(r)
I cannot get the futures completed despite of the .result() call, thus resulting in a infinite loop on futures_results. Why?

python AsyncResult.successful() does not return

I am trying my hands on python multiprocessing. I want a couple of processes which are independent to each other to run in parallel and as they return check if the process was successful or not using ApplyAsync.successful() utility. However when I call successful in the callback to my subprocess the script hangs.
import multiprocessing as mp
import time
result_map = {}
def foo_pool(x):
time.sleep(2)
print x
return x
result_list = []
def log_result(result):
print result_map[result].successful() #hangs
result_list.append(result)
def apply_async_with_callback():
pool = mp.Pool()
for i in range(10):
result_map[i] = pool.apply_async(foo_pool, args = (i, ), callback = log_result)
pool.close()
pool.join()
print(result_list)
if __name__ == '__main__':
apply_async_with_callback()

You don't need to check successful() because the callback is only called when the result was successful.
Following is the relevant code (multiprocessing/pool.py - AsyncResult)
def _set(self, i, obj):
self._success, self._value = obj
if self._callback and self._success: # <-----
self._callback(self._value) # <-----
self._cond.acquire()
try:
self._ready = True
self._cond.notify()
finally:
self._cond.release()
del self._cache[self._job]