I have implemented a parser like this,
import multiprocessing
import time
def foo(i):
try:
# some codes
except Exception, e:
print e
def worker(i):
foo(i)
time.sleep(i)
return i
if __name__ == "__main__":
pool = multiprocessing.Pool(processes=4)
result = pool.map_async(worker, range(15))
while not result.ready():
print("num left: {}".format(result._number_left))
time.sleep(1)
real_result = result.get()
pool.close()
pool.join()
My parser actually finishes all the processes but the results are not available ie, it's still inside the while loop and printing num left : 2. How I stop this? And I don't want the value of real_result variable.
I'm running Ubuntu 14.04, python 2.7
Corresponding part of my code looks like,
async_args = ((date, kw_dict) for date in dates)
pool = Pool(processes=4)
no_rec = []
def check_for_exit(msg):
print msg
if last_date in msg:
print 'Terminating the pool'
pool.terminate()
try:
result = pool.map_async(parse_date_range, async_args)
while not result.ready():
print("num left: {}".format(result._number_left))
sleep(1)
real_result = result.get(5)
passed_dates = []
for x, y in real_result:
passed_dates.append(x)
if y:
no_rec.append(y[0])
# if last_date in passed_dates:
# print 'Terminating the pool'
# pool.terminate()
pool.close()
except:
print 'Pool error'
pool.terminate()
print traceback.format_exc()
finally:
pool.join()
My bet is that you have faulty parse_date_range,
which causes a worker process to terminate without producing any result or py exception.
Probably libc's exit is called by a C module/lib due to a realy nasty error.
This code reproduces the infinite loop you observe:
import sys
import multiprocessing
import time
def parse_date_range(i):
if i == 5:
sys.exit(1) # or raise SystemExit;
# other exceptions are handled by the pool
time.sleep(i/19.)
return i
if __name__ == "__main__":
pool = multiprocessing.Pool(4)
result = pool.map_async(parse_date_range, range(15))
while not result.ready():
print("num left: {}".format(result._number_left))
time.sleep(1)
real_result = result.get()
pool.close()
pool.join()
Hope this'll help.
Related
I'm trying to run the same pool three times so that it adds the values in the same list:
import multiprocessing
def foo(name,archive):
print('hello ', name)
archive.append(f"hello {name}")
def main():
max_process = multiprocessing.cpu_count()-1 or 1
pool = multiprocessing.Pool(max_process)
manager = multiprocessing.Manager()
archive = manager.list()
arguments = ['home','away','draw']
for _ in range(3):
[pool.apply_async(foo, args=[name,archive]) for name in arguments]
pool.close()
pool.join()
print(archive)
if __name__ == '__main__':
main()
The first batch runs perfectly, but when the second batch goes, this error appears:
ValueError: Pool not running
How should I proceed to generate this looping?
As indicated by Nullman in comments, the error was in keeping pool.close() and pool.join() inside the loop, bringing them out, worked perfectly:
import multiprocessing
def foo(name,archive):
print('hello ', name)
archive.append(f"hello {name}")
def main():
max_process = multiprocessing.cpu_count()-1 or 1
pool = multiprocessing.Pool(max_process)
manager = multiprocessing.Manager()
archive = manager.list()
arguments = ['home','away','draw']
for _ in range(3):
[pool.apply_async(foo, args=[name,archive]) for name in arguments]
pool.close()
pool.join()
print(archive)
if __name__ == '__main__':
main()
I have n threads running simultaneously. These threads are processing a list containing m test cases. For example, thread n-1 is working on item m[i-1] while thread n is working on item m[i]. I want to stop all threads if for example thread n-1 failed or return a signal. How can I achieve this?
Here is a MWE:
This is my processing function
def process(input_addr):
i =+ 1
print('Total number of executed unit tests: {}'.format(i))
print("executed {}. thread".format(input_addr))
try:
command = 'python3 '+input_addr
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
msg, err = result.communicate()
if msg.decode('utf-8') != '':
stat = parse_shell(msg.decode('utf-8'))
if stat:
print('Test Failed')
return True
else:
stat = parse_shell(err)
if stat:
print('Test Failed')
return True
except Exception as e:
print("thread.\nMessage:{1}".format(e))
Here is my pool:
def pre_run_test_files(self):
with Pool(10) as p:
p.map(process, self.test_files)
I am using:
from multiprocessing import Pool
You can have your worker function, process simply raise an exception and use an error_callback function with apply_async that calls terminate on the pool as in the following demo:
from multiprocessing import Pool
def process(i):
import time
time.sleep(1)
if i == 6:
raise ValueError(f'Bad value: {i}')
print(i, flush=True)
def my_error_callback(e):
pool.terminate()
print(e)
if __name__ == '__main__':
pool = Pool(4)
for i in range(20):
pool.apply_async(process, args=(i,), error_callback=my_error_callback)
# wait for all tasks to complete
pool.close()
pool.join()
Prints:
0
1
3
2
4
5
7
Bad value: 6
You should be able to adapt the above code to your particular problem.
Update
Because your original code used the map method, there is a second solution that use methid imap_unordered, which will returns an iterator that on every iteration returns the next return value from your worker function, process, or raises an exception if your worker function raised an exception. With method imap_unordere these results are returned in an arbitrary completion order rather than in task submission order, but when the default chunksize argument of 1 is used, this arbitrary order is typically task-completion order. This is what you want so that you can detect an exception at the earliest possible time and terminate the pool. Of course, if you cared about the return values from process, then you would use method imap so that the results are returned in task-submission order. But in that case if when case i == 6 is when the exception is raised but that task happened to be the first task to complete, its exception could still not be returned until the tasks submitted for i == 1 though 5 were completed.
In the following code a pool size of 8 is used, and all tasks first sleep for 1 second before printing their arguments and returning except for the case of i == 6, which raises an exception immediately. Using imap_unordered we have:
from multiprocessing import Pool
def process(i):
import time
# raise an exception immediately for i == 6 without sleeping
if (i != 6):
time.sleep(1)
else:
raise ValueError(f'Bad value: {i}')
print(i, flush=True)
if __name__ == '__main__':
pool = Pool(8)
results = pool.imap_unordered(process, range(20))
try:
# Iterate results as task complete until
# we are done or one raises an exeption:
for result in results:
# we don't care about the return value:
pass
except Exception as e:
pool.terminate()
print(e)
pool.close()
pool.join()
Prints:
Bad value: 6
If we replace the call to imap_unordered with a call to imap, then the output is:
0
1
2
3
4
5
Bad value: 6
The first solution, using apply_async with a error_callback argument, allows for the exception to be acted upon as soon as it occurs and if you care about the results in task submission order, you can save the multiprocessing.AsyncResult objects returned by apply_async in a list and call get on these objects. Try the following code with RAISE_EXCEPTION set to True and then to False:
from multiprocessing import Pool
import time
RAISE_EXCEPTION = True
def process(i):
if RAISE_EXCEPTION and i == 6:
raise ValueError(f'Bad value: {i}')
time.sleep(1)
return i # instead of printing
def my_error_callback(e):
global got_exception
got_exception = True
pool.terminate()
print(e)
if __name__ == '__main__':
got_exception = False
pool = Pool(4)
async_results = [pool.apply_async(process, args=(i,), error_callback=my_error_callback) for i in range(20)]
# Wait for all tasks to complete:
pool.close()
pool.join()
if not got_exception:
for async_result in async_results:
print(async_result.get())
I found the solution:
def process(i, input_addr, event):
kill_flag = False
if not event.is_set():
print('Total number of executed unit tests: {}'.format(i))
print("executed {}. thread".format(input_addr))
try:
command = 'python3 '+input_addr
result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
msg, err = result.communicate()
if msg.decode('utf-8') != '':
stat = parse_shell(msg.decode('utf-8'))
if stat:
print('Test Failed')
kill_flag = True
# all_run.append(input_addr)
#write_list_to_txt(input_addr, valid_tests)
else:
kill_flag = False
else:
stat = parse_shell(err)
if stat:
print('Test Failed')
kill_flag = True
# all_run.append(input_addr)
#write_list_to_txt(input_addr, valid_tests)
else:
kill_flag = False
except Exception as e:
print("thread.\nMessage:{1}".format(e))
if kill_flag:
event.set()
def manager():
p= multiprocessing.Pool(10)
m = multiprocessing.Manager()
event = m.Event()
for i,f in enumerate(self.test_files):
p.apply_async(process, (i, f, event))
p.close()
event.wait()
p.terminate()
How to exit from a function called my multiprocessing.Pool
Here is an example of the code I am using, when I put a condition to exit from function worker when I use this as a script in terminal it halts and does not exit.
def worker(n):
if n == 4:
exit("wrong number") # tried to use sys.exit(1) did not work
return n*2
def caller(mylist, n=1):
n_cores = n if n > 1 else multiprocessing.cpu_count()
print(n_cores)
pool = multiprocessing.Pool(processes=n_cores)
result = pool.map(worker, mylist)
pool.close()
pool.join()
return result
l = [2, 3, 60, 4]
myresult = caller(l, 4)
As I said, I don't think you can exit the process running the main script from a worker process.
You haven't explained exactly why you want to do this, so this answer is a guess, but perhaps raising a custom Exception and handling it in an explict except as shown below would be an acceptable way to workaround the limitation.
import multiprocessing
import sys
class WorkerStopException(Exception):
pass
def worker(n):
if n == 4:
raise WorkerStopException()
return n*2
def caller(mylist, n=1):
n_cores = n if n > 1 else multiprocessing.cpu_count()
print(n_cores)
pool = multiprocessing.Pool(processes=n_cores)
try:
result = pool.map(worker, mylist)
except WorkerStopException:
sys.exit("wrong number")
pool.close()
pool.join()
return result
if __name__ == '__main__':
l = [2, 3, 60, 4]
myresult = caller(l, 4)
Output displayed when run:
4
wrong number
(The 4 is the number of CPUs my system has.)
The thing with pool.map is, that it will raise exceptions from child-processes only after all tasks are finished. But your comments sound like you need immediate abortion of all processing as soon as a wrong value is detected in any process. This would be a job for pool.apply_async then.
pool.apply_async offers error_callbacks, which you can use to let the pool terminate. Workers will be fed item-wise instead of chunk-wise like with the pool.map variants, so you get the chance for early exit on each processed argument.
I'm basically reusing my answer from here:
from time import sleep
from multiprocessing import Pool
def f(x):
sleep(x)
print(f"f({x})")
if x == 4:
raise ValueError(f'wrong number: {x}')
return x * 2
def on_error(e):
if type(e) is ValueError:
global terminated
terminated = True
pool.terminate()
print(f"oops: {type(e).__name__}('{e}')")
def main():
global pool
global terminated
terminated = False
pool = Pool(4)
results = [pool.apply_async(f, (x,), error_callback=on_error)
for x in range(10)]
pool.close()
pool.join()
if not terminated:
for r in results:
print(r.get())
if __name__ == '__main__':
main()
Output:
f(0)
f(1)
f(2)
f(3)
f(4)
oops: ValueError('wrong number: 4')
Process finished with exit code 0
I want to terminate my program when press Ctrl-C, code as follow:
#!/usr/bin/env python
# encoding: utf-8
import multiprocessing
import time
import signal
import sys
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def worker():
while(True):
time.sleep(1.1234)
print "Working..."
if __name__ == "__main__":
pool = multiprocessing.Pool(50, init_worker)
try:
for i in range(50):
pool.apply_async(worker)
# time.sleep(10)
pool.close()
pool.join()
except KeyboardInterrupt:
print "Caught KeyboardInterrupt, terminating workers"
pool.terminate()
pool.join()
but this can not work right
You can try this way:
import multiprocessing
import time
import signal
def init_worker():
signal.signal(signal.SIGINT, signal.SIG_IGN)
def worker():
while(True):
time.sleep(1.1234)
print "Working..."
if __name__ == "__main__":
pool = multiprocessing.Pool(10, init_worker)
result = []
for i in range(10):
result.append(pool.apply_async(worker))
try:
while True:
time.sleep(0.5)
if all([r.ready() for r in result]):
break
except KeyboardInterrupt:
pool.terminate()
pool.join()
else:
pool.close()
pool.join()
I have a python pool of processes , if an exception occurs in any one of the process i want to exit the execution of the pool
I have joined all the processes in the pool, so the join waits for every process to finish.
If i raise sys.exit(1) inside the target function the system goes on infinite wait because the join is still waiting for process to complete.
How can exit the execution while using join in the code
from multiprocessing import Pool
import time
import sys
def printer(ip):
try:
for _ in xrange(5):
print ip+str(_)
time.sleep(1.0)
except Exception as e:
print e
sys.exit(2)
def test():
pool = Pool(processes=2)
for i in ["hello",5]:
result = pool.apply_async(printer,(i,))
pool.close()
pool.join()
print "good bye"
test()
Just return to the parent process the status of the operation and use a callback to react to failures.
import time
from multiprocessing import Pool
def printer(ip):
try:
for _ in xrange(5):
print ip+str(_)
time.sleep(1.0)
return True
except Exception as e:
print e
return False
class Worker():
def __init__(self):
self.pool = Pool(processes=2)
def callback(self, result):
if not result:
print "Error raised in child process. Terminating.."
self.pool.terminate()
def do_job(self):
for i in ["hello", 5]:
self.pool.apply_async(printer, (i,), callback=self.callback)
self.pool.close()
self.pool.join()
print "good bye"
def test():
w = Worker()
w.do_job()