I would like to exit the program gracefully on Ctrl+C / SIGINT or on user input. If possible the terminal should prompt something like; "Hit enter to terminate".
Code to be executed by Python 3.6
def worker(process):
i = 0
while True:
print('Process %d count %d' % (process, i))
i += 1
def main():
available_num_cores = multiprocessing.cpu_count()
use_num_cores = available_num_cores - 1 if available_num_cores > 1 else 1
print('Using %d cores' % use_num_cores)
pool = multiprocessing.Pool(use_num_cores)
for i in range(0, use_num_cores):
pool.apply_async(worker, args=(i,))
pool.close()
pool.join()
if __name__ == '__main__':
main()
Accepted answer for this question Catch Ctrl+C / SIGINT and exit multiprocesses gracefully in python. Isn't working, it fail with error:
Process SpawnPoolWorker-1:
Process 0 count 1572
Process SpawnPoolWorker-2:
Process 1 count 1472
Process SpawnPoolWorker-3:
Traceback (most recent call last):
Any help would be appreciated. Thanks!
You need to make sure the SIGINT is ignored by the children processes.
Then you just either wait for user input or for a CTRL+C to be issued.
def initializer():
"""Ignore SIGINT in child workers."""
signal.signal(signal.SIGINT, signal.SIG_IGN)
def main():
try:
pool = multiprocessing.Pool(use_num_cores, initializer=initializer)
for i in range(0, use_num_cores):
pool.apply_async(worker, args=(i,))
pool.close()
input("Hit enter to terminate")
except KeyboardInterrupt:
print("CTRL+C")
finally:
pool.terminate()
pool.join()
print("Bye have a great time!")
Related
I know there's something stupid I'm doing, but I can't figure this out! I'm testing by doing 1 thread. It's supposed to add up to ten items in the queue, wait for it to finish, and then say "Finished". The thread function has q.task_done() at the end of it. But it'll say "Task Done" but then it just pauses and doesn't ever say "Finished".
def helper(q):
print("Started")
....
q.task_done()
print("Task done")
def main():
q = Queue(maxsize=0)
recid = 9000000
num_threads = 1
for i in range(num_threads):
worker = Thread(target=helper, args=(q))
worker.setDaemon(True)
worker.start()
while recid > 0:
batch.clear()
for x in range(10):
q.put(recid)
print("Added request.")
recid -= 1
print("Waiting for threads to finish.")
q.join()
print("Finished")
worker was dead before you called q.join(). maybe try to add some delays in helper() waiting for q.put()
def helper(q):
print("Started")
time.sleep(1) # wait for main() q.put()
while not q.empty(): # until queue is empty
print("request", q.get())
q.task_done() # trigger q.join in main()
print("Task done")
time.sleep(1) # wait for next q.put() if exists
Updated:
Basically, q.task_done() in helper() triggers q.join() in main() and worker.join() will be triggered once helper() is exited. An example code is as followed:
def main():
q = Queue(maxsize=0)
recid = 100
num_threads = 4
while recid > 0:
for i in range(num_threads):
print('''******** Hired Worker %d ********'''%(i),)
worker = Thread(target=helper, args=(q,))
worker.setDaemon(True)
worker.start()
# batch.clear()
for x in range(3):
q.put(recid)
print("Request Worker %d to do %d."%(i, recid))
recid -= 1
##############################################################
# Comment these lines when you are ready to test multiThread
##############################################################
print("Waiting Worker %d to finish %d."%(i, recid))
q.join() # triggered by q.task_done in helper()
print("Worker %d Finished %d"%(i, recid))
print("Waiting for Worker %d to leave."%i)
worker.join() # triggered once helper() is exited
print("!!!!!!!! Worker %d left !!!!!!!!"%i)
##############################################################
time.sleep(5) # wait until all requests are finished.
print("Finished")
Once you are happy with join() of Thread and Queue. You may try multithreading by commenting that block of code.
In the code below, I have raised an exception during the first call, and yet it seems the exception is absorbed, and I still got all other processes executed, what's the problem? What I want is that whenever the first exception occurs, print it, and stop the multiprocessing pool directly.
def func(i):
if i==0:
raise Exception()
else:
time.sleep(1)
print(i)
num_workers = 4
pool = multiprocessing.Pool(num_workers)
try:
for i in range(4):
pool.apply_async(func,args=(i,))
except:
print("err")
pool.close()
pool.join()
The following edited code according to HTF
import multiprocessing
import time
if __name__ == '__main__':
def func(i):
if i == 0:
raise Exception()
else:
time.sleep(1)
print(i)
num_workers = 4
pool = multiprocessing.Pool(num_workers)
results = [pool.apply_async(func, args=(i,)) for i in range(4)]
try:
for result in results:
result.get()
except:
print("err")
pool.close()
pool.join()
gives output
err
1
2
3
where I expect only err
You just scheduled the tasks but you need to wait for the results:
results = [pool.apply_async(func,args=(i,)) for i in range(4)]
try:
for result in results:
result.get()
except:
print("err")
Update Wed 7 Apr 20:42:59 UTC 2021:
You can try something like this:
import time
from functools import partial
from multiprocessing import Pool
def func(i):
if i == 0:
raise Exception("something bad happened")
else:
time.sleep(1)
print(i)
def quit(pool, err):
print(f"ERROR: {err}")
pool.terminate()
def main():
pool = Pool()
partial_quit = partial(quit, pool)
for i in range(4):
pool.apply_async(func, args=(i,), error_callback=partial_quit)
pool.close()
pool.join()
if __name__ == "__main__":
main()
Test:
$ python test1.py
ERROR: something bad happened
If you need the return value back it may be actually easier to use bare processes and a queue:
import time
from multiprocessing import Process, Queue
PROCS = 4
def worker(q, i):
if i == 10:
print("error")
q.put_nowait("ERROR")
else:
time.sleep(1)
print(i)
q.put_nowait(i)
def main():
q = Queue()
procs = []
for i in range(PROCS):
p = Process(target=worker, args=(q, i))
p.start()
procs.append(p)
count = len(procs)
while count:
result = q.get()
if result == "ERROR":
for p in procs:
p.terminate()
break
print(f"Result for: {result}")
count -= 1
if __name__ == "__main__":
main()
Test:
$ python test2.py
0
2
1
3
Result for: 0
Result for: 2
Result for: 1
Result for: 3
I want to stop asynchronous multiprocessing jobs with KeyboardInterrupt. But sometimes hang occurred when call terminate.
from multiprocessing.pool import ThreadPool
import multiprocessing
import time
import queue
import inspect
def worker(index):
print('{}: start'.format(index))
for i in range(5):
time.sleep(1)
print('{}: stop'.format(index))
return index, True
def wrapper(index, stopEvent, qResult):
if stopEvent.is_set() is True:
return index, False
try:
result = worker(index)
except:
print('*' * 50)
return index, False
else:
if result[1] == True:
qResult.put(result)
return result
def watcher(qResult, stopEvent):
cntQ = 0
while True:
try:
result = qResult.get(timeout=10)
qResult.task_done()
except queue.Empty:
if stopEvent.is_set() is True:
break
except KeyboardInterrupt:
stopEvent.set()
else:
cntQ += 1
print(result)
qResult.join()
qResult.close()
print('qResult count:', cntQ)
def main():
stopEvent = multiprocessing.Event()
qResult = multiprocessing.JoinableQueue()
qResult.cancel_join_thread()
watch = multiprocessing.Process(target=watcher, args=(qResult, stopEvent))
watch.start()
pool = ThreadPool()
lsRet = []
for i in range(100000):
try:
ret = pool.apply_async(wrapper, args=(i, stopEvent, qResult))
lsRet.append(ret)
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
break
if i+1 % 10 == 0:
time.sleep(2)
cntTotal = len(lsRet)
cntRet = 0
for ret in lsRet:
if stopEvent.is_set():
break
try:
ret.get()
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
else:
cntRet += 1
if stopEvent.is_set() is False:
stopEvent.set()
print(inspect.stack()[0][1:4])
if watch.is_alive() is True:
watch.join()
print(inspect.stack()[0][1:4])
pool.terminate() # Why hang??????????
print(inspect.stack()[0][1:4])
pool.join()
print(cntTotal, cntRet)
if __name__ == '__main__':
main()
main() invokes a watcher() thread and many wrapper() threads asynchronously using multiprocessing.pool.Threadpool.
wrapper() calls worker() and put its result to queue.
watcher() watches above queue of results.
If ctrl-c pressed, stopEvent is set.
When stopEvent is set, wrapper() stops calling worker(), and Watcher() indicates queue.Empty and stopEvent and exits loop.
Finally main() calls terminate() of pool.
Sometimes processes done well, but sometimes hang. It's different each time.
You should put the code in try except block and catch a built-in exception KeyboardInterrupt see the example here Capture keyboardinterrupt
Consider the following example I've been doing to learn multithreading. It's just an extended example of the Python 3.5 queue documentation.
It prints some numbers over 4 threads, produces one error in the queue, retries this element and should print the remaining queue if a KeyboardInterrupt exception occurs.
import threading
import queue
import time
import random
import traceback
def worker(q, active):
while True:
worker_item = q.get()
#if worker_item == None:
if not active.is_set():
break
time.sleep(random.random())
with threading.Lock():
if worker_item == 5 or worker_item == '5':
try:
print(threading.current_thread().name + ': ' + worker_item + ' | remaining queue: ' + str(list(q.queue)))
except TypeError:
print(threading.current_thread().name + ': ')
print(traceback.format_exc())
q.put(str(worker_item))
else:
print(threading.current_thread().name + ': ' + str(worker_item) + ' | remaining queue: ' + str(list(q.queue)))
q.task_done()
def main():
# INITIALIZE
num_threads = 4
stack1 = list(range(1, 21))
stack2 = list(range(101, 121))
q = queue.Queue()
active = threading.Event()
active.set()
# START THREADS
threads = []
for _ in range(num_threads):
t = threading.Thread(target=worker, args=(q, active))
t.start()
threads.append(t)
try:
# PUT STACK ITEMS ON QUEUE AND BLOCK UNTIL ALL TASKS ARE DONE
for stack1_item in stack1:
q.put(stack1_item)
q.join()
for stack2_item in stack2:
q.put(stack2_item)
q.join()
# STOP WORKER LOOP IN EVERY THREAD
#for _ in threads:
#q.put(None)
active.clear()
# WAIT UNTIL ALL THREADS TERMINATE
for t in threads:
t.join()
except KeyboardInterrupt:
print(traceback.format_exc())
print('remaining queue: ' + str(list(q.queue)))
#for _ in threads:
#q.put(None)
active.clear()
for t in threads:
t.join()
if __name__ == '__main__':
main()
If I run the script as it is (without a KeyboardInterrupt), it won't terminate. I have to kill the signal. But if I comment/uncomment the following lines (not using the event and doing it the docs way...)
comment / worker / if not active.is_set():
uncomment / worker / #if worker_item == None:
comment / main / active.clear()
uncomment / main / #for _ in threads:
#q.put(None)
comment / main / except / active.clear()
uncomment / main / except / #for _ in threads:
#q.put(None)
it does exit with exit code 0. Why?
Why is putting Nones to the queue necessary?
What would be the solution without putting Nones to the queue?
There are two types of threads: daemon and non-daemon. By default, all threads are non-daemon. The process is kept alive as long as there is at least one non-daemon thread.
This means that to stop the process, you either have to:
stop all of its threads (this is what your commented out code does by using None to kick the worker out of the infinite wait in q.get()); or
make the workers daemon threads, in which case the process will stop as soon as the main thread stops (this will require extra care if you want to ensure the workers have finished their tasks).
I have a python script that uses threading, here is my code:
def main():
ip_list = get_ip_list() #['ip1', 'ip2'..., 'ipn'] thousands
for x in ip_list:
q.put(x)
threads = []
threads_num = 50
for x in range(threads_num):
w = Worker(q, stantard_config_to_dict, id_dict, logger,ip_position)
threads.append(w)
w.start()
for w in threads:
w.join()
logger.debug('End of main thread, exit')
print 'End of main thread, exit'
def run(self):
while 1:
try:
ip = self.queue.get(False)
self.logger.debug('%s get %s from queue, left %s in queue' % (self.getName(), ip, self.queue.qsize()))
self.get_inter_conf(ip)
except Queue.Empty:
self.logger.debug('queue is empty, exit')
break
self.logger.info("%s finished" % (self.getName()))
The string 'End of main thread' doesn't get printed right now, only when I change the number of items in ip_list it gets printed.
Why does this happen?