Develop Reference

python multiprocessing.Process.terminate - How to kill child processes

This code:
import multiprocessing as mp
from threading import Thread
import subprocess
import time
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
class ControlThread(Thread):
def run():
jobs = []
for _ in range(2):
job = WorkerProcess()
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(2)
for job in jobs:
job.terminate()
if __name__ == "__main__":
controller = ControlThread()
controller.start()
When I terminate the spawned WorkerProcess instances. They die just fine, however the subprocesses python -c 'import time; time.sleep(1000) runs until completition. This is well documented in the official docs, but how do I kill the child processes of a killed process?
A possbile soultion might be:
Wrap WorkerProcess.run() method inside try/except block catching SIGTERM, and terminating the subprocess.call call. But I am not sure how to catch the SIGTERM in the WorkerProcess
I also tried setting signal.signal(signal.SIGINT, handler) in the WorkerProcess, but I am getting ValuError, because it is allowed to be set only in the main thread.
What do I do now?

EDIT: As #svalorzen pointed out in comments this doesn't really work since the reference to self.subprocess is lost.
Finally came to a clean, acceptable solution. Since mp.Process.terminate is a method, we can override it.
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
# HERE
def terminate(self):
self.subprocess.terminate()
super(WorkerProcess, self).terminate()

You can use queues to message to your subprocesses and ask them nicely to terminate their children before exiting themselves. You can't use signals in anywhere else but your main thread, so signals are not suitable for this.
Curiously, when I modify the code like this, even if I interrupt it with control+C, subprocesses will die as well. This may be OS related thing, though.
import multiprocessing as mp
from threading import Thread
import subprocess
import time
from Queue import Empty
class WorkerProcess(mp.Process):
def __init__(self,que):
super(WorkerProcess,self).__init__()
self.queue = que
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
while True:
a = self.subprocess.poll()
if a is None:
time.sleep(1)
try:
if self.queue.get(0) == "exit":
print "kill"
self.subprocess.kill()
self.subprocess.wait()
break
else:
pass
except Empty:
pass
print "run"
else:
print "exiting"
class ControlThread(Thread):
def run(self):
jobs = []
queues = []
for _ in range(2):
q = mp.Queue()
job = WorkerProcess(q)
queues.append(q)
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(5)
for q in queues:
q.put("exit")
time.sleep(30)
if __name__ == "__main__":
controller = ControlThread()
controller.start()
Hope this helps.
Hannu

PyQt Signals across threads

I've been messing around with PyQt and signals/slots across threads.
Here a situation where I can't find my mistake:
I have a class (MultipleProcessLauncher) that is able to launch multiple processes in separate threads.
I catch the stdout of each processes and send those messages to a single queue that is read by another thread (OutputWorker), this last thread should send a signal onNewMessage (I think it doesn't) catch on the main class but the callback function is never called.
The process threads populate the queue with messages
The reading thread catch all those messages (I can print them with print(item) in the while loop)
But:
- The signal of the reading thread doesn't seems to emit anything, so the callback function of the main thread is never called...
Your help would be much appreciated, I think I'm missing something with cross threads signals...
class OutputWorker(QObject):
onNewMessage = pyqtSignal(['QString'])
def __init__(self, queue, parent=None):
super(OutputWorker, self).__init__(parent)
self.queue = queue
def work(self):
while True:
item = self.queue.get()
self.onNewMessage.emit(item)
self.queue.task_done()
class MultipleProcessLauncher(QObject):
commandEvent = pyqtSignal(['QString'])
def __init__(self, parent=None):
super(MultipleProcessLauncher, self).__init__(parent)
self.messaging_queue = Queue()
# Start reading message
self.reading_thread = QThread()
self.worker = OutputWorker(self.messaging_queue)
self.worker.moveToThread(self.reading_thread)
self.worker.onNewMessage.connect(self.command_event)
self.reading_thread.started.connect(self.worker.work)
self.reading_thread.start()
def execute(self, command):
p = subprocess.Popen(command, stdout=subprocess.PIPE)
t = Thread(target=self.enqueue, args=(p.stdout, self.messaging_queue))
t.daemon = True
t.start()
def enqueue(self, stdout, queue):
for line in iter(stdout.readline, b''):
queue.put(line.decode())
stdout.close()
def command_event(self, event):
# This point is never reached
print('message received')
if __name__ == '__main__':
manager = MultipleProcessLauncher()
manager.execute('ipconfig')
time.sleep(100)

Qt's cross-thread signaling is based on event loop, so you need to exec a QApplication so that there is a main event loop to process signals from other threads. For example:
if __name__ == '__main__':
app = QApplication([])
manager = MultipleProcessLauncher()
manager.execute('ipconfig')
MAX_WAIT_MSEC = 100 * 1000 # 100 seconds
QTimer.singleShot(MAX_WAIT_MSEC, app.quit)
app.exec()
In your real application you will probably execute the manager based on user input so the execute would be in a slot, and there wouldn't be a need to quit, etc, but you get the idea.

How does a python process know when to exit?

When you execute a python script, does the process/interpreter exit because it reads an EOF character from the script? [i.e. is that the exit signal?]
The follow up to this is how/when a python child process knows to exit, namely, when you start a child process by overriding the run() method, as here:
class Example(multiprocessing.Process):
def __init__(self, task_queue, result_queue):
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
self.result_queue = result_queue
def run(self):
while True:
next_task = self.task_queue.get()
if next_task is None:
print '%s: Exiting' % proc_name
break
#more stuff...[assume there's some task_done stuff, etc]
if __name__ == '__main__':
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
processes = [ Example(tasks, results)
for i in range(5) ]
for i in processes:
i.start()
#more stuff...like populating the queue, etc.
Now, what I'm curious about is: Do the child processes automatically exit upon completion of the run() method? And if I kill the main thread during execution, will the child processes end immediately? Will they end if their run() calls can complete independently of the status of the parent process?

Yes, each child process terminates automatically after completion of the run method, even though I think you should avoid subclassing Process and use the target argument instead.
Note that in linux the child process may remain in zombie state if you do not read the exit status:
>>> from multiprocessing import Process
>>> def target():
... print("Something")
...
>>> Process(target=target).start()
>>> Something
>>>
If we look at the processes after this:
While if we read the exit status of the process (with Process.exitcode), this does not happen.
Each Process instance launches a new process in the background, how and when this subprocess is terminated is OS-dependant. Every OS provides some mean of communication between processes. Child processes are usually not terminated if you kill the "parent" process.
For example doing this:
>>> from multiprocessing import Process
>>> import time
>>> def target():
... while True:
... time.sleep(0.5)
...
>>> L = [Process(target=target) for i in range(10)]
>>> for p in L: p.start()
...
The main python process will have 10 children:
Now if we kill that process we obtain this:
Note how the child processes where inherited by init and are still running.
But, as I said, this is OS specific. On some OSes killing the parent process will kill all child processes.

Python close children when closing main process

I have have a main process that forks a number of subprocesses. I want to be able to kill these child processes off when my main process gets the kill signal. Ideally I would want to do something along the lines of:
def handler(signum, frame, pid_list):
log('Killing Process')
for pid in pid_list:
os.kill(pid, signal.SIGTERM)
os.waitpid(pid, 0) # need
sys.exit()
if __name__ == "__main__":
<code that creates child processes, pids>
signal.signal(signal.SIGTERM, handler(pid_list))
But of course, that doesn't work... any suggestions?

As #tony suggested you could set daemon=True flag on a child process created using multiprocessing module. To install it on python2.4, type: pip install multiprocessing.
The child processes won't be terminated if the main process is killed by a signal so you need to provide an appropriate signal handler:
#!/usr/bin/env python
import logging, signal, sys, time
import multiprocessing as mp # `pip install multiprocessing` on Python <2.6
class AddProcessNameFilter(logging.Filter):
"""Add missing on Python 2.4 `record.processName` attribute."""
def filter(self, r):
r.processName = getattr(r, 'processName', mp.current_process().name)
return logging.Filter.filter(self, r)
def print_dot():
while True:
mp.get_logger().info(".")
time.sleep(1)
def main():
logger = mp.log_to_stderr()
logger.setLevel(logging.INFO)
logger.addFilter(AddProcessNameFilter()) # fix logging records
# catch TERM signal to allow finalizers to run and reap daemonic children
signal.signal(signal.SIGTERM, lambda *args: sys.exit(-signal.SIGTERM))
# create daemonic child processes
processes = [mp.Process(target=print_dot) for _ in range(2)]
for p in processes:
p.daemon = True
p.start()
print_dot()
if __name__=="__main__":
mp.freeze_support()
main()

What about use this flag when you create a subprocess?

Python: nonblocking read from stdout of threaded subprocess

I have a script (worker.py) that prints unbuffered output in the form...
1
2
3
.
.
.
n
where n is some constant number of iterations a loop in this script will make. In another script (service_controller.py) I start a number of threads, each of which starts a subprocess using subprocess.Popen(stdout=subprocess.PIPE, ...); Now, in my main thread (service_controller.py) I want to read the output of each thread's worker.py subprocess and use it to calculate an estimate for the time remaining till completion.
I have all of the logic working that reads the stdout from worker.py and determines the last printed number. The problem is that I can not figure out how to do this in a non-blocking way. If I read a constant bufsize then each read will end up waiting for the same data from each of the workers. I have tried numerous ways including using fcntl, select + os.read, etc. What is my best option here? I can post my source if needed, but I figured the explanation describes the problem well enough.
Thanks for any help here.
EDIT
Adding sample code
I have a worker that starts a subprocess.
class WorkerThread(threading.Thread):
def __init__(self):
self.completed = 0
self.process = None
self.lock = threading.RLock()
threading.Thread.__init__(self)
def run(self):
cmd = ["/path/to/script", "arg1", "arg2"]
self.process = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=1, shell=False)
#flags = fcntl.fcntl(self.process.stdout, fcntl.F_GETFL)
#fcntl.fcntl(self.process.stdout.fileno(), fcntl.F_SETFL, flags | os.O_NONBLOCK)
def get_completed(self):
self.lock.acquire();
fd = select.select([self.process.stdout.fileno()], [], [], 5)[0]
if fd:
self.data += os.read(fd, 1)
try:
self.completed = int(self.data.split("\n")[-2])
except IndexError:
pass
self.lock.release()
return self.completed
I then have a ThreadManager.
class ThreadManager():
def __init__(self):
self.pool = []
self.running = []
self.lock = threading.Lock()
def clean_pool(self, pool):
for worker in [x for x in pool is not x.isAlive()]:
worker.join()
pool.remove(worker)
del worker
return pool
def run(self, concurrent=5):
while len(self.running) + len(self.pool) > 0:
self.clean_pool(self.running)
n = min(max(concurrent - len(self.running), 0), len(self.pool))
if n > 0:
for worker in self.pool[0:n]:
worker.start()
self.running.extend(self.pool[0:n])
del self.pool[0:n]
time.sleep(.01)
for worker in self.running + self.pool:
worker.join()
and some code to run it.
threadManager = ThreadManager()
for i in xrange(0, 5):
threadManager.pool.append(WorkerThread())
threadManager.run()
I have stripped out a log of the other code in hopes to try to pinpoint the issue.

Instead of having your service_controller being blocked by i/o access, only the thread loop should read its own controlled process output.
then, you can have method in the threaded object controlling the process to get the last polled output.
of course, don't forget in that case to use some locking mechanism to protect the buffer that will be used both by the thread to fill it and the method called by the controller to get it.

Your method WorkerThread.run() launches the subprocess and then terminates immediately. Run() needs to perform the polling and update WorkerThread.completed until the subprocess completes.