I came across a problem with multiprocessing:
class PythonHelper(object):
#staticmethod
def run_in_parallel(*functions):
processes=list()
for function in functions:
process=Process(target=function)
process.start()
processes.append(process)
for process in processes:
process.join()
The above stathic method is used by me to run several functions simultaneously (combining them in one process). Everything was okay until I came across the need to force the process terminating while one of 'subprocess' is terminated.
For example:
from PythonHelper import PythonHelper as ph
from Recorder import Recorder
class Logger(object):
def run_recorder_proc(self):
rec=Recorder()
rec.record_video()
def run_printer_proc(self):
#hypothetical function: execution takes a long time
for i in range(9000000):
print("number: {}".format(i))
def run_logger(self):
ph.run_in_parallel(self.run_printer_proc,self.run_recorder_proc)
self.run_printer_proc and self.run_recorder_proc are my subprocesses. How to 'kill' remaining subprocess while one of them was completed?
Edit:
Full source code:
class PythonHelper(object):
#staticmethod
#with your fix
def run_in_parallel(*functions):
processes={}
for function in functions:
process=Process(target=function)
process.start()
processes[process.pid]=process
# wait for any process to complete
pid, status = os.waitpid(-1, 0)
# one process terminated
# join it
processes[pid].join()
del processes[pid]
# terminate the rest
for process in processes.values():
process.terminate()
for process in processes.values():
process.join()
class Logger(object):
def run_numbers_1(self):
for i in range(900000):
print("number: {}".format(i))
def run_numbers_2(self):
for i in range(100000):
print("number: {}".format(i))
def run_logger(self):
ph.run_in_parallel(self.run_numbers_1,self.run_numbers_2)
if __name__=="__main__":
logger=Logger()
logger.run_logger()
Based on the above example I would like to force termination of run_numbers_1 while run_numbers_2 is completed.
You may achieve that by changing run_in_parallel() slightly:
def run_in_parallel(*functions):
processes={}
for function in functions:
process=Process(target=function)
process.start()
processes[process.pid]=process
# wait for any process to complete
pid, status = os.waitpid(-1, 0)
# one process terminated
# join it
processes[pid].join()
del processes[pid]
# terminate the rest
for process in processes.itervalues():
process.terminate()
for process in processes.itervalues():
process.join()
[Update]
Based on your complete code here is a working example. Instead of race-prone os.waitpid() it uses Event object, which other processes set when completed:
from multiprocessing import Process, Event
class MyProcess(Process):
def __init__(self, event, *args, **kwargs):
self.event = event
Process.__init__(self, *args, **kwargs)
def run(self):
Process.run(self)
self.event.set()
class PythonHelper(object):
#staticmethod
#with your fix
def run_in_parallel(*functions):
event = Event()
processes=[]
for function in functions:
process=MyProcess(event, target=function)
process.start()
processes.append(process)
# wait for any process to complete
event.wait()
# one process completed
# terminate all child processes
for process in processes:
process.terminate()
for process in processes:
process.join()
class Logger(object):
def run_numbers_1(self):
for i in range(90000):
print("1 number: {}".format(i))
def run_numbers_2(self):
for i in range(10000):
print("2 number: {}".format(i))
def run_logger(self):
PythonHelper.run_in_parallel(self.run_numbers_1,self.run_numbers_2)
if __name__=="__main__":
logger=Logger()
logger.run_logger()
Related
I want to manually join and close child processes from either multiprocessing.Pool or ProcessPoolExecutor. However whenever I try to join a child process created by either of those Pools the code hangs indefinitely.
from concurrent.futures import ProcessPoolExecutor, wait, FIRST_COMPLETED
from multiprocessing import current_process, active_children
class Example:
def start(self):
with ProcessPoolExecutor(max_workers=1) as executor:
futures = set()
futures.add(executor.submit(self.worker))
done, futures = wait(futures, return_when=FIRST_COMPLETED)
for task in done:
res = task.result()
print(f"Child pid is {res}")
for child in active_children():
if child.pid == res:
print("Child found")
child.join()
child.close()
print("Child closed")
def worker(self):
print("Worker called")
return current_process().pid
ex = Example()
ex.start()
from multiprocessing import current_process, active_children, Pool
class Example:
def start(self):
with Pool(processes=1) as pool:
task = pool.apply_async(self.worker)
res = task.get()
print(f"Child pid is {res}")
for child in active_children():
if child.pid == res:
print("Child found")
child.join()
child.close()
print("Child closed")
def worker(self):
print("Worker called")
return current_process().pid
ex = Example()
ex.start()
Output:
Worker called
Child pid is 284108
Child found
Why?
A multiprocessing pool is made up of one or more processes that takes a submitted "task" from an input queue and runs the task until it is complete and then goes back to get the next task from the queue. These processes run until you implicitly or explicitly terminate the entire pool via one of the methods describe below. But the important point is that when you submit a task to the pool (e.g. using concurrent.futures.ProcessPoolExecutor.submit or multiprocessing.pool.Pool.apply_async) the specified worker function that you pass to these calls is executed in an already running process and that process cannot be joined until it has terminated and will not terminate unless you take a specific action to terminate it.
But there is no reason for explicitly calling join on individual processes in a processing pool and it will cause you to block because these processes will not terminate until for a concurrent.futures.ProcessPoolExecutor pool you either first call shutdown(wait=True) and all submitted tasks have completed or you call shutdown(wait=False) or if you are dealing with a multiprocess.pool.Pool pool, you first call either terminate or the sequence pool.close() followed by pool.join() (which joins all the pool processes , which will terminate when all submitted tasks have completed). But at that point there are no longer any running pool processes. For example, if we call pool.terminate():
from multiprocessing import current_process, active_children, Pool
class Example:
def start(self):
pool = Pool(processes=1)
task = pool.apply_async(self.worker)
res = task.get()
pool.terminate() # Now there are no more running processes:
print(f"Child pid is {res}")
# This will not find any active children:
for child in active_children():
if child.pid == res:
print("Child found")
child.join()
child.close()
print("Child closed")
def worker(self):
print("Worker called")
return current_process().pid
# Required for Windows:
if __name__ == '__main__':
ex = Example()
ex.start()
Prints:
Worker called
Child pid is 18076
Or if we wait for all tasks to complete and the pool processes terminate of their own accord:
from multiprocessing import current_process, active_children, Pool
class Example:
def start(self):
pool = Pool(processes=1)
task = pool.apply_async(self.worker)
res = task.get()
# wait for all tasks to complete:
pool.close()
pool.join()
print(f"Child pid is {res}")
# This will not find any active children:
for child in active_children():
if child.pid == res:
print("Child found")
child.join()
child.close()
print("Child closed")
def worker(self):
print("Worker called")
return current_process().pid
# Required for Windows:
if __name__ == '__main__':
ex = Example()
ex.start()
Prints:
Worker called
Child pid is 19936
I'm trying to code a kind of task manager in Python. It's based on a job queue, the main thread is in charge of adding jobs to this queue. I have made this class to handle the jobs queued, able to limit the number of concurrent processes and handle the output of the finished processes.
Here comes the problem, the _check_jobs function I don't get updated the returncode value of each process, independently of its status (running, finished...) job.returncode is always None, therefore I can't run if statement and remove jobs from the processing job list.
I know it can be done with process.communicate() or process.wait() but I don't want to block the thread that launches the processes. Is there any other way to do it, maybe using a ProcessPoolExecutor? The queue can be hit by processes at any time and I need to be able to handle them.
Thank you all for your time and support :)
from queue import Queue
import subprocess
from threading import Thread
from time import sleep
class JobQueueManager(Queue):
def __init__(self, maxsize: int):
super().__init__(maxsize)
self.processing_jobs = []
self.process = None
self.jobs_launcher=Thread(target=self._worker_job)
self.processing_jobs_checker=Thread(target=self._check_jobs_status)
self.jobs_launcher.start()
self.processing_jobs_checker.start()
def _worker_job(self):
while True:
# Run at max 3 jobs concurrently
if self.not_empty and len(self.processing_jobs) < 3:
# Get job from queue
job = self.get()
# Execute a task without blocking the thread
self.process = subprocess.Popen(job)
self.processing_jobs.append(self.process)
# util if queue.join() is used to block the queue
self.task_done()
else:
print("Waiting 4s for jobs")
sleep(4)
def _check_jobs_status(self):
while True:
# Check if jobs are finished
for job in self.processing_jobs:
# Sucessfully completed
if job.returncode == 0:
self.processing_jobs.remove(job)
# Wait 4 seconds and repeat
sleep(4)
def main():
q = JobQueueManager(100)
task = ["stress", "--cpu", "1", "--timeout", "20"]
for i in range(10): #put 10 tasks in the queue
q.put(task)
q.join() #block until all tasks are done
if __name__ == "__main__":
main()
I answer myself, I have come up with a working solution. The JobExecutor class handles in a custom way the Pool of processes. The watch_completed_tasks function tries to watch and handle the output of the tasks when they are done. This way everything is done with only two threads and the main thread is not blocked when submitting processes.
import subprocess
from threading import Timer
from concurrent.futures import ProcessPoolExecutor, as_completed
import logging
def launch_job(job):
process = subprocess.Popen(job, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(f"launching {process.pid}")
return [process.pid, process.stdout.read(), process.stderr.read()]
class JobExecutor(ProcessPoolExecutor):
def __init__(self, max_workers: int):
super().__init__(max_workers)
self.futures = []
self.watch_completed_tasks()
def submit(self, command):
future = super().submit(launch_job, command)
self.futures.append(future)
return future
def watch_completed_tasks(self):
# Manage tasks completion
for completed_task in as_completed(self.futures):
print(f"FINISHED task with PID {completed_task.result()[0]}")
self.futures.remove(completed_task)
# call this function evevery 5 seconds
timer_thread = Timer(5.0, self.watch_completed_tasks)
timer_thread.setName("TasksWatcher")
timer_thread.start()
def main():
executor = JobExecutor(max_workers=5)
for i in range(10):
task = ["stress",
"--cpu", "1",
"--timeout", str(i+5)]
executor.submit(task)
I create sub-class from multiprocessing.Process.
Object p.run() can update instance.ret_value from the long_runtime_proc, but p.start() can't get the ret_value updated though long_runtime_proc called and ran.
How can I get ret_value with p.start()?
*class myProcess (multiprocessing.Process):
def __init__(self, pid, name, ret_value=0):
multiprocessing.Process.__init__(self)
self.id = pid
self.ret_value = ret_value
def run(self):
self.ret_value = long_runtime_proc (self.id)*
Calling Process.run() directly does not start a new process, i.e. the code in Process.run() is executed in the same process that invoked it. That's why changes to self.ret_value are effective. However, you are not supposed to call Process.run() directly.
When you start the subprocess with Process.start() a new child process is created and then the code in Process.run() is executed in this new process. When you assign the return value of long_runtime_proc to self.ret_value, this occurs in the child process, not the parent and thus the parent ret_vaule is not updated.
What you probably need to do is to use a pipe or a queue to send the return value to the parent process. See the documentation for details. Here is an example using a queue:
import time
import multiprocessing
def long_runtime_proc():
'''Simulate a long running process'''
time.sleep(10)
return 1234
class myProcess(multiprocessing.Process):
def __init__(self, result_queue):
self.result_queue = result_queue
super(myProcess, self).__init__()
def run(self):
self.result_queue.put(long_runtime_proc())
q = multiprocessing.Queue()
p = myProcess(q)
p.start()
ret_value = q.get()
p.join()
With this code ret_value will end up being assigned the value off the queue which will be 1234.
I'm learning python multithreading and queues. The following creates a bunch of threads that pass data through a queue to another thread for printing:
import time
import threading
import Queue
queue = Queue.Queue()
def add(data):
return ["%sX" % x for x in data]
class PrintThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
data = self.queue.get()
print data
self.queue.task_done()
class MyThread(threading.Thread):
def __init__(self, queue, data):
threading.Thread.__init__(self)
self.queue = queue
self.data = data
def run(self):
self.queue.put(add(self.data))
if __name__ == "__main__":
a = MyThread(queue, ["a","b","c"])
a.start()
b = MyThread(queue, ["d","e","f"])
b.start()
c = MyThread(queue, ["g","h","i"])
c.start()
printme = PrintThread(queue)
printme.start()
queue.join()
However, I see only the data from the first thread print out:
['aX', 'bX', 'cX']
Then nothing else, but the program doesn't exit. I have to kill the process to have it exit.
Ideally, after each MyThread does it data processing and puts the result to the queue, that thread should exit? Simultaneously the PrintThread should take whatever is on the queue and print it.
After all MyThread threads have finished and the PrintThread thread has finished processing everything on the queue, the program should exit cleanly.
What have I done wrong?
EDIT:
If each MyThread thread takes a while to process, is there a way to guarantee that the PrintThread thread will wait for all the MyThread threads to finish before it will exit itself?
That way the print thread will definitely have processed every possible data on the queue because all the other threads have already exited.
For example,
class MyThread(threading.Thread):
def __init__(self, queue, data):
threading.Thread.__init__(self)
self.queue = queue
self.data = data
def run(self):
time.sleep(10)
self.queue.put(add(self.data))
The above modification will wait for 10 seconds before putting anything on the queue. The print thread will run, but I think it's exiting too early since there is not data on the queue yet, so the program prints out nothing.
Your PrintThread does not loop but instead only prints out a single queue item and then stops running.
Therefore, the queue will never be empty and the queue.join() statement will prevent the main program from terminating
Change the run() method of your PrintThread into the following code in order to have all queue items processed:
try:
while True:
data = self.queue.get_nowait()
print data
self.queue.task_done()
except queue.Empty:
# All items have been taken off the queue
pass
Does
import multiprocessing
import schedule
def worker():
#do some stuff
def sched(argv):
schedule.every(0.01).minutes.do(worker)
while True:
schedule.run_pending()
processs = []
..
..
p = multiprocessing.Process(target=sched,args)
..
..
processs.append(p)
for p in processs:
p.terminate()
kills gracefully a list of processes ?
If not what is the simplest way to do it ?
The goal is to reload the configuration file into memory, so I would like to kill all children processes and create others instead, those latter will read the new config file.
Edit : Added more code to explain that I am running a while True loop
Edit : This is the new code after #dano suggestion
def get_config(self):
from ConfigParser import SafeConfigParser
..
return argv
def sched(self, args, event):
#schedule instruction:
schedule.every(0.01).minutes.do(self.worker,args)
while not event.is_set():
schedule.run_pending()
def dispatch_processs(self, conf):
processs = []
event = multiprocessing.Event()
for conf in self.get_config():
process = multiprocessing.Process(target=self.sched,args=( i for i in conf), kwargs={'event' : event})
processs.append((process, event)
return processs
def start_process(self, process):
process.start()
def gracefull_process(self, process):
process.join()
def main(self):
while True:
processs = self.dispatch_processs(self.get_config())
print ("%s processes running " % len(processs) )
for process, event in processs:
self.start_process(process)
time.sleep(1)
event.set()
self.gracefull_process(process)
The good thing about the code, is that I can edit config file and the process will reload its config also.
The problem is that only the first process runs and the others are ignored.
Edit : This saved my life , working with while True in schedule() is not a good idea, so I set up refresh_time instead
def sched(self, args, event):
schedule.every(0.01).minutes.do(self.worker,args)
for i in range(refresh_time):
schedule.run_pending()
time.sleep(1)
def start_processs(self, processs):
for p,event in processs:
if not p.is_alive():
p.start()
time.sleep(1)
event.set()
self.gracefull_processs(processs)
def gracefull_processs(self, processs):
for p,event in processs:
p.join()
processs = self.dispatch_processs(self.get_config())
self.start_processs(processs)
def main(self):
while True:
processs = self.dispatch_processs(self.get_config())
self.start_processs(processs)
break
print ("Reloading function main")
self.main()
If you don't mind only aborting after worker has completed all of its work, its very simple to add a multiprocessing.Event to handle exiting gracefully:
import multiprocessing
import schedule
def worker():
#do some stuff
def sched(argv, event=None):
schedule.every(0.01).minutes.do(worker)
while not event.is_set(): # Run until we're told to shut down.
schedule.run_pending()
processes = []
..
..
event = multiprocessing.Event()
p = multiprocessing.Process(target=sched,args, kwargs={'event' : event})
..
..
processes.append((p, event))
# Tell all processes to shut down
for _, event in processes:
event.set()
# Now actually wait for them to shut down
for p, _ in processes:
p.join()
A: No, both .terminate() & SIG_* methods are rather brutal
In a need to arrange a gracefull end of any process, as described in your post, there rather shall be some "soft-signalling" layer, that allows, on both ends, to send/receive smart-signalls without being dependent on the O/S interpretations ( O/S knows nothing about your application-level context and state of the respective work-unit, that is currently being processed ).
You may want to read about such soft-signalling approach in links referred from >>> https://stackoverflow.com/a/25373416/3666197
No, it doesn't kill a process according to your own definition of gracefully - unless you take some additional steps.
Assuming you're using a unix system (since you mentioned scp), terminate sends a SIGTERM signal to the child process. You can catch this signal in the child process, and act accordingly (wait for scp to finish):
import signal
def on_terminate(signum, stack):
wait_for_current_scp_operation()
signal.signal(signal.SIGTERM, on_terminate)
Here's a tutorial about handling and sending signals